feat: add shares inclusion proofs

app/app.go

@@ -493,6 +493,7 @@ func New(
 	)
 
 	app.QueryRouter().AddRoute(prove.TxInclusionQueryPath, prove.QueryTxInclusionProof)
+	app.QueryRouter().AddRoute(prove.ShareInclusionQueryPath, prove.QueryShareInclusionProof)
 
 	app.mm.RegisterInvariants(&app.CrisisKeeper)
 	app.mm.RegisterRoutes(app.Router(), app.QueryRouter(), encodingConfig.Amino)

app/test/block_size_test.go → app/test/integration_test.go

@@ -6,22 +6,22 @@ import (
 	"testing"
 	"time"
 
-	"github.com/cosmos/cosmos-sdk/client"
-	"github.com/cosmos/cosmos-sdk/crypto/keyring"
+	"github.com/celestiaorg/celestia-app/testutil/blobfactory"
+	blobtypes "github.com/celestiaorg/celestia-app/x/blob/types"
 	"github.com/stretchr/testify/require"
-	"github.com/stretchr/testify/suite"
 
+	"github.com/cosmos/cosmos-sdk/client"
+	"github.com/cosmos/cosmos-sdk/crypto/keyring"
 	cosmosnet "github.com/cosmos/cosmos-sdk/testutil/network"
+	sdk "github.com/cosmos/cosmos-sdk/types"
+	"github.com/stretchr/testify/suite"
 
 	"github.com/celestiaorg/celestia-app/app"
 	"github.com/celestiaorg/celestia-app/app/encoding"
 	"github.com/celestiaorg/celestia-app/pkg/appconsts"
-	"github.com/celestiaorg/celestia-app/testutil/blobfactory"
+	"github.com/celestiaorg/celestia-app/pkg/prove"
 	"github.com/celestiaorg/celestia-app/testutil/network"
 	"github.com/celestiaorg/celestia-app/x/blob"
-	blobtypes "github.com/celestiaorg/celestia-app/x/blob/types"
-
-	sdk "github.com/cosmos/cosmos-sdk/types"
 	abci "github.com/tendermint/tendermint/abci/types"
 	tmrand "github.com/tendermint/tendermint/libs/rand"
 	rpctypes "github.com/tendermint/tendermint/rpc/core/types"
@@ -284,3 +284,73 @@ func queryTx(clientCtx client.Context, hashHexStr string, prove bool) (*rpctypes
 
 	return node.Tx(context.Background(), hash, prove)
 }
+
+func (s *IntegrationTestSuite) TestSharesInclusionProof() {
+	require := s.Require()
+	val := s.network.Validators[0]
+
+	// generate 100 randomly sized txs (max size == 100kb)
+	txs := blobfactory.RandBlobTxsWithAccounts(
+		s.cfg.TxConfig.TxEncoder(),
+		s.kr,
+		val.ClientCtx.GRPCClient,
+		100000,
+		true,
+		s.cfg.ChainID,
+		s.accounts[:20],
+	)
+
+	hashes := make([]string, len(txs))
+
+	for i, tx := range txs {
+		res, err := val.ClientCtx.BroadcastTxSync(tx)
+		require.NoError(err)
+		require.Equal(abci.CodeTypeOK, res.Code)
+		hashes[i] = res.TxHash
+	}
+
+	// wait a few blocks to clear the txs
+	for i := 0; i < 20; i++ {
+		require.NoError(s.network.WaitForNextBlock())
+	}
+
+	for _, hash := range hashes {
+		txResp, err := queryTx(val.ClientCtx, hash, true)
+		require.NoError(err)
+		require.Equal(abci.CodeTypeOK, txResp.TxResult.Code)
+
+		// verify that the transaction inclusion proof is valid
+		require.True(txResp.Proof.VerifyProof())
+
+		// get the transaction shares
+		node, err := val.ClientCtx.GetNode()
+		require.NoError(err)
+		blockRes, err := node.Block(context.Background(), &txResp.Height)
+		require.NoError(err)
+		beginTxShare, endTxShare, err := prove.TxSharePosition(blockRes.Block.Txs, uint64(txResp.Index))
+		require.NoError(err)
+
+		txProof, err := node.ProveShares(
+			context.Background(),
+			uint64(txResp.Height),
+			beginTxShare,
+			endTxShare,
+		)
+		require.NoError(err)
+		require.NoError(txProof.Validate(blockRes.Block.DataHash))
+
+		// get the message shares
+		beginMsgShare, endMsgShare, err := prove.MsgSharesPosition(blockRes.Block.Txs[txResp.Index])
+		require.NoError(err)
+
+		// verify the message shares proof
+		msgProof, err := node.ProveShares(
+			context.Background(),
+			uint64(txResp.Height),
+			beginMsgShare,
+			endMsgShare,
+		)
+		require.NoError(err)
+		require.NoError(msgProof.Validate(blockRes.Block.DataHash))
+	}
+}

pkg/prove/proof.go

@@ -1,24 +1,34 @@
 package prove
 
 import (
+	"bytes"
 	"errors"
+	"fmt"
 
+	sdk "github.com/cosmos/cosmos-sdk/types"
+
+	"github.com/celestiaorg/celestia-app/app/encoding"
 	"github.com/celestiaorg/celestia-app/pkg/appconsts"
+	"github.com/celestiaorg/celestia-app/pkg/da"
 	"github.com/celestiaorg/celestia-app/pkg/shares"
 	"github.com/celestiaorg/celestia-app/pkg/wrapper"
+	blobmodule "github.com/celestiaorg/celestia-app/x/blob"
+	blobtypes "github.com/celestiaorg/celestia-app/x/blob/types"
+	"github.com/celestiaorg/nmt/namespace"
 	"github.com/celestiaorg/rsmt2d"
+	"github.com/tendermint/tendermint/crypto/merkle"
 	tmbytes "github.com/tendermint/tendermint/libs/bytes"
 	tmproto "github.com/tendermint/tendermint/proto/tendermint/types"
 	"github.com/tendermint/tendermint/types"
 )
 
 // TxInclusion uses the provided block data to progressively generate rows
-// of a data square, and then using those shares to creates nmt inclusion proofs
+// of a data square, and then using those shares to creates nmt inclusion proofs.
 // It is possible that a transaction spans more than one row. In that case, we
 // have to return more than one proof.
 func TxInclusion(codec rsmt2d.Codec, data types.Data, txIndex uint64) (types.TxProof, error) {
 	// calculate the index of the shares that contain the tx
-	startPos, endPos, err := txSharePosition(data.Txs, txIndex)
+	startPos, endPos, err := TxSharePosition(data.Txs, txIndex)
 	if err != nil {
 		return types.TxProof{}, err
 	}
@@ -84,10 +94,10 @@ func TxInclusion(codec rsmt2d.Codec, data types.Data, txIndex uint64) (types.TxP
 	}, nil
 }
 
-// txSharePosition returns the start and end positions for the shares that
+// TxSharePosition returns the start and end positions for the shares that
 // include a given txIndex. Returns an error if index is greater than the length
 // of txs.
-func txSharePosition(txs types.Txs, txIndex uint64) (startSharePos, endSharePos uint64, err error) {
+func TxSharePosition(txs types.Txs, txIndex uint64) (startSharePos, endSharePos uint64, err error) {
 	if txIndex >= uint64(len(txs)) {
 		return startSharePos, endSharePos, errors.New("transaction index is greater than the number of txs")
 	}
@@ -107,6 +117,49 @@ func txSharePosition(txs types.Txs, txIndex uint64) (startSharePos, endSharePos
 	return startSharePos, endSharePos, nil
 }
 
+// MsgSharesPosition returns the start and end positions for the shares
+// where a given message, referenced by its wrapped pfb transaction, was published at.
+// Note: only supports transactions containing a single message
+func MsgSharesPosition(tx types.Tx) (beginShare uint64, endShare uint64, err error) {
+	unwrappedTx, isMalleated := types.UnmarshalIndexWrapper(tx)
+	if !isMalleated {
+		return beginShare, endShare, fmt.Errorf("not a malleated tx")
+	}
+
+	encCfg := encoding.MakeConfig(blobmodule.AppModuleBasic{})
+	decoder := encCfg.TxConfig.TxDecoder()
+
+	decodedTx, err := decoder(unwrappedTx.Tx)
+	if err != nil {
+		return beginShare, endShare, err
+	}
+
+	if len(decodedTx.GetMsgs()) == 0 {
+		return beginShare, endShare, fmt.Errorf("pfb contains no messages")
+	}
+
+	if len(decodedTx.GetMsgs()) > 1 {
+		return beginShare, endShare, fmt.Errorf("pfb containing multiple messages. not currently supported")
+	}
+
+	if sdk.MsgTypeURL(decodedTx.GetMsgs()[0]) != blobtypes.URLMsgPayForBlob {
+		return beginShare, endShare, fmt.Errorf("transaction is not pfb")
+	}
+
+	pfb, ok := decodedTx.GetMsgs()[0].(*blobtypes.MsgPayForBlob)
+	if !ok {
+		return beginShare, endShare, fmt.Errorf("unable to decode pfb")
+	}
+
+	if err = pfb.ValidateBasic(); err != nil {
+		return beginShare, endShare, err
+	}
+
+	beginShare = uint64(unwrappedTx.ShareIndexes[0])
+	sharesUsed := shares.SparseSharesNeeded(pfb.BlobSize)
+	return beginShare, beginShare + uint64(sharesUsed) - 1, nil
+}
+
 // txShareIndex returns the index of the compact share that would contain
 // transactions with totalTxLen
 func txShareIndex(totalTxLen int) (index uint64) {
@@ -193,3 +246,96 @@ func splitIntoRows(squareSize uint64, s []shares.Share) [][]shares.Share {
 	}
 	return rows
 }
+
+// SharesInclusion generates an nmt inclusion proof for a set of shares to the data root.
+// expects the shares range to be pre-validated.
+// Note: only supports inclusion proofs for shares belonging to the same namespace.
+func SharesInclusion(
+	allRawShares []shares.Share,
+	squareSize uint64,
+	namespaceID namespace.ID,
+	startShare uint64,
+	endShare uint64,
+) (types.SharesProof, error) {
+	startRow := startShare / squareSize
+	endRow := endShare / squareSize
+	startLeaf := startShare % squareSize
+	endLeaf := endShare % squareSize
+
+	eds, err := da.ExtendShares(squareSize, shares.ToBytes(allRawShares))
+	if err != nil {
+		return types.SharesProof{}, err
+	}
+
+	edsRowRoots := eds.RowRoots()
+
+	// create the binary merkle inclusion proof, for all the square rows, to the data root
+	_, allProofs := merkle.ProofsFromByteSlices(append(edsRowRoots, eds.ColRoots()...))
+	rowsProofs := make([]*merkle.Proof, endRow-startRow+1)
+	rowsRoots := make([]tmbytes.HexBytes, endRow-startRow+1)
+	for i := startRow; i <= endRow; i++ {
+		rowsProofs[i-startRow] = allProofs[i]
+		rowsRoots[i-startRow] = edsRowRoots[i]
+	}
+
+	// get the extended rows containing the shares.
+	rows := make([][]shares.Share, endRow-startRow+1)
+	for i := startRow; i <= endRow; i++ {
+		rows[i-startRow] = shares.FromBytes(eds.Row(uint(i)))
+	}
+
+	var sharesProofs []*tmproto.NMTProof //nolint:prealloc
+	var rawShares [][]byte
+	for i, row := range rows {
+		// create an nmt to generate a proof.
+		// we have to re-create the tree as the eds one is not accessible.
+		tree := wrapper.NewErasuredNamespacedMerkleTree(squareSize, uint(i))
+		for _, share := range row {
+			tree.Push(
+				share,
+			)
+		}
+
+		startLeafPos := startLeaf
+		endLeafPos := endLeaf
+
+		// if this is not the first row, then start with the first leaf
+		if i > 0 {
+			startLeafPos = 0
+		}
+		// if this is not the last row, then select for the rest of the row
+		if i != (len(rows) - 1) {
+			endLeafPos = squareSize - 1
+		}
+
+		rawShares = append(rawShares, shares.ToBytes(row[startLeafPos:endLeafPos+1])...)
+		proof, err := tree.Tree().ProveRange(int(startLeafPos), int(endLeafPos+1))
+		if err != nil {
+			return types.SharesProof{}, err
+		}
+
+		sharesProofs = append(sharesProofs, &tmproto.NMTProof{
+			Start:    int32(proof.Start()),
+			End:      int32(proof.End()),
+			Nodes:    proof.Nodes(),
+			LeafHash: proof.LeafHash(),
+		})
+
+		// make sure that the generated root is the same as the eds row root.
+		if !bytes.Equal(rowsRoots[i].Bytes(), tree.Root()) {
+			return types.SharesProof{}, errors.New("eds row root is different than tree root")
+		}
+	}
+
+	return types.SharesProof{
+		RowsProof: types.RowsProof{
+			RowsRoots: rowsRoots,
+			Proofs:    rowsProofs,
+			StartRow:  uint32(startRow),
+			EndRow:    uint32(endRow),
+		},
+		Data:         rawShares,
+		SharesProofs: sharesProofs,
+		NamespaceID:  namespaceID,
+	}, nil
+}