From 7d7f167ecb7726b76e66045c1f323392be6b6c68 Mon Sep 17 00:00:00 2001 From: Wondertan Date: Fri, 20 Sep 2024 14:10:19 +0200 Subject: [PATCH] fix(shwap/bitswap): bump boxo and update blockstore options --- go.mod | 4 ++-- go.sum | 6 ++++++ share/shwap/p2p/bitswap/bitswap.go | 4 ++++ share/shwap/p2p/bitswap/block_fetch_test.go | 4 ++++ share/shwap/p2p/bitswap/block_store.go | 16 +++++++++++----- 5 files changed, 27 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 9cef3d24fc..dfc2d9f83c 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.7 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/imdario/mergo v0.3.16 - github.com/ipfs/boxo v0.22.0 + github.com/ipfs/boxo v0.23.1-0.20240920171645-1364a16755c3 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 @@ -39,7 +39,7 @@ require ( github.com/ipfs/go-ipld-format v0.6.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/klauspost/reedsolomon v1.12.1 - github.com/libp2p/go-libp2p v0.36.2 + github.com/libp2p/go-libp2p v0.36.3 github.com/libp2p/go-libp2p-kad-dht v0.26.1 github.com/libp2p/go-libp2p-pubsub v0.12.0 github.com/libp2p/go-libp2p-record v0.2.0 diff --git a/go.sum b/go.sum index 024131f320..986db6464c 100644 --- a/go.sum +++ b/go.sum @@ -1011,6 +1011,10 @@ github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/ipfs/boxo v0.22.0 h1:QTC+P5uhsBNq6HzX728nsLyFW6rYDeR/5hggf9YZX78= github.com/ipfs/boxo v0.22.0/go.mod h1:yp1loimX0BDYOR0cyjtcXHv15muEh5V1FqO2QLlzykw= +github.com/ipfs/boxo v0.23.1-0.20240914020010-ff77293b4d47 h1:+Lb6B7Y6XASZiaPDn2VkB0y4w2pJWRRv7SDfuIO+Ovs= +github.com/ipfs/boxo v0.23.1-0.20240914020010-ff77293b4d47/go.mod h1:ulu5I6avTmgGmvjuCaBRKwsaOOKjBfQw1EiOOQp8M6E= +github.com/ipfs/boxo v0.23.1-0.20240920171645-1364a16755c3 h1:STC1B6+L6toikFAHKCFvEheaWu9U+gPn1EuP3WnFfBw= +github.com/ipfs/boxo v0.23.1-0.20240920171645-1364a16755c3/go.mod h1:ulu5I6avTmgGmvjuCaBRKwsaOOKjBfQw1EiOOQp8M6E= github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= @@ -1179,6 +1183,8 @@ github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnF github.com/libp2p/go-libp2p v0.19.0/go.mod h1:Ki9jJXLO2YqrTIFxofV7Twyd3INWPT97+r8hGt7XPjI= github.com/libp2p/go-libp2p v0.36.2 h1:BbqRkDaGC3/5xfaJakLV/BrpjlAuYqSB0lRvtzL3B/U= github.com/libp2p/go-libp2p v0.36.2/go.mod h1:XO3joasRE4Eup8yCTTP/+kX+g92mOgRaadk46LmPhHY= +github.com/libp2p/go-libp2p v0.36.3 h1:NHz30+G7D8Y8YmznrVZZla0ofVANrvBl2c+oARfMeDQ= +github.com/libp2p/go-libp2p v0.36.3/go.mod h1:4Y5vFyCUiJuluEPmpnKYf6WFx5ViKPUYs/ixe9ANFZ8= github.com/libp2p/go-libp2p-asn-util v0.1.0/go.mod h1:wu+AnM9Ii2KgO5jMmS1rz9dvzTdj8BXqsPR9HR0XB7I= github.com/libp2p/go-libp2p-asn-util v0.4.1 h1:xqL7++IKD9TBFMgnLPZR6/6iYhawHKHl950SO9L6n94= github.com/libp2p/go-libp2p-asn-util v0.4.1/go.mod h1:d/NI6XZ9qxw67b4e+NgpQexCIiFYJjErASrYW4PFDN8= diff --git a/share/shwap/p2p/bitswap/bitswap.go b/share/shwap/p2p/bitswap/bitswap.go index f63efe4012..95f2f72913 100644 --- a/share/shwap/p2p/bitswap/bitswap.go +++ b/share/shwap/p2p/bitswap/bitswap.go @@ -64,6 +64,9 @@ const ( // We set it to be equal to targetMessageSize * N, so there can max N messages being prepared for // a peer at once. outstandingBytesPerPeer = targetMessageSize * 4 + // replaceHasWithBlockMaxSize configures Bitswap to use Has method instead of GetSize to check existence + // of a CID in Blockstore. + replaceHasWithBlockMaxSize = 0 ) // NewNetwork constructs Bitswap network for Shwap protocol composition. @@ -114,6 +117,7 @@ func NewServer( server.MaxQueuedWantlistEntriesPerPeer(maxServerWantListsPerPeer), server.WithTargetMessageSize(targetMessageSize), server.MaxOutstandingBytesPerPeer(outstandingBytesPerPeer), + server.WithReplaceHasWithBlockMaxSize(replaceHasWithBlockMaxSize), } return server.New(ctx, net, bstore, opts...) } diff --git a/share/shwap/p2p/bitswap/block_fetch_test.go b/share/shwap/p2p/bitswap/block_fetch_test.go index 6642801efe..05b0c78b19 100644 --- a/share/shwap/p2p/bitswap/block_fetch_test.go +++ b/share/shwap/p2p/bitswap/block_fetch_test.go @@ -159,6 +159,10 @@ func (t testAccessorGetter) GetByHeight(context.Context, uint64) (eds.AccessorSt return t.AccessorStreamer, nil } +func (t testAccessorGetter) HasByHeight(context.Context, uint64) (bool, error) { + return true, nil +} + type testFetcher struct { Fetched int diff --git a/share/shwap/p2p/bitswap/block_store.go b/share/shwap/p2p/bitswap/block_store.go index dffa45e63f..fa7c141684 100644 --- a/share/shwap/p2p/bitswap/block_store.go +++ b/share/shwap/p2p/bitswap/block_store.go @@ -18,6 +18,8 @@ import ( type AccessorGetter interface { // GetByHeight returns an Accessor by its height. GetByHeight(ctx context.Context, height uint64) (eds.AccessorStreamer, error) + // HasByHeight reports whether an Accessor for the height exists. + HasByHeight(ctx context.Context, height uint64) (bool, error) } // Blockstore implements generalized Bitswap compatible storage over Shwap containers @@ -64,10 +66,9 @@ func (b *Blockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) } func (b *Blockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) { - // TODO(@Wondertan): There must be a way to derive size without reading, proving, serializing and - // allocating Sample's block.Block or we could do hashing - // NOTE:Bitswap uses GetSize also to determine if we have content stored or not - // so simply returning constant size is not an option + // TODO(@Wondertan): Bitswap checks the size of the data(GetSize) before serving it via Get. This means + // GetSize may do an unnecessary read from disk which we can avoid by either caching on Blockstore level + // or returning constant size(we know at that point that we have requested data) blk, err := b.Get(ctx, cid) if err != nil { return 0, err @@ -76,10 +77,15 @@ func (b *Blockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) { } func (b *Blockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) { - _, err := b.Get(ctx, cid) + blk, err := EmptyBlock(cid) if err != nil { return false, err } + + _, err = b.Getter.HasByHeight(ctx, blk.Height()) + if err != nil { + return false, fmt.Errorf("checking EDS Accessor for height %v: %w", blk.Height(), err) + } return true, nil }