From 143c9f8f44d54c2a289adccc788722783a61d71f Mon Sep 17 00:00:00 2001 From: Giulio rebuffo Date: Mon, 15 Apr 2024 17:43:09 +0200 Subject: [PATCH 1/5] Caplin: Fixed deadline exceeded in block download (#9943) --- cl/phase1/execution_client/execution_client_direct.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cl/phase1/execution_client/execution_client_direct.go b/cl/phase1/execution_client/execution_client_direct.go index 977bed8e499..389442d59cd 100644 --- a/cl/phase1/execution_client/execution_client_direct.go +++ b/cl/phase1/execution_client/execution_client_direct.go @@ -86,7 +86,7 @@ func (cc *ExecutionClientDirect) SupportInsertion() bool { } func (cc *ExecutionClientDirect) InsertBlocks(ctx context.Context, blocks []*types.Block, wait bool) error { - if !wait { + if wait { return cc.chainRW.InsertBlocksAndWait(ctx, blocks) } return cc.chainRW.InsertBlocks(ctx, blocks) From b1efa162072375534d5d446c9f7f1cd1d2d5d4ee Mon Sep 17 00:00:00 2001 From: milen <94537774+taratorio@users.noreply.github.com> Date: Mon, 15 Apr 2024 19:06:30 +0300 Subject: [PATCH 2/5] polygon/p2p: remove no longer needed trackingPenalizer (#9929) After the fix done in https://github.com/ledgerwatch/erigon/pull/9928 trackingPenalizer is no longer needed --- polygon/p2p/fetcher_tracking_test.go | 3 ++- polygon/p2p/peer_penalizer.go | 17 ----------------- polygon/p2p/peer_tracker.go | 12 +++++++++--- polygon/p2p/peer_tracker_test.go | 5 ++++- polygon/p2p/service.go | 4 ++-- 5 files changed, 17 insertions(+), 24 deletions(-) diff --git a/polygon/p2p/fetcher_tracking_test.go b/polygon/p2p/fetcher_tracking_test.go index 63b586f5c79..89d9a340951 100644 --- a/polygon/p2p/fetcher_tracking_test.go +++ b/polygon/p2p/fetcher_tracking_test.go @@ -161,8 +161,9 @@ func TestTrackingFetcherFetchBodiesUpdatesPeerTracker(t *testing.T) { func newTrackingFetcherTest(t *testing.T, requestIdGenerator RequestIdGenerator) *trackingFetcherTest { fetcherTest := newFetcherTest(t, requestIdGenerator) + logger := fetcherTest.logger peerTracker := newPeerTracker(PreservingPeerShuffle) - unregister := fetcherTest.messageListener.RegisterPeerEventObserver(NewPeerEventObserver(peerTracker)) + unregister := fetcherTest.messageListener.RegisterPeerEventObserver(NewPeerEventObserver(logger, peerTracker)) t.Cleanup(unregister) trackingFetcher := newTrackingFetcher(fetcherTest.fetcher, peerTracker) return &trackingFetcherTest{ diff --git a/polygon/p2p/peer_penalizer.go b/polygon/p2p/peer_penalizer.go index 72ed9598689..baec519723b 100644 --- a/polygon/p2p/peer_penalizer.go +++ b/polygon/p2p/peer_penalizer.go @@ -11,23 +11,6 @@ type PeerPenalizer interface { Penalize(ctx context.Context, peerId *PeerId) error } -func NewTrackingPeerPenalizer(peerPenalizer PeerPenalizer, peerTracker PeerTracker) PeerPenalizer { - return &trackingPeerPenalizer{ - PeerPenalizer: peerPenalizer, - peerTracker: peerTracker, - } -} - -type trackingPeerPenalizer struct { - PeerPenalizer - peerTracker PeerTracker -} - -func (p *trackingPeerPenalizer) Penalize(ctx context.Context, peerId *PeerId) error { - p.peerTracker.PeerDisconnected(peerId) - return p.PeerPenalizer.Penalize(ctx, peerId) -} - func NewPeerPenalizer(sentryClient direct.SentryClient) PeerPenalizer { return &peerPenalizer{ sentryClient: sentryClient, diff --git a/polygon/p2p/peer_tracker.go b/polygon/p2p/peer_tracker.go index e3760fd4f99..9ec620f604e 100644 --- a/polygon/p2p/peer_tracker.go +++ b/polygon/p2p/peer_tracker.go @@ -3,6 +3,8 @@ package p2p import ( "sync" + "github.com/ledgerwatch/log/v3" + "github.com/ledgerwatch/erigon-lib/gointerfaces/sentry" ) @@ -90,13 +92,17 @@ func (pt *peerTracker) updatePeerSyncProgress(peerId *PeerId, update func(psp *p update(peerSyncProgress) } -func NewPeerEventObserver(peerTracker PeerTracker) MessageObserver[*sentry.PeerEvent] { +func NewPeerEventObserver(logger log.Logger, peerTracker PeerTracker) MessageObserver[*sentry.PeerEvent] { return func(message *sentry.PeerEvent) { + peerId := PeerIdFromH512(message.PeerId) + + logger.Debug("[p2p.peerEventObserver] received new peer event", "id", message.EventId, "peerId", peerId) + switch message.EventId { case sentry.PeerEvent_Connect: - peerTracker.PeerConnected(PeerIdFromH512(message.PeerId)) + peerTracker.PeerConnected(peerId) case sentry.PeerEvent_Disconnect: - peerTracker.PeerDisconnected(PeerIdFromH512(message.PeerId)) + peerTracker.PeerDisconnected(peerId) } } } diff --git a/polygon/p2p/peer_tracker_test.go b/polygon/p2p/peer_tracker_test.go index 8359141ae54..a8aa8e4b5a7 100644 --- a/polygon/p2p/peer_tracker_test.go +++ b/polygon/p2p/peer_tracker_test.go @@ -7,9 +7,11 @@ import ( "testing" "time" + "github.com/ledgerwatch/log/v3" "github.com/stretchr/testify/require" "github.com/ledgerwatch/erigon-lib/gointerfaces/sentry" + "github.com/ledgerwatch/erigon/turbo/testlog" ) func TestPeerTracker(t *testing.T) { @@ -48,8 +50,9 @@ func TestPeerTracker(t *testing.T) { func TestPeerTrackerPeerEventObserver(t *testing.T) { t.Parallel() + logger := testlog.Logger(t, log.LvlInfo) peerTracker := newPeerTracker(PreservingPeerShuffle) - peerTrackerPeerEventObserver := NewPeerEventObserver(peerTracker) + peerTrackerPeerEventObserver := NewPeerEventObserver(logger, peerTracker) messageListenerTest := newMessageListenerTest(t) messageListenerTest.mockSentryStreams() messageListenerTest.run(func(ctx context.Context, t *testing.T) { diff --git a/polygon/p2p/service.go b/polygon/p2p/service.go index b8f3108b0b6..06ecbf24c58 100644 --- a/polygon/p2p/service.go +++ b/polygon/p2p/service.go @@ -43,9 +43,9 @@ func newService( requestIdGenerator RequestIdGenerator, ) *service { peerTracker := NewPeerTracker() - peerPenalizer := NewTrackingPeerPenalizer(NewPeerPenalizer(sentryClient), peerTracker) + peerPenalizer := NewPeerPenalizer(sentryClient) messageListener := NewMessageListener(logger, sentryClient, statusDataFactory, peerPenalizer) - messageListener.RegisterPeerEventObserver(NewPeerEventObserver(peerTracker)) + messageListener.RegisterPeerEventObserver(NewPeerEventObserver(logger, peerTracker)) messageSender := NewMessageSender(sentryClient) fetcher := NewFetcher(fetcherConfig, messageListener, messageSender, requestIdGenerator) fetcher = NewPenalizingFetcher(logger, fetcher, peerPenalizer) From 1e72f64a0ee06cb2cf5f6cf0e73d01aebe2d10fc Mon Sep 17 00:00:00 2001 From: milen <94537774+taratorio@users.noreply.github.com> Date: Mon, 15 Apr 2024 19:20:18 +0300 Subject: [PATCH 3/5] polygon/heimdall: use value receiver for checkpoints sort (#9947) minor tidy up of code I wrote last week --- polygon/heimdall/checkpoint.go | 14 ++++++-------- polygon/heimdall/heimdall.go | 2 +- 2 files changed, 7 insertions(+), 9 deletions(-) diff --git a/polygon/heimdall/checkpoint.go b/polygon/heimdall/checkpoint.go index 81b994a6f47..9c43ac56faa 100644 --- a/polygon/heimdall/checkpoint.go +++ b/polygon/heimdall/checkpoint.go @@ -63,18 +63,16 @@ func (c *Checkpoint) UnmarshalJSON(b []byte) error { type Checkpoints []*Checkpoint -func (cs *Checkpoints) Len() int { - return len(*cs) +func (cs Checkpoints) Len() int { + return len(cs) } -func (cs *Checkpoints) Less(i, j int) bool { - v := *cs - return v[i].StartBlock().Uint64() < v[j].StartBlock().Uint64() +func (cs Checkpoints) Less(i, j int) bool { + return cs[i].StartBlock().Uint64() < cs[j].StartBlock().Uint64() } -func (cs *Checkpoints) Swap(i, j int) { - v := *cs - v[i], v[j] = v[j], v[i] +func (cs Checkpoints) Swap(i, j int) { + cs[i], cs[j] = cs[j], cs[i] } type CheckpointResponse struct { diff --git a/polygon/heimdall/heimdall.go b/polygon/heimdall/heimdall.go index 0c6f04b1917..fe7a58ef435 100644 --- a/polygon/heimdall/heimdall.go +++ b/polygon/heimdall/heimdall.go @@ -633,7 +633,7 @@ func (h *heimdall) batchFetchCheckpoints( page++ } - sort.Sort(&checkpoints) + sort.Sort(checkpoints) for i, checkpoint := range checkpoints[lastStored:] { err := store.PutCheckpoint(ctx, CheckpointId(i+1), checkpoint) From f5692a5db49c48263f8ab51b0b179cd0aa4d11dc Mon Sep 17 00:00:00 2001 From: awskii Date: Mon, 15 Apr 2024 17:35:39 +0100 Subject: [PATCH 4/5] changed behaviour --webseed flag and removed check of s3 etag format (#9946) --- cmd/downloader/main.go | 12 +++++++----- erigon-lib/downloader/webseed.go | 9 +++++---- 2 files changed, 12 insertions(+), 9 deletions(-) diff --git a/cmd/downloader/main.go b/cmd/downloader/main.go index 649660be3a3..b91e3db0cc6 100644 --- a/cmd/downloader/main.go +++ b/cmd/downloader/main.go @@ -403,8 +403,10 @@ var torrentMagnet = &cobra.Command{ func manifestVerify(ctx context.Context, logger log.Logger) error { webseedsList := common.CliString2Array(webseeds) - if known, ok := snapcfg.KnownWebseeds[chain]; ok { - webseedsList = append(webseedsList, known...) + if len(webseedsList) == 0 { + if known, ok := snapcfg.KnownWebseeds[chain]; ok { + webseedsList = append(webseedsList, known...) + } } webseedUrlsOrFiles := webseedsList @@ -438,9 +440,9 @@ func manifestVerify(ctx context.Context, logger log.Logger) error { continue } } - - _ = webseedFileProviders // todo add support of file providers - logger.Warn("file providers are not supported yet", "fileProviders", webseedFileProviders) + if len(webseedFileProviders) > 0 { + logger.Warn("file providers are not supported yet", "fileProviders", webseedFileProviders) + } wseed := downloader.NewWebSeeds(webseedHttpProviders, log.LvlDebug, logger) return wseed.VerifyManifestedBuckets(ctx, verifyFailfast) diff --git a/erigon-lib/downloader/webseed.go b/erigon-lib/downloader/webseed.go index fcd3b6fe2c6..a1676fc9070 100644 --- a/erigon-lib/downloader/webseed.go +++ b/erigon-lib/downloader/webseed.go @@ -455,10 +455,11 @@ func (d *WebSeeds) retrieveFileEtag(ctx context.Context, file *url.URL) (string, if etag == "" { return "", fmt.Errorf("webseed.http: file has no etag, url=%s", file.String()) } - etag = strings.Trim(etag, "\"") - if strings.Contains(etag, "-") { - return etag, ErrInvalidEtag - } + // Todo(awskii): figure out reason why multipart etags contains "-" and remove this check + //etag = strings.Trim(etag, "\"") + //if strings.Contains(etag, "-") { + // return etag, ErrInvalidEtag + //} return etag, nil } From 6a9ce5a463a0b9f1104f9e1fe3b988759af2e24b Mon Sep 17 00:00:00 2001 From: Alex Sharov Date: Tue, 16 Apr 2024 02:47:21 +0700 Subject: [PATCH 5/5] manifest-verify: allow `host.com/v2/manifest.txt` (#9942) --- erigon-lib/downloader/webseed.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/erigon-lib/downloader/webseed.go b/erigon-lib/downloader/webseed.go index a1676fc9070..8a01c769440 100644 --- a/erigon-lib/downloader/webseed.go +++ b/erigon-lib/downloader/webseed.go @@ -465,11 +465,8 @@ func (d *WebSeeds) retrieveFileEtag(ctx context.Context, file *url.URL) (string, func (d *WebSeeds) retrieveManifest(ctx context.Context, webSeedProviderUrl *url.URL) (snaptype.WebSeedsFromProvider, error) { baseUrl := webSeedProviderUrl.String() - ref, err := url.Parse("manifest.txt") - if err != nil { - return nil, err - } - u := webSeedProviderUrl.ResolveReference(ref) + webSeedProviderUrl.Path += "/manifest.txt" // allow: host.com/v2/manifest.txt + u := webSeedProviderUrl request, err := http.NewRequest(http.MethodGet, u.String(), nil) if err != nil { return nil, err