Data Transfer V2 Integration

Voucher refactor integration (#707)

* refactor(storagemarket): update storagemarket

* refactor(retrievalmarket): use new voucher system

refactor for predictability and correctness using new voucher system

* chore(deps): update go-data-transfer v2

* style(lint): prep for pr

* docs(retrievalmarket): add comments

* chore(deps): update go-statemachine

* style(imports): fix imports

chore(retrievalmarket): remove old types (#712)

feat(ipld): bindnode support for all voucher types (#713)

* feat(ipld): new data-transfer ipld vouchers + bindnode

* feat(ipld): simplify ipldutils API

* feat(ipld): use new bindnode registry in go-ipld-prime

Ref: ipld/go-ipld-prime#437

feat(deps): update data transfer 61f0756c

feat(deps): update data transfer and other deps

update to master data transfer with libp2p v0.22.0 plus associated other deps

.circleci/config.yml

@@ -6,7 +6,7 @@ orbs:
 executors:
   golang:
     docker:
-      - image: cimg/go:1.17.9
+      - image: cimg/go:1.18.7-node
     resource_class: large
 
 commands:

CONTRIBUTING.md

@@ -81,7 +81,7 @@ import (
 	ipld "github.com/ipfs/go-ipld-format"
 	"github.com/stretchr/testify/assert"
 
-	datatransfer "github.com/filecoin-project/go-data-transfer"
+	datatransfer "github.com/filecoin-project/go-data-transfer/v2"
 
 	"github.com/filecoin-project/go-fil-markets/filestore/file"
 )

discovery/impl/local_test.go

@@ -1,23 +1,17 @@
 package discoveryimpl_test
 
 import (
-	"bytes"
 	"context"
 	"testing"
 	"time"
 
 	"github.com/ipfs/go-datastore"
-	dshelp "github.com/ipfs/go-ipfs-ds-help"
-	"github.com/libp2p/go-libp2p-core/peer"
+	"github.com/libp2p/go-libp2p/core/peer"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 
-	"github.com/filecoin-project/go-address"
-
 	discoveryimpl "github.com/filecoin-project/go-fil-markets/discovery/impl"
-	"github.com/filecoin-project/go-fil-markets/discovery/migrations"
 	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
-	retrievalmigrations "github.com/filecoin-project/go-fil-markets/retrievalmarket/migrations"
 	"github.com/filecoin-project/go-fil-markets/shared_testutil"
 )
 
@@ -70,57 +64,3 @@ func TestLocal_AddPeer(t *testing.T) {
 		})
 	}
 }
-
-func TestLocalMigrations(t *testing.T) {
-	ctx := context.Background()
-	ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
-	defer cancel()
-	ds := datastore.NewMapDatastore()
-
-	peers := shared_testutil.GeneratePeers(4)
-	pieceCIDs := shared_testutil.GenerateCids(4)
-	payloadCids := shared_testutil.GenerateCids(2)
-	for i, c := range payloadCids {
-		rps := migrations.RetrievalPeers0{
-			Peers: []retrievalmigrations.RetrievalPeer0{
-				{
-					Address:  address.TestAddress,
-					ID:       peers[i*2],
-					PieceCID: &pieceCIDs[i*2],
-				},
-				{
-					Address:  address.TestAddress2,
-					ID:       peers[i*2+1],
-					PieceCID: &pieceCIDs[i*2+1],
-				},
-			},
-		}
-		buf := new(bytes.Buffer)
-		err := rps.MarshalCBOR(buf)
-		require.NoError(t, err)
-		err = ds.Put(ctx, dshelp.MultihashToDsKey(c.Hash()), buf.Bytes())
-		require.NoError(t, err)
-	}
-
-	l, err := discoveryimpl.NewLocal(ds)
-	require.NoError(t, err)
-	shared_testutil.StartAndWaitForReady(ctx, t, l)
-
-	for i, c := range payloadCids {
-		expectedPeers := []retrievalmarket.RetrievalPeer{
-			{
-				Address:  address.TestAddress,
-				ID:       peers[i*2],
-				PieceCID: &pieceCIDs[i*2],
-			},
-			{
-				Address:  address.TestAddress2,
-				ID:       peers[i*2+1],
-				PieceCID: &pieceCIDs[i*2+1],
-			},
-		}
-		peers, err := l.GetPeers(c)
-		require.NoError(t, err)
-		require.Equal(t, expectedPeers, peers)
-	}
-}

discovery/migrations/migrations.go

@@ -1,37 +1,8 @@
 package migrations
 
 import (
-	versioning "github.com/filecoin-project/go-ds-versioning/pkg"
 	"github.com/filecoin-project/go-ds-versioning/pkg/versioned"
-
-	"github.com/filecoin-project/go-fil-markets/discovery"
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket"
-	"github.com/filecoin-project/go-fil-markets/retrievalmarket/migrations"
 )
 
-//go:generate cbor-gen-for RetrievalPeers0
-
-// RetrievalPeers0 is version 0 of RetrievalPeers
-type RetrievalPeers0 struct {
-	Peers []migrations.RetrievalPeer0
-}
-
-// MigrateRetrievalPeers0To1 migrates a tuple encoded list of retrieval peers to a map encoded list
-func MigrateRetrievalPeers0To1(oldRps *RetrievalPeers0) (*discovery.RetrievalPeers, error) {
-	peers := make([]retrievalmarket.RetrievalPeer, 0, len(oldRps.Peers))
-	for _, oldRp := range oldRps.Peers {
-		peers = append(peers, retrievalmarket.RetrievalPeer{
-			Address:  oldRp.Address,
-			ID:       oldRp.ID,
-			PieceCID: oldRp.PieceCID,
-		})
-	}
-	return &discovery.RetrievalPeers{
-		Peers: peers,
-	}, nil
-}
-
 // RetrievalPeersMigrations are migrations for the store local discovery list of peers we can retrieve from
-var RetrievalPeersMigrations = versioned.BuilderList{
-	versioned.NewVersionedBuilder(MigrateRetrievalPeers0To1, versioning.VersionKey("1")),
-}
+var RetrievalPeersMigrations = versioned.BuilderList{}

docs/retrievalprovider.mmd

@@ -16,14 +16,18 @@ stateDiagram-v2
 	state "DealStatusCancelled" as DealStatusCancelled
 	DealStatusUnsealing : On entry runs UnsealData
 	DealStatusUnsealed : On entry runs UnpauseDeal
-	DealStatusFundsNeededUnseal : On entry runs TrackTransfer
+	DealStatusFundsNeededUnseal : On entry runs UpdateFunding
 	DealStatusFailing : On entry runs CancelDeal
+	DealStatusFundsNeeded : On entry runs UpdateFunding
+	DealStatusFundsNeededLastPayment : On entry runs UpdateFunding
 	DealStatusCompleting : On entry runs CleanupDeal
 	DealStatusCancelling : On entry runs CancelDeal
 	[*] --> DealStatusNew
 	note right of DealStatusNew
 		The following events are not shown cause they can trigger from any state.
 
+		ProviderEventProcessPayment - does not transition state
+		ProviderEventComplete - transitions state to DealStatusCompleting
 		ProviderEventDataTransferError - transitions state to DealStatusErrored
 		ProviderEventMultiStoreError - transitions state to DealStatusErrored
 		ProviderEventClientCancelled - transitions state to DealStatusCancelling
@@ -35,20 +39,20 @@ stateDiagram-v2
 	DealStatusUnsealing --> DealStatusUnsealed : ProviderEventUnsealComplete
 	DealStatusUnsealed --> DealStatusOngoing : ProviderEventBlockSent
 	DealStatusOngoing --> DealStatusOngoing : ProviderEventBlockSent
-	DealStatusOngoing --> DealStatusBlocksComplete : ProviderEventBlocksCompleted
 	DealStatusNew --> DealStatusFundsNeededUnseal : ProviderEventPaymentRequested
 	DealStatusUnsealed --> DealStatusFundsNeeded : ProviderEventPaymentRequested
 	DealStatusOngoing --> DealStatusFundsNeeded : ProviderEventPaymentRequested
-	DealStatusBlocksComplete --> DealStatusFundsNeededLastPayment : ProviderEventPaymentRequested
+	DealStatusUnsealed --> DealStatusFundsNeededLastPayment : ProviderEventLastPaymentRequested
+	DealStatusOngoing --> DealStatusFundsNeededLastPayment : ProviderEventLastPaymentRequested
+	DealStatusFundsNeededUnseal --> DealStatusFailing : ProviderEventSaveVoucherFailed
 	DealStatusFundsNeeded --> DealStatusFailing : ProviderEventSaveVoucherFailed
 	DealStatusFundsNeededLastPayment --> DealStatusFailing : ProviderEventSaveVoucherFailed
+	DealStatusFundsNeededUnseal --> DealStatusFundsNeededUnseal : ProviderEventPartialPaymentReceived
 	DealStatusFundsNeeded --> DealStatusFundsNeeded : ProviderEventPartialPaymentReceived
 	DealStatusFundsNeededLastPayment --> DealStatusFundsNeededLastPayment : ProviderEventPartialPaymentReceived
 	DealStatusFundsNeededUnseal --> DealStatusUnsealing : ProviderEventPaymentReceived
 	DealStatusFundsNeeded --> DealStatusOngoing : ProviderEventPaymentReceived
 	DealStatusFundsNeededLastPayment --> DealStatusFinalizing : ProviderEventPaymentReceived
-	DealStatusBlocksComplete --> DealStatusCompleting : ProviderEventComplete
-	DealStatusFinalizing --> DealStatusCompleting : ProviderEventComplete
 	DealStatusCompleting --> DealStatusCompleted : ProviderEventCleanupComplete
 	DealStatusFailing --> DealStatusErrored : ProviderEventCancelComplete
 	DealStatusCancelling --> DealStatusCancelled : ProviderEventCancelComplete