Skip to content

Commit

Permalink
Data Transfer V2 Integration (#757)
Browse files Browse the repository at this point in the history
Voucher refactor integration (#707)

* refactor(storagemarket): update storagemarket

* refactor(retrievalmarket): use new voucher system

refactor for predictability and correctness using new voucher system

* chore(deps): update go-data-transfer v2

* style(lint): prep for pr

* docs(retrievalmarket): add comments

* chore(deps): update go-statemachine

* style(imports): fix imports

chore(retrievalmarket): remove old types (#712)

feat(ipld): bindnode support for all voucher types (#713)

* feat(ipld): new data-transfer ipld vouchers + bindnode

* feat(ipld): simplify ipldutils API

* feat(ipld): use new bindnode registry in go-ipld-prime

Ref: ipld/go-ipld-prime#437

feat(deps): update data transfer 61f0756c

feat(deps): update data transfer and other deps

update to master data transfer with libp2p v0.22.0 plus associated other deps

Update retrievalmarket/types.go

Update storagemarket/types.go

chore(deps): update to rc candidate

chore(dtutils): update for new data transfer configuration interface

chore(deps): update to rc4
  • Loading branch information
hannahhoward committed Feb 18, 2023
1 parent 5a8363b commit 955fd43
Show file tree
Hide file tree
Showing 81 changed files with 2,717 additions and 5,940 deletions.
2 changes: 1 addition & 1 deletion .circleci/config.yml
Original file line number Diff line number Diff line change
Expand Up @@ -6,7 +6,7 @@ orbs:
executors:
golang:
docker:
- image: cimg/go:1.17.9
- image: cimg/go:1.18.7-node
resource_class: large

commands:
Expand Down
2 changes: 1 addition & 1 deletion CONTRIBUTING.md
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ import (
ipld "github.com/ipfs/go-ipld-format"
"github.com/stretchr/testify/assert"

datatransfer "github.com/filecoin-project/go-data-transfer"
datatransfer "github.com/filecoin-project/go-data-transfer/v2"

"github.com/filecoin-project/go-fil-markets/filestore/file"
)
Expand Down
60 changes: 0 additions & 60 deletions discovery/impl/local_test.go
Original file line number Diff line number Diff line change
@@ -1,23 +1,17 @@
package discoveryimpl_test

import (
"bytes"
"context"
"testing"
"time"

"github.com/ipfs/go-datastore"
dshelp "github.com/ipfs/go-ipfs-ds-help"
"github.com/libp2p/go-libp2p/core/peer"
"github.com/stretchr/testify/assert"
"github.com/stretchr/testify/require"

"github.com/filecoin-project/go-address"

discoveryimpl "github.com/filecoin-project/go-fil-markets/discovery/impl"
"github.com/filecoin-project/go-fil-markets/discovery/migrations"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
retrievalmigrations "github.com/filecoin-project/go-fil-markets/retrievalmarket/migrations"
"github.com/filecoin-project/go-fil-markets/shared_testutil"
)

Expand Down Expand Up @@ -70,57 +64,3 @@ func TestLocal_AddPeer(t *testing.T) {
})
}
}

func TestLocalMigrations(t *testing.T) {
ctx := context.Background()
ctx, cancel := context.WithTimeout(ctx, 5*time.Second)
defer cancel()
ds := datastore.NewMapDatastore()

peers := shared_testutil.GeneratePeers(4)
pieceCIDs := shared_testutil.GenerateCids(4)
payloadCids := shared_testutil.GenerateCids(2)
for i, c := range payloadCids {
rps := migrations.RetrievalPeers0{
Peers: []retrievalmigrations.RetrievalPeer0{
{
Address: address.TestAddress,
ID: peers[i*2],
PieceCID: &pieceCIDs[i*2],
},
{
Address: address.TestAddress2,
ID: peers[i*2+1],
PieceCID: &pieceCIDs[i*2+1],
},
},
}
buf := new(bytes.Buffer)
err := rps.MarshalCBOR(buf)
require.NoError(t, err)
err = ds.Put(ctx, dshelp.MultihashToDsKey(c.Hash()), buf.Bytes())
require.NoError(t, err)
}

l, err := discoveryimpl.NewLocal(ds)
require.NoError(t, err)
shared_testutil.StartAndWaitForReady(ctx, t, l)

for i, c := range payloadCids {
expectedPeers := []retrievalmarket.RetrievalPeer{
{
Address: address.TestAddress,
ID: peers[i*2],
PieceCID: &pieceCIDs[i*2],
},
{
Address: address.TestAddress2,
ID: peers[i*2+1],
PieceCID: &pieceCIDs[i*2+1],
},
}
peers, err := l.GetPeers(c)
require.NoError(t, err)
require.Equal(t, expectedPeers, peers)
}
}
31 changes: 1 addition & 30 deletions discovery/migrations/migrations.go
Original file line number Diff line number Diff line change
@@ -1,37 +1,8 @@
package migrations

import (
versioning "github.com/filecoin-project/go-ds-versioning/pkg"
"github.com/filecoin-project/go-ds-versioning/pkg/versioned"

"github.com/filecoin-project/go-fil-markets/discovery"
"github.com/filecoin-project/go-fil-markets/retrievalmarket"
"github.com/filecoin-project/go-fil-markets/retrievalmarket/migrations"
)

//go:generate cbor-gen-for RetrievalPeers0

// RetrievalPeers0 is version 0 of RetrievalPeers
type RetrievalPeers0 struct {
Peers []migrations.RetrievalPeer0
}

// MigrateRetrievalPeers0To1 migrates a tuple encoded list of retrieval peers to a map encoded list
func MigrateRetrievalPeers0To1(oldRps *RetrievalPeers0) (*discovery.RetrievalPeers, error) {
peers := make([]retrievalmarket.RetrievalPeer, 0, len(oldRps.Peers))
for _, oldRp := range oldRps.Peers {
peers = append(peers, retrievalmarket.RetrievalPeer{
Address: oldRp.Address,
ID: oldRp.ID,
PieceCID: oldRp.PieceCID,
})
}
return &discovery.RetrievalPeers{
Peers: peers,
}, nil
}

// RetrievalPeersMigrations are migrations for the store local discovery list of peers we can retrieve from
var RetrievalPeersMigrations = versioned.BuilderList{
versioned.NewVersionedBuilder(MigrateRetrievalPeers0To1, versioning.VersionKey("1")),
}
var RetrievalPeersMigrations = versioned.BuilderList{}
105 changes: 0 additions & 105 deletions discovery/migrations/migrations_cbor_gen.go

This file was deleted.

14 changes: 9 additions & 5 deletions docs/retrievalprovider.mmd
Original file line number Diff line number Diff line change
Expand Up @@ -16,14 +16,18 @@ stateDiagram-v2
state "DealStatusCancelled" as DealStatusCancelled
DealStatusUnsealing : On entry runs UnsealData
DealStatusUnsealed : On entry runs UnpauseDeal
DealStatusFundsNeededUnseal : On entry runs TrackTransfer
DealStatusFundsNeededUnseal : On entry runs UpdateFunding
DealStatusFailing : On entry runs CancelDeal
DealStatusFundsNeeded : On entry runs UpdateFunding
DealStatusFundsNeededLastPayment : On entry runs UpdateFunding
DealStatusCompleting : On entry runs CleanupDeal
DealStatusCancelling : On entry runs CancelDeal
[*] --> DealStatusNew
note right of DealStatusNew
The following events are not shown cause they can trigger from any state.
ProviderEventProcessPayment - does not transition state
ProviderEventComplete - transitions state to DealStatusCompleting
ProviderEventDataTransferError - transitions state to DealStatusErrored
ProviderEventMultiStoreError - transitions state to DealStatusErrored
ProviderEventClientCancelled - transitions state to DealStatusCancelling
Expand All @@ -35,20 +39,20 @@ stateDiagram-v2
DealStatusUnsealing --> DealStatusUnsealed : ProviderEventUnsealComplete
DealStatusUnsealed --> DealStatusOngoing : ProviderEventBlockSent
DealStatusOngoing --> DealStatusOngoing : ProviderEventBlockSent
DealStatusOngoing --> DealStatusBlocksComplete : ProviderEventBlocksCompleted
DealStatusNew --> DealStatusFundsNeededUnseal : ProviderEventPaymentRequested
DealStatusUnsealed --> DealStatusFundsNeeded : ProviderEventPaymentRequested
DealStatusOngoing --> DealStatusFundsNeeded : ProviderEventPaymentRequested
DealStatusBlocksComplete --> DealStatusFundsNeededLastPayment : ProviderEventPaymentRequested
DealStatusUnsealed --> DealStatusFundsNeededLastPayment : ProviderEventLastPaymentRequested
DealStatusOngoing --> DealStatusFundsNeededLastPayment : ProviderEventLastPaymentRequested
DealStatusFundsNeededUnseal --> DealStatusFailing : ProviderEventSaveVoucherFailed
DealStatusFundsNeeded --> DealStatusFailing : ProviderEventSaveVoucherFailed
DealStatusFundsNeededLastPayment --> DealStatusFailing : ProviderEventSaveVoucherFailed
DealStatusFundsNeededUnseal --> DealStatusFundsNeededUnseal : ProviderEventPartialPaymentReceived
DealStatusFundsNeeded --> DealStatusFundsNeeded : ProviderEventPartialPaymentReceived
DealStatusFundsNeededLastPayment --> DealStatusFundsNeededLastPayment : ProviderEventPartialPaymentReceived
DealStatusFundsNeededUnseal --> DealStatusUnsealing : ProviderEventPaymentReceived
DealStatusFundsNeeded --> DealStatusOngoing : ProviderEventPaymentReceived
DealStatusFundsNeededLastPayment --> DealStatusFinalizing : ProviderEventPaymentReceived
DealStatusBlocksComplete --> DealStatusCompleting : ProviderEventComplete
DealStatusFinalizing --> DealStatusCompleting : ProviderEventComplete
DealStatusCompleting --> DealStatusCompleted : ProviderEventCleanupComplete
DealStatusFailing --> DealStatusErrored : ProviderEventCancelComplete
DealStatusCancelling --> DealStatusCancelled : ProviderEventCancelComplete
Expand Down
Binary file modified docs/retrievalprovider.mmd.png
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
6 changes: 3 additions & 3 deletions docs/retrievalprovider.mmd.svg
Loading
Sorry, something went wrong. Reload?
Sorry, we cannot display this file.
Sorry, this file is invalid so it cannot be displayed.
Loading

0 comments on commit 955fd43

Please sign in to comment.