Feat/refactor transport protocol update (#337)

* refactor(network): move network to subdirectory

move the network to a subdirectory, also cleanup usage of all selector, move gstestdata to the itest
directory

* feat(network): introduce wrapped protocol

introduces a new wrapped protocol that encodes the transport id so that we can support multiple
transports using the network protocol

* style(lint): fix imports

* fix(rebase): fix issues after rebase

Author: hannahhoward

benchmarks/benchmark_test.go

@@ -24,9 +24,7 @@ import (
 	"github.com/ipfs/go-merkledag"
 	"github.com/ipfs/go-unixfs/importer/balanced"
 	ihelper "github.com/ipfs/go-unixfs/importer/helpers"
-	basicnode "github.com/ipld/go-ipld-prime/node/basic"
-	ipldselector "github.com/ipld/go-ipld-prime/traversal/selector"
-	"github.com/ipld/go-ipld-prime/traversal/selector/builder"
+	selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse"
 	mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
 	"github.com/stretchr/testify/require"
 
@@ -77,10 +75,6 @@ func p2pStrestTest(ctx context.Context, b *testing.B, numfiles int, df distFunc,
 		thisCids := df(ctx, b, instances[:1])
 		allCids = append(allCids, thisCids...)
 	}
-	ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any)
-
-	allSelector := ssb.ExploreRecursive(ipldselector.RecursionLimitNone(),
-		ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node()
 
 	runtime.GC()
 	b.ResetTimer()
@@ -105,7 +99,7 @@ func p2pStrestTest(ctx context.Context, b *testing.B, numfiles int, df distFunc,
 		timer := time.NewTimer(30 * time.Second)
 		start := time.Now()
 		for j := 0; j < numfiles; j++ {
-			_, err := pusher.Manager.OpenPushDataChannel(ctx, receiver.Peer, testutil.NewTestTypedVoucher(), allCids[j], allSelector)
+			_, err := pusher.Manager.OpenPushDataChannel(ctx, receiver.Peer, testutil.NewTestTypedVoucher(), allCids[j], selectorparse.CommonSelector_ExploreAllRecursively)
 			if err != nil {
 				b.Fatalf("received error on request: %s", err.Error())
 			}

benchmarks/testinstance/testinstance.go

@@ -21,9 +21,9 @@ import (
 	datatransfer "github.com/filecoin-project/go-data-transfer/v2"
 	tn "github.com/filecoin-project/go-data-transfer/v2/benchmarks/testnet"
 	dtimpl "github.com/filecoin-project/go-data-transfer/v2/impl"
-	dtnet "github.com/filecoin-project/go-data-transfer/v2/network"
 	"github.com/filecoin-project/go-data-transfer/v2/testutil"
 	gstransport "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync"
+	dtnet "github.com/filecoin-project/go-data-transfer/v2/transport/helpers/network"
 )
 
 // TempDirGenerator is any interface that can generate temporary directories

benchmarks/testnet/interface.go

@@ -4,7 +4,7 @@ import (
 	gsnet "github.com/ipfs/go-graphsync/network"
 	"github.com/libp2p/go-libp2p-core/peer"
 
-	dtnet "github.com/filecoin-project/go-data-transfer/v2/network"
+	dtnet "github.com/filecoin-project/go-data-transfer/v2/transport/helpers/network"
 )
 
 // Network is an interface for generating graphsync network interfaces

benchmarks/testnet/peernet.go

@@ -7,7 +7,7 @@ import (
 	"github.com/libp2p/go-libp2p-core/peer"
 	mockpeernet "github.com/libp2p/go-libp2p/p2p/net/mock"
 
-	dtnet "github.com/filecoin-project/go-data-transfer/v2/network"
+	dtnet "github.com/filecoin-project/go-data-transfer/v2/transport/helpers/network"
 )
 
 type peernet struct {

impl/initiating_test.go

@@ -11,6 +11,7 @@ import (
 	dss "github.com/ipfs/go-datastore/sync"
 	"github.com/ipld/go-ipld-prime/datamodel"
 	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
+	selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse"
 	"github.com/libp2p/go-libp2p-core/peer"
 	"github.com/stretchr/testify/require"
 
@@ -333,7 +334,7 @@ func TestDataTransferInitiating(t *testing.T) {
 				events:         make(chan datatransfer.EventCode, len(verify.expectedEvents)),
 			}
 			ev.setup(t, dt)
-			h.stor = testutil.AllSelector()
+			h.stor = selectorparse.CommonSelector_ExploreAllRecursively
 			h.voucher = testutil.NewTestTypedVoucher()
 			h.voucherResult = testutil.NewTestTypedVoucher()
 			require.NoError(t, err)
@@ -580,7 +581,7 @@ func TestDataTransferRestartInitiating(t *testing.T) {
 			ev.setup(t, dt)
 
 			// setup voucher processing
-			h.stor = testutil.AllSelector()
+			h.stor = selectorparse.CommonSelector_ExploreAllRecursively
 			h.voucher = testutil.NewTestTypedVoucher()
 			require.NoError(t, h.dt.RegisterVoucherType(h.voucher.Type, h.voucherValidator))
 			h.voucherResult = testutil.NewTestTypedVoucher()

impl/responding_test.go

@@ -13,6 +13,7 @@ import (
 	"github.com/ipld/go-ipld-prime"
 	"github.com/ipld/go-ipld-prime/datamodel"
 	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
+	selectorparse "github.com/ipld/go-ipld-prime/traversal/selector/parse"
 	"github.com/libp2p/go-libp2p-core/peer"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
@@ -617,7 +618,7 @@ func TestDataTransferResponding(t *testing.T) {
 				events:         make(chan datatransfer.EventCode, len(verify.expectedEvents)),
 			}
 			ev.setup(t, dt)
-			h.stor = testutil.AllSelector()
+			h.stor = selectorparse.CommonSelector_ExploreAllRecursively
 			h.voucher = testutil.NewTestTypedVoucher()
 			h.baseCid = testutil.GenerateCids(1)[0]
 			h.id = datatransfer.TransferID(rand.Int31())
@@ -999,7 +1000,7 @@ func TestDataTransferRestartResponding(t *testing.T) {
 				events:         make(chan datatransfer.EventCode, len(verify.expectedEvents)),
 			}
 			ev.setup(t, dt)
-			h.stor = testutil.AllSelector()
+			h.stor = selectorparse.CommonSelector_ExploreAllRecursively
 			h.voucher = testutil.NewTestTypedVoucher()
 			h.baseCid = testutil.GenerateCids(1)[0]
 			h.id = datatransfer.TransferID(rand.Int31())

testutil/fixtures/lorem.txt renamed to itest/fixtures/lorem.txt

@@ -1,4 +1,4 @@
-package testutil
+package itest
 
 import (
 	"bytes"
@@ -29,37 +29,30 @@ import (
 	"github.com/ipfs/go-unixfs/importer/balanced"
 	ihelper "github.com/ipfs/go-unixfs/importer/helpers"
 	"github.com/ipld/go-ipld-prime"
-	"github.com/ipld/go-ipld-prime/datamodel"
 	cidlink "github.com/ipld/go-ipld-prime/linking/cid"
-	basicnode "github.com/ipld/go-ipld-prime/node/basic"
-	"github.com/ipld/go-ipld-prime/traversal/selector"
-	"github.com/ipld/go-ipld-prime/traversal/selector/builder"
 	"github.com/libp2p/go-libp2p-core/host"
 	"github.com/libp2p/go-libp2p-core/protocol"
 	mocknet "github.com/libp2p/go-libp2p/p2p/net/mock"
 	"github.com/stretchr/testify/require"
 
 	datatransfer "github.com/filecoin-project/go-data-transfer/v2"
-	"github.com/filecoin-project/go-data-transfer/v2/network"
 	gstransport "github.com/filecoin-project/go-data-transfer/v2/transport/graphsync"
 	"github.com/filecoin-project/go-data-transfer/v2/transport/graphsync/extension"
+	"github.com/filecoin-project/go-data-transfer/v2/transport/helpers/network"
 )
 
-var allSelector datamodel.Node
-
 const loremFile = "lorem.txt"
+const loremFileTransferBytes = 20439
 
-func init() {
-	ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any)
-	allSelector = ssb.ExploreRecursive(selector.RecursionLimitNone(),
-		ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node()
-}
+const loremLargeFile = "lorem_large.txt"
+const loremLargeFileTransferBytes = 217452
 
 const unixfsChunkSize uint64 = 1 << 10
 const unixfsLinksPerLevel = 1024
 
 var extsForProtocol = map[protocol.ID]graphsync.ExtensionName{
-	datatransfer.ProtocolDataTransfer1_2: extension.ExtensionDataTransfer1_1,
+	network.ProtocolDataTransfer1_2:    extension.ExtensionDataTransfer1_1,
+	network.ProtocolFilDataTransfer1_2: extension.ExtensionDataTransfer1_1,
 }
 
 // GraphsyncTestingData is a test harness for testing data transfer on top of
@@ -83,7 +76,6 @@ type GraphsyncTestingData struct {
 	GsNet2         gsnet.GraphSyncNetwork
 	DtNet1         network.DataTransferNetwork
 	DtNet2         network.DataTransferNetwork
-	AllSelector    datamodel.Node
 	OrigBytes      []byte
 	TempDir1       string
 	TempDir2       string
@@ -152,7 +144,6 @@ func NewGraphsyncTestingData(ctx context.Context, t *testing.T, host1Protocols [
 	require.NoError(t, err)
 	gsData.TempDir2 = tempdir
 	// create a selector for the whole UnixFS dag
-	gsData.AllSelector = allSelector
 	gsData.host1Protocols = host1Protocols
 	gsData.host2Protocols = host2Protocols
 	return gsData