diff --git a/.circleci/config.yml b/.circleci/config.yml index 6b95728cd..acba9ebe8 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -325,6 +325,12 @@ workflows: suite: itest-markets_v1_deal target: "./itests/markets_v1_deal_test.go" + + - test: + name: test-itest-markets_v1_retrieval + suite: itest-markets_v1_retrieval + target: "./itests/markets_v1_retrieval_test.go" + - test: name: test-itest-markets_v1_offline_deal suite: itest-markets_v1_offline_deal diff --git a/go.mod b/go.mod index 8c64bc09c..1cab1009f 100644 --- a/go.mod +++ b/go.mod @@ -51,7 +51,7 @@ require ( github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-cidutil v0.1.0 github.com/ipfs/go-datastore v0.6.0 - github.com/ipfs/go-graphsync v0.14.6 + github.com/ipfs/go-graphsync v0.14.8 github.com/ipfs/go-ipfs-blockstore v1.3.0 // indirect github.com/ipfs/go-ipfs-blocksutil v0.0.1 github.com/ipfs/go-ipfs-chunker v0.0.5 // indirect @@ -67,8 +67,8 @@ require ( github.com/ipfs/go-metrics-interface v0.0.1 github.com/ipfs/go-unixfs v0.4.5 // indirect github.com/ipld/go-car v0.6.1 - github.com/ipld/go-car/v2 v2.10.1 - github.com/ipld/go-ipld-prime v0.20.0 + github.com/ipld/go-car/v2 v2.11.0 + github.com/ipld/go-ipld-prime v0.21.0 github.com/ipld/go-ipld-selector-text-lite v0.0.1 github.com/ipni/index-provider v0.13.4 github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c @@ -223,7 +223,7 @@ require ( github.com/ipfs/go-ipld-cbor v0.0.6 github.com/ipfs/go-log v1.0.5 // indirect github.com/ipfs/go-peertaskqueue v0.8.1 // indirect - github.com/ipfs/go-unixfsnode v1.7.1 + github.com/ipfs/go-unixfsnode v1.8.0 github.com/ipfs/go-verifcid v0.0.2 // indirect github.com/ipld/go-codec-dagpb v1.6.0 github.com/ipld/go-ipld-adl-hamt v0.0.0-20220616142416-9004dbd839e0 // indirect @@ -334,7 +334,7 @@ require ( require ( github.com/filecoin-project/boost-gfm v1.26.7 - github.com/filecoin-project/boost-graphsync v0.13.6 + github.com/filecoin-project/boost-graphsync v0.13.7 github.com/filecoin-project/go-data-transfer/v2 v2.0.0-rc7 github.com/filecoin-project/go-fil-markets v1.28.3 github.com/filecoin-project/lotus v1.23.2-0.20230622154405-168d022018ce diff --git a/go.sum b/go.sum index e17996b8f..9f6da5c53 100644 --- a/go.sum +++ b/go.sum @@ -333,8 +333,8 @@ github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYF github.com/felixge/httpsnoop v1.0.3 h1:s/nj+GCswXYzN5v2DpNMuMQYe+0DDwt5WVCU6CWBdXk= github.com/filecoin-project/boost-gfm v1.26.7 h1:ENJEqx1OzY072QnUP37YrGVmUiCewRwHAjbtTxyW74Y= github.com/filecoin-project/boost-gfm v1.26.7/go.mod h1:OhG2y7WeDx3KU9DPjgWllS+3/ospPjm8/XDrvN6uOfk= -github.com/filecoin-project/boost-graphsync v0.13.6 h1:UZZHwUt9epr3r1KUAy7A01vbmkgYRgpm9O0ZCVa/fzE= -github.com/filecoin-project/boost-graphsync v0.13.6/go.mod h1:WL0AQccJeNASH4XG5CaXJ0zbkqoTaqOZCekyCFly75s= +github.com/filecoin-project/boost-graphsync v0.13.7 h1:c/CQOSd6Bpw7qnv9KUL/hbWOfN0l5xwyX8UNxQk4ElM= +github.com/filecoin-project/boost-graphsync v0.13.7/go.mod h1:bc2M5ZLZJtXHl8kjnqtn4L1MsdEqpJErDaIeY0bJ9wk= github.com/filecoin-project/dagstore v0.7.0 h1:IS0R+69za8dguYWeqz/MI+nb7ONpk03tAkxPCBXEKm0= github.com/filecoin-project/dagstore v0.7.0/go.mod h1:YKn4qXih+/2xQWpfJsaKGOi4POw5vH5grDmfPCCnx8g= github.com/filecoin-project/go-address v0.0.3/go.mod h1:jr8JxKsYx+lQlQZmF5i2U0Z+cGQ59wMIps/8YW/lDj8= @@ -442,7 +442,7 @@ github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/frankban/quicktest v1.14.2/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= -github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= @@ -804,8 +804,8 @@ github.com/ipfs/go-ds-measure v0.2.0/go.mod h1:SEUD/rE2PwRa4IQEC5FuNAmjJCyYObZr9 github.com/ipfs/go-fs-lock v0.0.6/go.mod h1:OTR+Rj9sHiRubJh3dRhD15Juhd/+w6VPOY28L7zESmM= github.com/ipfs/go-fs-lock v0.0.7 h1:6BR3dajORFrFTkb5EpCUFIAypsoxpGpDSVUdFwzgL9U= github.com/ipfs/go-fs-lock v0.0.7/go.mod h1:Js8ka+FNYmgQRLrRXzU3CB/+Csr1BwrRilEcvYrHhhc= -github.com/ipfs/go-graphsync v0.14.6 h1:NPxvuUy4Z08Mg8dwpBzwgbv/PGLIufSJ1sle6iAX8yo= -github.com/ipfs/go-graphsync v0.14.6/go.mod h1:yT0AfjFgicOoWdAlUJ96tQ5AkuGI4r1taIQX/aHbBQo= +github.com/ipfs/go-graphsync v0.14.8 h1:NFFHquTNnwPi05tJhdpPj4CJMnqRBLxpZd+IfPRauf4= +github.com/ipfs/go-graphsync v0.14.8/go.mod h1:qyHjUvHey6EfKUDMQPwCuVkMOurRG3hcjRm+FaVP6bE= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= github.com/ipfs/go-ipfs-blockstore v0.2.1/go.mod h1:jGesd8EtCM3/zPgx+qr0/feTXGUeRai6adgwC+Q+JvE= @@ -915,8 +915,8 @@ github.com/ipfs/go-unixfs v0.3.1/go.mod h1:h4qfQYzghiIc8ZNFKiLMFWOTzrWIAtzYQ59W/ github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU= github.com/ipfs/go-unixfs v0.4.5/go.mod h1:BIznJNvt/gEx/ooRMI4Us9K8+qeGO7vx1ohnbk8gjFg= github.com/ipfs/go-unixfsnode v1.4.0/go.mod h1:qc7YFFZ8tABc58p62HnIYbUMwj9chhUuFWmxSokfePo= -github.com/ipfs/go-unixfsnode v1.7.1 h1:RRxO2b6CSr5UQ/kxnGzaChTjp5LWTdf3Y4n8ANZgB/s= -github.com/ipfs/go-unixfsnode v1.7.1/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= +github.com/ipfs/go-unixfsnode v1.8.0 h1:yCkakzuE365glu+YkgzZt6p38CSVEBPgngL9ZkfnyQU= +github.com/ipfs/go-unixfsnode v1.8.0/go.mod h1:HxRu9HYHOjK6HUqFBAi++7DVoWAHn0o4v/nZ/VA+0g8= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs= github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU= @@ -927,8 +927,8 @@ github.com/ipld/go-car v0.6.1 h1:blWbEHf1j62JMWFIqWE//YR0m7k5ZMw0AuUOU5hjrH8= github.com/ipld/go-car v0.6.1/go.mod h1:oEGXdwp6bmxJCZ+rARSkDliTeYnVzv3++eXajZ+Bmr8= github.com/ipld/go-car/v2 v2.1.1/go.mod h1:+2Yvf0Z3wzkv7NeI69i8tuZ+ft7jyjPYIWZzeVNeFcI= github.com/ipld/go-car/v2 v2.4.1/go.mod h1:zjpRf0Jew9gHqSvjsKVyoq9OY9SWoEKdYCQUKVaaPT0= -github.com/ipld/go-car/v2 v2.10.1 h1:MRDqkONNW9WRhB79u+Z3U5b+NoN7lYA5B8n8qI3+BoI= -github.com/ipld/go-car/v2 v2.10.1/go.mod h1:sQEkXVM3csejlb1kCCb+vQ/pWBKX9QtvsrysMQjOgOg= +github.com/ipld/go-car/v2 v2.11.0 h1:lkAPwbbTFqbdfawgm+bfmFc8PjGC7D12VcaLXPCLNfM= +github.com/ipld/go-car/v2 v2.11.0/go.mod h1:aDszqev0zjtU8l96g4lwXHaU9bzArj56Y7eEN0q/xqA= github.com/ipld/go-codec-dagpb v1.2.0/go.mod h1:6nBN7X7h8EOsEejZGqC7tej5drsdBAXbMHyBT+Fne5s= github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA= github.com/ipld/go-codec-dagpb v1.3.1/go.mod h1:ErNNglIi5KMur/MfFE/svtgQthzVvf+43MrzLbpcIZY= @@ -944,8 +944,8 @@ github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHt github.com/ipld/go-ipld-prime v0.14.0/go.mod h1:9ASQLwUFLptCov6lIYc70GRB4V7UTyLD0IJtrDJe6ZM= github.com/ipld/go-ipld-prime v0.16.0/go.mod h1:axSCuOCBPqrH+gvXr2w9uAOulJqBPhHPT2PjoiiU1qA= github.com/ipld/go-ipld-prime v0.19.0/go.mod h1:Q9j3BaVXwaA3o5JUDNvptDDr/x8+F7FG6XJ8WI3ILg4= -github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g= -github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M= +github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH9C2E= +github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ= github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5/go.mod h1:gcvzoEDBjwycpXt3LBE061wT9f46szXGHAmj9uoP6fU= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73/go.mod h1:2PJ0JgxyB08t0b2WKrcuqI3di0V+5n6RS/LTUJhkoxY= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo= @@ -1871,7 +1871,7 @@ github.com/vishvananda/netlink v1.1.0/go.mod h1:cTgwzPIzzgDAYoQrMm0EdrjRUBkTqKYp github.com/vishvananda/netns v0.0.0-20191106174202-0a2b9b5464df/go.mod h1:JP3t17pCcGlemwknint6hfoeCVQrEMVwxRLRjXpq+BU= github.com/warpfork/go-testmark v0.3.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0= github.com/warpfork/go-testmark v0.10.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0= -github.com/warpfork/go-testmark v0.11.0 h1:J6LnV8KpceDvo7spaNU4+DauH2n1x+6RaO2rJrmpQ9U= +github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= diff --git a/itests/dummydeal_test.go b/itests/dummydeal_test.go index c869de169..af571f889 100644 --- a/itests/dummydeal_test.go +++ b/itests/dummydeal_test.go @@ -94,6 +94,6 @@ func TestDummydealOnline(t *testing.T) { require.NoError(t, err) // rootCid is an identity CID - outFile := f.RetrieveDirect(ctx, t, rootCid, &res.DealParams.ClientDealProposal.Proposal.PieceCID, true) + outFile := f.RetrieveDirect(ctx, t, rootCid, &res.DealParams.ClientDealProposal.Proposal.PieceCID, true, true, nil) kit.AssertFilesEqual(t, randomFilepath, outFile) } diff --git a/itests/framework/framework.go b/itests/framework/framework.go index ffa266246..314023675 100644 --- a/itests/framework/framework.go +++ b/itests/framework/framework.go @@ -59,9 +59,13 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" ipldcbor "github.com/ipfs/go-ipld-cbor" - ipld "github.com/ipfs/go-ipld-format" + ipldformat "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log/v2" "github.com/ipld/go-car" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/codec/dagjson" + "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/traversal/selector" "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/core/host" "github.com/multiformats/go-multihash" @@ -711,7 +715,7 @@ func (f *TestFramework) WaitDealSealed(ctx context.Context, deal *cid.Cid) error } } -func (f *TestFramework) Retrieve(ctx context.Context, t *testing.T, deal *cid.Cid, root cid.Cid, carExport bool) string { +func (f *TestFramework) Retrieve(ctx context.Context, t *testing.T, deal *cid.Cid, root cid.Cid, extractCar bool, selectorNode datamodel.Node) string { // perform retrieval. info, err := f.FullNode.ClientGetDealInfo(ctx, *deal) require.NoError(t, err) @@ -720,7 +724,7 @@ func (f *TestFramework) Retrieve(ctx context.Context, t *testing.T, deal *cid.Ci require.NoError(t, err) require.NotEmpty(t, offers, "no offers") - return f.retrieve(ctx, t, offers[0], carExport) + return f.retrieve(ctx, t, offers[0], extractCar, selectorNode) } func (f *TestFramework) ExtractFileFromCAR(ctx context.Context, t *testing.T, file *os.File) string { @@ -739,7 +743,7 @@ func (f *TestFramework) ExtractFileFromCAR(ctx context.Context, t *testing.T, fi require.NoError(t, err) } - reg := ipld.Registry{} + reg := ipldformat.Registry{} reg.Register(cid.DagProtobuf, dag.DecodeProtobufBlock) reg.Register(cid.DagCBOR, ipldcbor.DecodeBlock) reg.Register(cid.Raw, dag.DecodeRawBlock) @@ -758,15 +762,17 @@ func (f *TestFramework) ExtractFileFromCAR(ctx context.Context, t *testing.T, fi return tmpFile } -func (f *TestFramework) RetrieveDirect(ctx context.Context, t *testing.T, root cid.Cid, pieceCid *cid.Cid, carExport bool) string { +func (f *TestFramework) RetrieveDirect(ctx context.Context, t *testing.T, root cid.Cid, pieceCid *cid.Cid, carExport bool, extractCar bool, selectorNode datamodel.Node) string { offer, err := f.FullNode.ClientMinerQueryOffer(ctx, f.MinerAddr, root, pieceCid) require.NoError(t, err) - return f.retrieve(ctx, t, offer, carExport) + return f.retrieve(ctx, t, offer, extractCar, selectorNode) } -func (f *TestFramework) retrieve(ctx context.Context, t *testing.T, offer lapi.QueryOffer, carExport bool) string { +func (f *TestFramework) retrieve(ctx context.Context, t *testing.T, offer lapi.QueryOffer, extractCar bool, selectorNode datamodel.Node) string { p := path.Join(t.TempDir(), "ret-car-"+t.Name()) + err := os.MkdirAll(path.Dir(p), 0755) + require.NoError(t, err) carFile, err := os.Create(p) require.NoError(t, err) @@ -779,7 +785,17 @@ func (f *TestFramework) retrieve(ctx context.Context, t *testing.T, offer lapi.Q updates, err := f.FullNode.ClientGetRetrievalUpdates(updatesCtx) require.NoError(t, err) - retrievalRes, err := f.FullNode.ClientRetrieve(ctx, offer.Order(caddr)) + order := offer.Order(caddr) + if selectorNode != nil { + _, err := selector.CompileSelector(selectorNode) + require.NoError(t, err) + jsonSelector, err := ipld.Encode(selectorNode, dagjson.Encode) + require.NoError(t, err) + sel := lapi.Selector(jsonSelector) + order.DataSelector = &sel + } + + retrievalRes, err := f.FullNode.ClientRetrieve(ctx, order) require.NoError(t, err) consumeEvents: for { @@ -812,11 +828,11 @@ consumeEvents: }, lapi.FileRef{ Path: carFile.Name(), - IsCAR: carExport, + IsCAR: true, })) ret := carFile.Name() - if carExport { + if extractCar { ret = f.ExtractFileFromCAR(ctx, t, carFile) } diff --git a/itests/markets_v1_deal_test.go b/itests/markets_v1_deal_test.go index 37649d4f4..9be1e988d 100644 --- a/itests/markets_v1_deal_test.go +++ b/itests/markets_v1_deal_test.go @@ -48,7 +48,7 @@ func TestMarketsV1Deal(t *testing.T) { require.NoError(t, err) log.Debugw("deal is sealed, starting retrieval", "cid", dealProposalCid, "root", res.Root) - outPath := f.Retrieve(ctx, t, dealProposalCid, res.Root, true) + outPath := f.Retrieve(ctx, t, dealProposalCid, res.Root, true, nil) log.Debugw("retrieval is done, compare in- and out- files", "in", inPath, "out", outPath) kit.AssertFilesEqual(t, inPath, outPath) diff --git a/itests/markets_v1_offline_deal_test.go b/itests/markets_v1_offline_deal_test.go index 3e700158a..32859a223 100644 --- a/itests/markets_v1_offline_deal_test.go +++ b/itests/markets_v1_offline_deal_test.go @@ -83,7 +83,7 @@ func TestMarketsV1OfflineDeal(t *testing.T) { require.NoError(t, err) log.Debugw("offline deal is sealed, starting retrieval", "cid", dealProposalCid, "root", res.Root) - outPath := f.Retrieve(ctx, t, dealProposalCid, res.Root, true) + outPath := f.Retrieve(ctx, t, dealProposalCid, res.Root, true, nil) log.Debugw("retrieval of offline deal is done, compare in- and out- files", "in", inPath, "out", outPath) kit.AssertFilesEqual(t, inPath, outPath) diff --git a/itests/markets_v1_retrieval_test.go b/itests/markets_v1_retrieval_test.go new file mode 100644 index 000000000..2c6165ae5 --- /dev/null +++ b/itests/markets_v1_retrieval_test.go @@ -0,0 +1,186 @@ +package itests + +import ( + "context" + "math" + "testing" + + "github.com/filecoin-project/boost/itests/framework" + "github.com/filecoin-project/boost/testutil" + lapi "github.com/filecoin-project/lotus/api" + "github.com/filecoin-project/lotus/itests/kit" + "github.com/ipfs/go-cid" + "github.com/ipfs/go-unixfsnode" + "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/node/basicnode" + "github.com/ipld/go-ipld-prime/traversal/selector/builder" + "github.com/stretchr/testify/require" +) + +func TestMarketsV1DealRetrieval(t *testing.T) { + ctx := context.Background() + log := framework.Log + + kit.QuietMiningLogs() + framework.SetLogLevel() + var opts []framework.FrameworkOpts + opts = append(opts, framework.EnableLegacyDeals(true)) + f := framework.NewTestFramework(ctx, t, opts...) + err := f.Start() + require.NoError(t, err) + defer f.Stop() + + // Create a CAR file + log.Debugw("using tempdir", "dir", f.HomeDir) + rseed := 0 + size := 7 << 20 // 7MiB file + + inPath, dirEnt := testutil.CreateRandomUnixfsFileInCar(t, f.HomeDir, rseed, size) + root := dirEnt.Root + leaves := dirEnt.SelfCids[:len(dirEnt.SelfCids)-1] + /* + + For a 7MiB file with seed 0, we expect to have this structure which we + can perform range selections on. The root is dirEnt.Root, the raw leaf + blocks are dirEnt.SelfCid, in order, minus the last one which is also + the root. + + bafybeiet25r42mzboo5osbxmze5gq45zb4peqksscavjb4ixycwjkbwioa | File | /[0:7340031] (7340032 B) + bafkreifrdfke6yoyrjhhqbwkddacapg42vuzcbvwyau6ol24fbjxgfrsha | RawLeaf | ↳ /0[0:256143] (256144 B) + bafkreigmdnzqhuaaqrkzhwo2cqa7heshji5neoygkqku2vrubuvzz4twti | RawLeaf | /0[256144:512287] (256144 B) + bafkreidoafwiruc3tqk4fsvql6jfjhsi36pvmr334qwvfjzupkjnws2c7a | RawLeaf | /0[512288:768431] (256144 B) + bafkreibcmiojncftk5kbjo4xdv4onlxacg6dfi3d3objfl4y7dqhld4a3e | RawLeaf | /0[768432:1024575] (256144 B) + bafkreidpvdovcyfj4ypodcw47xgodpkwlq7yokmofzpxto5v3fjcwtaerm | RawLeaf | /0[1024576:1280719] (256144 B) + bafkreicm6tt6ahmimqwuoxuc7m2jdlmlexiofxmsigfsvi4nsuda45kfpu | RawLeaf | /0[1280720:1536863] (256144 B) + bafkreibnq24344buagftasgfpeomzjf6bbnhku4sw4vv3lnisx7wmvv7de | RawLeaf | /0[1536864:1793007] (256144 B) + bafkreieeqrvyz7z3ezshaqlp24zd5jfnmlzb2gnsuie3rltyhxlyb3f46m | RawLeaf | /0[1793008:2049151] (256144 B) + bafkreiaee3232qc3f3dsv5hezroy27va3cmcf35evzkw426liggu4qqguq | RawLeaf | /0[2049152:2305295] (256144 B) + bafkreiccfzitpdvr3d2tyo2ly6e3qjqcc6vmnr4jfir3ys3zxsvaviaqmy | RawLeaf | /0[2305296:2561439] (256144 B) + bafkreiek37qlpud6koj4jybkrwudic7bvdkraion37xk2x7hxmwyrqxzdi | RawLeaf | /0[2561440:2817583] (256144 B) + bafkreifdib756bpqzucggrqp3tbv3c2mvf3kl6ir5erscbsmxt35lv3qf4 | RawLeaf | /0[2817584:3073727] (256144 B) + bafkreiaflhrfx65ukovmuacvktlopj3j5d6iacvmxgi5vyyelqe77laxda | RawLeaf | /0[3073728:3329871] (256144 B) + bafkreia43b3mtjeo5ycrur5dmj72wpktnt5r2tt6u26djl3syuwt2wirmq | RawLeaf | /0[3329872:3586015] (256144 B) + bafkreic5vfpokxgadhzrx5s2yjoqtif6px6hlkqndvgscd3kgxf22omogy | RawLeaf | /0[3586016:3842159] (256144 B) + bafkreibejeisigfddgxjq3gqirbsjcaikpm5ukp6hyelcdarim3mw4gusm | RawLeaf | /0[3842160:4098303] (256144 B) + bafkreibua7gdtdndlwdpntzdzoleomfy4cuawbou2ibqcwdiqcd5c2cktq | RawLeaf | /0[4098304:4354447] (256144 B) + bafkreihce3ojxvomz2afkl4sr67qlje3vvxqcnuugb2mekjocuf2m3axh4 | RawLeaf | /0[4354448:4610591] (256144 B) + bafkreih6gxdebrtqukp7dgpz6uq4mcljkk42aont4f2fmu7yw3qvst5s44 | RawLeaf | /0[4610592:4866735] (256144 B) + bafkreihsyox3ebuboypzrzdnja7mypcq7lugd7yajnyxswaphoqrfj3mvi | RawLeaf | /0[4866736:5122879] (256144 B) + bafkreihqf3wqz5tv3vqlk7s2no7hchw2327ptwowfak773uvddarif2lzi | RawLeaf | /0[5122880:5379023] (256144 B) + bafkreig6hrykcrosz7jwljalgb2bhdh3r2e43gatvy2q3zn7nbck77dnou | RawLeaf | /0[5379024:5635167] (256144 B) + bafkreihblmdcjlmwp5p4esqn6e2h6ko5oytxilpb3msosjjdw5eaarakye | RawLeaf | /0[5635168:5891311] (256144 B) + bafkreihfmufz2fss7oqxqg2gsz4ow5l7i3byhzuqmxev73bu5fi4vdklc4 | RawLeaf | /0[5891312:6147455] (256144 B) + bafkreid3smar6sdcmi7io6e3ewdv3b3gdcyq7yrdznounhsrkswjdmptoa | RawLeaf | /0[6147456:6403599] (256144 B) + bafkreif54hyudjfodmtdphrhass4rxrrle36xvzj5osfcuue52ck4i7pr4 | RawLeaf | /0[6403600:6659743] (256144 B) + bafkreideyfbj2ljnows5gxnsibvpziyhr26azpae6ilesnhhaissbx5ori | RawLeaf | /0[6659744:6915887] (256144 B) + bafkreighy3cp6pvp3pluq4ragvgrycom4s5upargqkawcyu3rw4flturky | RawLeaf | /0[6915888:7172031] (256144 B) + bafkreifokzy5zcluf3hj23nkrvr7tx6sivpshkd4be5tpfibk6vm2mzlxy | RawLeaf | /0[7172032:7340031] (168000 B) + */ + + // Import and make a deal to store + + res, err := f.FullNode.ClientImport(ctx, lapi.FileRef{Path: inPath, IsCAR: true}) + require.NoError(t, err) + + dp := f.DefaultMarketsV1DealParams() + dp.Data.Root = res.Root + + log.Debugw("starting deal", "root", res.Root) + dealProposalCid, err := f.FullNode.ClientStartDeal(ctx, &dp) + require.NoError(t, err) + + log.Debugw("got deal proposal cid", "cid", dealProposalCid) + + err = f.WaitDealSealed(ctx, dealProposalCid) + require.NoError(t, err) + + // Deal is stored and sealed, attempt different retrieval forms + + retrievalCases := []struct { + name string + selector datamodel.Node + matcherFrom, matcherTo int64 + expectCids []cid.Cid + }{ + { + name: "full file, explore-all", + selector: unixfsnode.UnixFSPathSelectorBuilder("", unixfsnode.ExploreAllRecursivelySelector, false), + expectCids: append([]cid.Cid{root}, leaves...), + }, + { + name: "slice: 0 to 7MiB", + matcherFrom: 0, + matcherTo: 7 << 20, + expectCids: append([]cid.Cid{root}, leaves...), + }, + { + name: "slice: 1MiB to 2MiB", + matcherFrom: 1 << 20, + matcherTo: 2 << 20, + expectCids: append([]cid.Cid{root}, leaves[4:9]...), + }, + { + name: "slice: first byte", + matcherFrom: 0, + matcherTo: 1, + expectCids: append([]cid.Cid{root}, leaves[0]), + }, + { + name: "slice: last byte", + matcherFrom: 7340031, + matcherTo: 7340032, + expectCids: append([]cid.Cid{root}, leaves[len(leaves)-1]), + }, + { + name: "slice: last two blocks, negative range, boundary", + matcherFrom: -168000 - 1, + matcherTo: math.MaxInt64, + expectCids: append([]cid.Cid{root}, leaves[len(leaves)-2:]...), + }, + { + name: "slice: last block, negative range, boundary", + matcherFrom: -168000, + matcherTo: math.MaxInt64, + expectCids: append([]cid.Cid{root}, leaves[len(leaves)-1]), + }, + { + // In this case we are attempting to traverse beyond the file to a + // path that doesn't exist; we expect to only match the root and + // return that. This is not strictly an error case, it's up to the + // consumer of this data to verify the path doesn't resolve in the + // data they get back. + name: "path beyond file", + selector: unixfsnode.UnixFSPathSelectorBuilder("not/a/path", unixfsnode.ExploreAllRecursivelySelector, false), + expectCids: []cid.Cid{root}, + }, + } + + for _, tc := range retrievalCases { + t.Run(tc.name, func(t *testing.T) { + selNode := tc.selector + if selNode == nil { + // build a selector from the specified slice matcher range + ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) + ss := ssb.ExploreInterpretAs("unixfs", ssb.MatcherSubset(tc.matcherFrom, tc.matcherTo)) + selNode = ss.Node() + } + + log.Debugw("deal is sealed, starting retrieval", "cid", dealProposalCid, "root", res.Root) + outPath := f.Retrieve(ctx, t, dealProposalCid, res.Root, false, selNode) + + // Inspect what we got + gotCids, err := testutil.CidsInCar(outPath) + require.NoError(t, err) + require.Equal(t, toStr(tc.expectCids), toStr(gotCids)) + }) + } +} + +// for nicer debugging given that we know the CIDs we expect +func toStr(c []cid.Cid) []string { + out := make([]string, len(c)) + for i, v := range c { + out[i] = v.String() + } + return out +} diff --git a/testutil/car.go b/testutil/car.go index cd8d731b7..4bb7a9ebe 100644 --- a/testutil/car.go +++ b/testutil/car.go @@ -6,6 +6,7 @@ import ( "io" "math/rand" "os" + "testing" "github.com/ipfs/boxo/blockservice" bstore "github.com/ipfs/boxo/blockstore" @@ -20,9 +21,13 @@ import ( ds "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" ipldformat "github.com/ipfs/go-ipld-format" + unixfs "github.com/ipfs/go-unixfsnode/testutil" "github.com/ipld/go-car/v2" "github.com/ipld/go-car/v2/blockstore" + storagecar "github.com/ipld/go-car/v2/storage" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/multiformats/go-multihash" + "github.com/stretchr/testify/require" ) const ( @@ -62,6 +67,29 @@ func CreateRandomFile(dir string, rseed, size int) (string, error) { return file.Name(), nil } +func CreateRandomUnixfsFileInCar(t *testing.T, dir string, rseed, size int) (string, unixfs.DirEntry) { + req := require.New(t) + + file, err := os.CreateTemp(dir, "sourcecar.dat") + req.NoError(err) + + carWriter, err := storagecar.NewWritable(file, []cid.Cid{cid.MustParse("baeaaaiaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa")}, car.WriteAsCarV1(true)) + req.NoError(err) + + lsys := cidlink.DefaultLinkSystem() + lsys.SetWriteStorage(carWriter) + + dirEnt := unixfs.GenerateFile(t, &lsys, rand.New(rand.NewSource(int64(rseed))), size) + + err = file.Close() + req.NoError(err) + + err = car.ReplaceRootsInFile(file.Name(), []cid.Cid{dirEnt.Root}) + req.NoError(err) + + return file.Name(), dirEnt +} + func CreateDenseCARv2(dir, src string) (cid.Cid, string, error) { cs := int64(unixfsChunkSize) maxlinks := unixfsLinksPerLevel @@ -181,3 +209,26 @@ func WriteUnixfsDAGTo(path string, into ipldformat.DAGService, chunksize int64, return nd.Cid(), nil } + +func CidsInCar(path string) ([]cid.Cid, error) { + outReader, err := os.Open(path) + if err != nil { + return nil, err + } + blockReader, err := car.NewBlockReader(outReader) + if err != nil { + return nil, err + } + gotCids := []cid.Cid{} + for { + next, err := blockReader.Next() + if err != nil { + if err != io.EOF { + return nil, err + } + break + } + gotCids = append(gotCids, next.Cid()) + } + return gotCids, nil +}