Skip to content

Commit c5ec428

Browse files
hannahhowardrvagg
hannahhoward
and
rvagg
authored
Minimal alternate metadata type support (#365)
* feat(message): minimal alternate metadata support * Update requestmanager/reconciledloader/reconciledloader_test.go Co-authored-by: Rod Vagg <[email protected]> * Update graphsync.go Co-authored-by: Rod Vagg <[email protected]> Co-authored-by: Rod Vagg <[email protected]>
1 parent f4afe52 commit c5ec428

File tree

7 files changed

+175
-20
lines changed

7 files changed

+175
-20
lines changed

‎graphsync.go

+14
Original file line numberDiff line numberDiff line change
@@ -227,11 +227,25 @@ const (
227227
// is included a this message
228228
LinkActionPresent = LinkAction("Present")
229229

230+
// LinkActionDuplicateNotSent means the linked block was present on this machine,
231+
// but I am not sending it (most likely duplicate)
232+
LinkActionDuplicateNotSent = LinkAction("DuplicateNotSent")
233+
230234
// LinkActionMissing means I did not have the linked block, so I skipped over
231235
// this part of the traversal
232236
LinkActionMissing = LinkAction("Missing")
237+
238+
// LinkActionDuplicateDAGSkipped means the DAG with this link points toward has already
239+
// been traversed entirely in the course of this request so I am skipping over it
240+
LinkActionDuplicateDAGSkipped = LinkAction("DuplicateDAGSkipped")
233241
)
234242

243+
// DidFollowLink indicates whether the remote actually loaded the block and
244+
// followed it in its selector traversal
245+
func (l LinkAction) DidFollowLink() bool {
246+
return l == LinkActionPresent || l == LinkActionDuplicateNotSent
247+
}
248+
235249
// LinkMetadataIterator is used to access individual link metadata through a
236250
// LinkMetadata object
237251
type LinkMetadataIterator func(cid.Cid, LinkAction)

‎message/builder_test.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ func TestMessageBuilding(t *testing.T) {
6060
rb.AddResponseCode(requestID2, graphsync.RequestCompletedFull)
6161

6262
rb.AddLink(requestID3, links[0], graphsync.LinkActionPresent)
63-
rb.AddLink(requestID3, links[1], graphsync.LinkActionPresent)
63+
rb.AddLink(requestID3, links[1], graphsync.LinkActionDuplicateNotSent)
6464

6565
rb.AddResponseCode(requestID4, graphsync.RequestCompletedFull)
6666
rb.AddExtensionData(requestID1, extension1)
@@ -101,7 +101,7 @@ func TestMessageBuilding(t *testing.T) {
101101
require.Equal(t, graphsync.PartialResponse, response3.Status(), "did not generate partial response")
102102
assertMetadata(t, response3, []GraphSyncLinkMetadatum{
103103
{Link: links[0].(cidlink.Link).Cid, Action: graphsync.LinkActionPresent},
104-
{Link: links[1].(cidlink.Link).Cid, Action: graphsync.LinkActionPresent},
104+
{Link: links[1].(cidlink.Link).Cid, Action: graphsync.LinkActionDuplicateNotSent},
105105
})
106106
assertExtension(t, response3, extension2)
107107

‎message/ipldbind/schema.ipldsch

+2-2
Original file line numberDiff line numberDiff line change
@@ -16,14 +16,14 @@ type GraphSyncLinkAction enum {
1616
| Present ("p")
1717
# DuplicateNotSent means the linked block was present on this machine, but I
1818
# am not sending it (most likely duplicate)
19-
# TODO: | DuplicateNotSent ("d")
19+
| DuplicateNotSent ("d")
2020
# Missing means I did not have the linked block, so I skipped over this part
2121
# of the traversal
2222
| Missing ("m")
2323
# DuplicateDAGSkipped means the DAG with this link points toward has already
2424
# been traversed entirely in the course of this request
2525
# so I am skipping over it entirely
26-
# TODO: | DuplicateDAGSkipped ("s")
26+
| DuplicateDAGSkipped ("s")
2727
} representation string
2828

2929
# Metadata for each "link" in the DAG being communicated, each block gets one of

‎message/v2/ipld_roundtrip_test.go

+9-6
Original file line numberDiff line numberDiff line change
@@ -41,11 +41,13 @@ func TestIPLDRoundTrip(t *testing.T) {
4141
id2: message.NewRequest(id2, root2, selectorparse.CommonSelector_ExploreAllRecursively, graphsync.Priority(202)),
4242
}
4343

44-
metadata := message.GraphSyncLinkMetadatum{Link: root2, Action: graphsync.LinkActionMissing}
44+
metadata := []message.GraphSyncLinkMetadatum{{Link: root2, Action: graphsync.LinkActionMissing}, {Link: root1, Action: graphsync.LinkActionDuplicateNotSent}}
45+
46+
metadata2 := []message.GraphSyncLinkMetadatum{{Link: root2, Action: graphsync.LinkActionDuplicateDAGSkipped}, {Link: root1, Action: graphsync.LinkActionPresent}}
4547

4648
responses := map[graphsync.RequestID]message.GraphSyncResponse{
47-
id1: message.NewResponse(id1, graphsync.RequestFailedContentNotFound, []message.GraphSyncLinkMetadatum{metadata}),
48-
id2: message.NewResponse(id2, graphsync.PartialResponse, nil, extension2),
49+
id1: message.NewResponse(id1, graphsync.RequestFailedContentNotFound, metadata),
50+
id2: message.NewResponse(id2, graphsync.PartialResponse, metadata2, extension2),
4951
}
5052

5153
blks := testutil.GenerateBlocksOfSize(2, 100)
@@ -115,11 +117,12 @@ func TestIPLDRoundTrip(t *testing.T) {
115117
require.Equal(t, rtresmap[id2].Status(), graphsync.PartialResponse)
116118
gslm1, ok := rtresmap[id1].Metadata().(message.GraphSyncLinkMetadata)
117119
require.True(t, ok)
118-
require.Len(t, gslm1.RawMetadata(), 1)
119-
require.Equal(t, gslm1.RawMetadata()[0], metadata)
120+
require.Len(t, gslm1.RawMetadata(), 2)
121+
require.Equal(t, gslm1.RawMetadata(), metadata)
120122
gslm2, ok := rtresmap[id2].Metadata().(message.GraphSyncLinkMetadata)
121123
require.True(t, ok)
122-
require.Empty(t, gslm2.RawMetadata())
124+
require.Len(t, gslm2.RawMetadata(), 2)
125+
require.Equal(t, gslm2.RawMetadata(), metadata2)
123126
require.Empty(t, rtresmap[id1].ExtensionNames())
124127
require.Len(t, rtresmap[id2].ExtensionNames(), 1)
125128
rtext2, exists := rtresmap[id2].Extension(extension2Name)

‎requestmanager/reconciledloader/load.go

+2-2
Original file line numberDiff line numberDiff line change
@@ -53,7 +53,7 @@ func (rl *ReconciledLoader) blockReadOpener(lctx linking.LinkContext, link datam
5353
}
5454

5555
// only attempt remote load if after reconciliation we're not on a missing path
56-
if !rl.pathTracker.stillOnMissingRemotePath(lctx.LinkPath) {
56+
if !rl.pathTracker.stillOnUnfollowedRemotePath(lctx.LinkPath) {
5757
data, err := rl.loadRemote(lctx, link)
5858
if data != nil {
5959
return true, types.AsyncLoadResult{Data: data, Local: false}
@@ -164,7 +164,7 @@ func (rl *ReconciledLoader) waitRemote() (bool, error) {
164164
path := rl.verifier.CurrentPath()
165165
head := rl.remoteQueue.first()
166166
rl.remoteQueue.consume()
167-
err := rl.verifier.VerifyNext(head.link, head.action != graphsync.LinkActionMissing)
167+
err := rl.verifier.VerifyNext(head.link, head.action.DidFollowLink())
168168
if err != nil {
169169
return true, err
170170
}

‎requestmanager/reconciledloader/pathtracker.go

+8-8
Original file line numberDiff line numberDiff line change
@@ -8,22 +8,22 @@ import (
88
// pathTracker is just a simple utility to track whether we're on a missing
99
// path for the remote
1010
type pathTracker struct {
11-
lastMissingRemotePath datamodel.Path
11+
lastUnfollowedRemotePath datamodel.Path
1212
}
1313

14-
// stillOnMissingRemotePath determines whether the next link load will be from
14+
// stillOnUnfollowedRemotePath determines whether the next link load will be from
1515
// a path missing from the remote
1616
// if it won't be, based on the linear nature of selector traversals, it wipes
1717
// the last missing state
18-
func (pt *pathTracker) stillOnMissingRemotePath(newPath datamodel.Path) bool {
18+
func (pt *pathTracker) stillOnUnfollowedRemotePath(newPath datamodel.Path) bool {
1919
// is there a known missing path?
20-
if pt.lastMissingRemotePath.Len() == 0 {
20+
if pt.lastUnfollowedRemotePath.Len() == 0 {
2121
return false
2222
}
2323
// are we still on it?
24-
if newPath.Len() <= pt.lastMissingRemotePath.Len() {
24+
if newPath.Len() <= pt.lastUnfollowedRemotePath.Len() {
2525
// if not, reset to no known missing remote path
26-
pt.lastMissingRemotePath = datamodel.NewPath(nil)
26+
pt.lastUnfollowedRemotePath = datamodel.NewPath(nil)
2727
return false
2828
}
2929
// otherwise we're on a missing path
@@ -34,8 +34,8 @@ func (pt *pathTracker) stillOnMissingRemotePath(newPath datamodel.Path) bool {
3434
// at the given path
3535
func (pt *pathTracker) recordRemoteLoadAttempt(currentPath datamodel.Path, action graphsync.LinkAction) {
3636
// if the last remote link was missing
37-
if action == graphsync.LinkActionMissing {
37+
if !action.DidFollowLink() {
3838
// record the last known missing path
39-
pt.lastMissingRemotePath = currentPath
39+
pt.lastUnfollowedRemotePath = currentPath
4040
}
4141
}

‎requestmanager/reconciledloader/reconciledloader_test.go

+138
Original file line numberDiff line numberDiff line change
@@ -218,6 +218,29 @@ func TestReconciledLoader(t *testing.T) {
218218
},
219219
),
220220
},
221+
"remote skips DAG": {
222+
root: testChain.TipLink.(cidlink.Link).Cid,
223+
baseStore: testBCStorage,
224+
presentRemoteBlocks: testChain.AllBlocks(),
225+
remoteSeq: append(metadataRange(testChain, 0, 30, false),
226+
message.GraphSyncLinkMetadatum{
227+
Link: testChain.LinkTipIndex(30).(cidlink.Link).Cid,
228+
Action: graphsync.LinkActionDuplicateDAGSkipped,
229+
}),
230+
steps: append(append([]step{
231+
goOnline{},
232+
injest{metadataStart: 0, metadataEnd: 31},
233+
}, syncLoadRange(testChain, 0, 30, false)...),
234+
// we should get an error that we're missing a block for our response
235+
syncLoad{
236+
loadSeq: 30,
237+
expectedResult: types.AsyncLoadResult{Local: true, Err: graphsync.RemoteMissingBlockErr{
238+
Link: testChain.LinkTipIndex(30),
239+
Path: testChain.PathTipIndex(30),
240+
}},
241+
},
242+
),
243+
},
221244
"remote missing chain that local has": {
222245
root: testChain.TipLink.(cidlink.Link).Cid,
223246
baseStore: testBCStorage,
@@ -251,6 +274,39 @@ func TestReconciledLoader(t *testing.T) {
251274
syncLoadRange(testChain, 30, 100, true)...,
252275
),
253276
},
277+
"remote skips chain that local has": {
278+
root: testChain.TipLink.(cidlink.Link).Cid,
279+
baseStore: testBCStorage,
280+
presentRemoteBlocks: testChain.AllBlocks(),
281+
presentLocalBlocks: testChain.Blocks(30, 100),
282+
remoteSeq: append(metadataRange(testChain, 0, 30, false),
283+
message.GraphSyncLinkMetadatum{
284+
Link: testChain.LinkTipIndex(30).(cidlink.Link).Cid,
285+
Action: graphsync.LinkActionDuplicateDAGSkipped,
286+
}),
287+
steps: append(append(append(
288+
[]step{
289+
goOnline{},
290+
injest{metadataStart: 0, metadataEnd: 31},
291+
},
292+
// load the blocks the remote has
293+
syncLoadRange(testChain, 0, 30, false)...),
294+
[]step{
295+
// load the block the remote missing says it's missing locally
296+
syncLoadRange(testChain, 30, 31, true)[0],
297+
asyncLoad{loadSeq: 31},
298+
// at this point we have no more remote responses, since it's a linear chain
299+
verifyNoAsyncResult{},
300+
// we'd expect the remote would terminate here, since we've sent the last missing block
301+
goOffline{},
302+
// this will cause us to start loading locally only again
303+
verifyAsyncResult{
304+
expectedResult: types.AsyncLoadResult{Local: true, Data: testChain.Blocks(31, 32)[0].RawData()},
305+
},
306+
}...),
307+
syncLoadRange(testChain, 30, 100, true)...,
308+
),
309+
},
254310
"remote missing chain that local has partial": {
255311
root: testChain.TipLink.(cidlink.Link).Cid,
256312
baseStore: testBCStorage,
@@ -326,6 +382,62 @@ func TestReconciledLoader(t *testing.T) {
326382
syncLoad{loadSeq: 7, expectedResult: types.AsyncLoadResult{Local: true, Data: testTree.LeafAlphaBlock.RawData()}},
327383
},
328384
},
385+
"remote duplicate not sent, blocks can load from local": {
386+
root: testTree.RootBlock.Cid(),
387+
baseStore: testTree.Storage,
388+
presentRemoteBlocks: []blocks.Block{
389+
testTree.RootBlock,
390+
testTree.MiddleListBlock,
391+
testTree.MiddleMapBlock,
392+
testTree.LeafAlphaBlock,
393+
testTree.LeafBetaBlock,
394+
},
395+
presentLocalBlocks: nil,
396+
remoteSeq: []message.GraphSyncLinkMetadatum{
397+
{Link: testTree.RootBlock.Cid(), Action: graphsync.LinkActionPresent},
398+
{Link: testTree.MiddleListBlock.Cid(), Action: graphsync.LinkActionPresent},
399+
{Link: testTree.LeafAlphaBlock.Cid(), Action: graphsync.LinkActionPresent},
400+
{Link: testTree.LeafAlphaBlock.Cid(), Action: graphsync.LinkActionDuplicateNotSent},
401+
{Link: testTree.LeafBetaBlock.Cid(), Action: graphsync.LinkActionPresent},
402+
{Link: testTree.LeafAlphaBlock.Cid(), Action: graphsync.LinkActionDuplicateNotSent},
403+
{Link: testTree.MiddleMapBlock.Cid(), Action: graphsync.LinkActionPresent},
404+
{Link: testTree.LeafAlphaBlock.Cid(), Action: graphsync.LinkActionDuplicateNotSent},
405+
},
406+
steps: []step{
407+
goOnline{},
408+
injest{metadataStart: 0, metadataEnd: 8},
409+
syncLoad{loadSeq: 0, expectedResult: types.AsyncLoadResult{Local: false, Data: testTree.RootBlock.RawData()}},
410+
syncLoad{loadSeq: 1, expectedResult: types.AsyncLoadResult{Local: false, Data: testTree.MiddleListBlock.RawData()}},
411+
syncLoad{loadSeq: 2, expectedResult: types.AsyncLoadResult{Local: false, Data: testTree.LeafAlphaBlock.RawData()}},
412+
syncLoad{loadSeq: 3, expectedResult: types.AsyncLoadResult{Local: true, Data: testTree.LeafAlphaBlock.RawData()}},
413+
syncLoad{loadSeq: 4, expectedResult: types.AsyncLoadResult{Local: false, Data: testTree.LeafBetaBlock.RawData()}},
414+
syncLoad{loadSeq: 5, expectedResult: types.AsyncLoadResult{Local: true, Data: testTree.LeafAlphaBlock.RawData()}},
415+
syncLoad{loadSeq: 6, expectedResult: types.AsyncLoadResult{Local: false, Data: testTree.MiddleMapBlock.RawData()}},
416+
syncLoad{loadSeq: 7, expectedResult: types.AsyncLoadResult{Local: true, Data: testTree.LeafAlphaBlock.RawData()}},
417+
},
418+
},
419+
"remote duplicate not sent load from local even when present in message": {
420+
root: testTree.RootBlock.Cid(),
421+
baseStore: testTree.Storage,
422+
presentRemoteBlocks: []blocks.Block{
423+
testTree.RootBlock,
424+
testTree.MiddleListBlock,
425+
},
426+
presentLocalBlocks: []blocks.Block{
427+
testTree.MiddleListBlock,
428+
},
429+
remoteSeq: []message.GraphSyncLinkMetadatum{
430+
{Link: testTree.RootBlock.Cid(), Action: graphsync.LinkActionPresent},
431+
{Link: testTree.MiddleListBlock.Cid(), Action: graphsync.LinkActionDuplicateNotSent},
432+
},
433+
steps: []step{
434+
goOnline{},
435+
injest{metadataStart: 0, metadataEnd: 2},
436+
syncLoad{loadSeq: 0, expectedResult: types.AsyncLoadResult{Local: false, Data: testTree.RootBlock.RawData()}},
437+
syncLoad{loadSeq: 1, expectedResult: types.AsyncLoadResult{Local: true, Data: testTree.MiddleListBlock.RawData()}},
438+
},
439+
},
440+
329441
"remote missing branch finishes to end": {
330442
root: testTree.RootBlock.Cid(),
331443
baseStore: testTree.Storage,
@@ -351,6 +463,32 @@ func TestReconciledLoader(t *testing.T) {
351463
syncLoad{loadSeq: 7, expectedResult: types.AsyncLoadResult{Local: false, Data: testTree.LeafAlphaBlock.RawData()}},
352464
},
353465
},
466+
467+
"remote skipping branch finishes to end": {
468+
root: testTree.RootBlock.Cid(),
469+
baseStore: testTree.Storage,
470+
presentRemoteBlocks: []blocks.Block{
471+
testTree.RootBlock,
472+
testTree.MiddleMapBlock,
473+
testTree.LeafAlphaBlock,
474+
},
475+
presentLocalBlocks: nil,
476+
remoteSeq: []message.GraphSyncLinkMetadatum{
477+
{Link: testTree.RootBlock.Cid(), Action: graphsync.LinkActionPresent},
478+
// missing the whole list tree
479+
{Link: testTree.MiddleListBlock.Cid(), Action: graphsync.LinkActionDuplicateDAGSkipped},
480+
{Link: testTree.MiddleMapBlock.Cid(), Action: graphsync.LinkActionPresent},
481+
{Link: testTree.LeafAlphaBlock.Cid(), Action: graphsync.LinkActionPresent},
482+
},
483+
steps: []step{
484+
goOnline{},
485+
injest{metadataStart: 0, metadataEnd: 4},
486+
syncLoad{loadSeq: 0, expectedResult: types.AsyncLoadResult{Local: false, Data: testTree.RootBlock.RawData()}},
487+
syncLoad{loadSeq: 1, expectedResult: types.AsyncLoadResult{Local: true, Err: graphsync.RemoteMissingBlockErr{Link: testTree.MiddleListNodeLnk, Path: datamodel.ParsePath("linkedList")}}},
488+
syncLoad{loadSeq: 6, expectedResult: types.AsyncLoadResult{Local: false, Data: testTree.MiddleMapBlock.RawData()}},
489+
syncLoad{loadSeq: 7, expectedResult: types.AsyncLoadResult{Local: false, Data: testTree.LeafAlphaBlock.RawData()}},
490+
},
491+
},
354492
"remote missing branch with partial local": {
355493
root: testTree.RootBlock.Cid(),
356494
baseStore: testTree.Storage,

0 commit comments

Comments
 (0)