From 546aa2265e884e2418c14baffaad20c18b180ac9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Fri, 17 Dec 2021 18:27:41 +0000 Subject: [PATCH 01/21] WIP: add the ipld plus dag-cbor protocol v1.1 --- go.mod | 2 +- go.sum | 4 +- message/ipldcbor/gen_test.go | 26 +++++++++++ message/ipldcbor/schema.go | 23 ++++++++++ message/ipldcbor/schema.ipldsch | 78 +++++++++++++++++++++++++++++++++ message/ipldcbor/schema_test.go | 6 +++ message/ipldcbor/types.go | 2 + 7 files changed, 138 insertions(+), 3 deletions(-) create mode 100644 message/ipldcbor/gen_test.go create mode 100644 message/ipldcbor/schema.go create mode 100644 message/ipldcbor/schema.ipldsch create mode 100644 message/ipldcbor/schema_test.go create mode 100644 message/ipldcbor/types.go diff --git a/go.mod b/go.mod index 641b1177..1a2a4d47 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/ipfs/go-peertaskqueue v0.7.1 github.com/ipfs/go-unixfs v0.2.4 github.com/ipld/go-codec-dagpb v1.3.0 - github.com/ipld/go-ipld-prime v0.14.3 + github.com/ipld/go-ipld-prime v0.14.4-0.20211217152141-008fd70fc96f github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c github.com/libp2p/go-buffer-pool v0.0.2 github.com/libp2p/go-libp2p v0.16.0 diff --git a/go.sum b/go.sum index 3ee093b9..0567bc1c 100644 --- a/go.sum +++ b/go.sum @@ -457,8 +457,8 @@ github.com/ipld/go-codec-dagpb v1.3.0 h1:czTcaoAuNNyIYWs6Qe01DJ+sEX7B+1Z0LcXjSat github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA= github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= -github.com/ipld/go-ipld-prime v0.14.3 h1:cGUmxSws2IHurn00/iLMDapeXsnf9+FyAtYVy8G/JsQ= -github.com/ipld/go-ipld-prime v0.14.3/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= +github.com/ipld/go-ipld-prime v0.14.4-0.20211217152141-008fd70fc96f h1:6ISKbCjgF2pR2W/YRNBeTV7XL0+d+r6h9vRoer0zFm8= +github.com/ipld/go-ipld-prime v0.14.4-0.20211217152141-008fd70fc96f/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= diff --git a/message/ipldcbor/gen_test.go b/message/ipldcbor/gen_test.go new file mode 100644 index 00000000..e25c333b --- /dev/null +++ b/message/ipldcbor/gen_test.go @@ -0,0 +1,26 @@ +//go:build bindnodegen +// +build bindnodegen + +package ipldcbor + +import ( + "fmt" + "os" + "testing" + + "github.com/ipld/go-ipld-prime/node/bindnode" +) + +func TestGenerate(t *testing.T) { + f, err := os.Create("types.go") + if err != nil { + t.Fatal(err) + } + fmt.Fprintf(f, "package ipldcbor\n\n") + if err := bindnode.ProduceGoTypes(f, schemaTypeSystem); err != nil { + t.Fatal(err) + } + if err := f.Close(); err != nil { + t.Fatal(err) + } +} diff --git a/message/ipldcbor/schema.go b/message/ipldcbor/schema.go new file mode 100644 index 00000000..61f5fc02 --- /dev/null +++ b/message/ipldcbor/schema.go @@ -0,0 +1,23 @@ +package ipldcbor + +//go:generate go test -run=Generate -vet=off -tags=bindnodegen + +import ( + _ "embed" + + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/schema" +) + +//go:embed schema.ipldsch +var embedSchema []byte + +var schemaTypeSystem *schema.TypeSystem + +func init() { + ts, err := ipld.LoadSchemaBytes(embedSchema) + if err != nil { + panic(err) + } + schemaTypeSystem = ts +} diff --git a/message/ipldcbor/schema.ipldsch b/message/ipldcbor/schema.ipldsch new file mode 100644 index 00000000..84609eae --- /dev/null +++ b/message/ipldcbor/schema.ipldsch @@ -0,0 +1,78 @@ +# TODO(mvdan): upstream must support Any sooner than later +type Any union { + | Bool bool + | Int int + | Float float + | String string + | Bytes bytes + # TODO(mvdan): these are missing from upstream because of Any, too + # | Map map + # | List list + # | Link link +} representation kinded + +type GraphSyncExtensions {String:Any} +type GraphSyncRequestID int +type GraphSyncPriority int + +type GraphSyncMetadatum struct { + link Link # TODO(mvdan): support "&" links in the ipld-prime schema loader + blockPresent Bool +} representation tuple + +type GraphSyncMetadata [GraphSyncMetadatum] + +type GraphSyncResponseCode enum { + # Informational Codes (request in progress) + + | RequestAcknowledged ("10") + | AdditionalPeers ("11") + | NotEnoughGas ("12") + | OtherProtocol ("13") + | PartialResponse ("14") + | RequestPaused ("15") + + # Success Response Codes (request terminated) + + | RequestCompletedFull ("20") + | RequestCompletedPartial ("21") + + # Error Response Codes (request terminated) + + | RequestRejected ("30") + | RequestFailedBusy ("31") + | RequestFailedUnknown ("32") + | RequestFailedLegal ("33") + | RequestFailedContentNotFound ("34") + | RequestCancelled ("35") +} representation int + +type GraphSyncRequest struct { + id GraphSyncRequestID (rename "ID") # unique id set on the requester side + root Link (rename "Root") # a CID for the root node in the query + # TODO(mvdan): must we copy-paste the selector schema here? + # selector Selector (rename "Sel") # see https://github.com/ipld/specs/blob/master/selectors/selectors.md + extensions GraphSyncExtensions (rename "Ext") # side channel information + priority GraphSyncPriority (rename "Pri") # the priority (normalized). default to 1 + cancel Bool (rename "Canc") # whether this cancels a request + update Bool (rename "Updt") # whether this is an update to an in progress request +} representation map + +type GraphSyncResponse struct { + id GraphSyncRequestID (rename "ID") # the request id we are responding to + # TODO(mvdan): this type appears to be missing + # status GraphSyncResponseStatusCode (rename "Stat") # a status code. + metadata GraphSyncMetadata (rename "Meta") # metadata about response + extensions GraphSyncExtensions (rename "Ext") # side channel information +} representation map + +type GraphSyncBlock struct { + prefix Bytes (rename "Pre") # CID prefix (cid version, multicodec and multihash prefix (type + length) + data Bytes (rename "Data") +} representation map + +type GraphSyncMessage struct { + requests [GraphSyncRequest] (rename "Reqs") + responses [GraphSyncResponse] (rename "Rsps") + blocks [GraphSyncBlock] (rename "Blks") +} representation map diff --git a/message/ipldcbor/schema_test.go b/message/ipldcbor/schema_test.go new file mode 100644 index 00000000..f6f5eeb3 --- /dev/null +++ b/message/ipldcbor/schema_test.go @@ -0,0 +1,6 @@ +package ipldcbor_test + +import "testing" + +func TestSchema(t *testing.T) { +} diff --git a/message/ipldcbor/types.go b/message/ipldcbor/types.go new file mode 100644 index 00000000..1127cb7a --- /dev/null +++ b/message/ipldcbor/types.go @@ -0,0 +1,2 @@ +package ipldcbor + From c06a9cd8aaf7baee2fd7b313bdd38053363cc345 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Mon, 10 Jan 2022 15:44:38 +0000 Subject: [PATCH 02/21] more WIP: upstream supports Any now --- go.mod | 2 ++ message/ipldcbor/schema.ipldsch | 13 ----------- message/ipldcbor/types.go | 40 +++++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index 1a2a4d47..3d33da9a 100644 --- a/go.mod +++ b/go.mod @@ -44,3 +44,5 @@ require ( golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 google.golang.org/protobuf v1.27.1 ) + +replace github.com/ipld/go-ipld-prime => ../../src/ipld diff --git a/message/ipldcbor/schema.ipldsch b/message/ipldcbor/schema.ipldsch index 84609eae..73c3379c 100644 --- a/message/ipldcbor/schema.ipldsch +++ b/message/ipldcbor/schema.ipldsch @@ -1,16 +1,3 @@ -# TODO(mvdan): upstream must support Any sooner than later -type Any union { - | Bool bool - | Int int - | Float float - | String string - | Bytes bytes - # TODO(mvdan): these are missing from upstream because of Any, too - # | Map map - # | List list - # | Link link -} representation kinded - type GraphSyncExtensions {String:Any} type GraphSyncRequestID int type GraphSyncPriority int diff --git a/message/ipldcbor/types.go b/message/ipldcbor/types.go index 1127cb7a..3fc989e2 100644 --- a/message/ipldcbor/types.go +++ b/message/ipldcbor/types.go @@ -1,2 +1,42 @@ package ipldcbor +type Map struct { + Keys []string + Values map[string]datamodel.Node +} +type List []datamodel.Node +type GraphSyncExtensions struct { + Keys []string + Values map[string]datamodel.Node +} +type GraphSyncMetadatum struct { + Link datamodel.Link + BlockPresent bool +} +type GraphSyncMetadata []GraphSyncMetadatum +type GraphSyncResponseCode string +type GraphSyncRequest struct { + Id int + Root datamodel.Link + Extensions GraphSyncExtensions + Priority int + Cancel bool + Update bool +} +type GraphSyncResponse struct { + Id int + Metadata GraphSyncMetadata + Extensions GraphSyncExtensions +} +type GraphSyncBlock struct { + Prefix []uint8 + Data []uint8 +} +type List__GraphSyncRequest []GraphSyncRequest +type List__GraphSyncResponse []GraphSyncResponse +type List__GraphSyncBlock []GraphSyncBlock +type GraphSyncMessage struct { + Requests List__GraphSyncRequest + Responses List__GraphSyncResponse + Blocks List__GraphSyncBlock +} From 483c4f5981d54fd6206a401de0ae6b844a53cc29 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Mon, 10 Jan 2022 15:48:54 +0000 Subject: [PATCH 03/21] start using bindnode.Prototype --- message/ipldcbor/schema.go | 7 +++++++ message/ipldcbor/types.go | 2 ++ 2 files changed, 9 insertions(+) diff --git a/message/ipldcbor/schema.go b/message/ipldcbor/schema.go index 61f5fc02..a2876c2a 100644 --- a/message/ipldcbor/schema.go +++ b/message/ipldcbor/schema.go @@ -6,6 +6,7 @@ import ( _ "embed" "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/node/bindnode" "github.com/ipld/go-ipld-prime/schema" ) @@ -14,10 +15,16 @@ var embedSchema []byte var schemaTypeSystem *schema.TypeSystem +var Prototype struct { + Message schema.TypedPrototype +} + func init() { ts, err := ipld.LoadSchemaBytes(embedSchema) if err != nil { panic(err) } schemaTypeSystem = ts + + Prototype.Message = bindnode.Prototype((*GraphSyncMessage)(nil), ts.TypeByName("GraphSyncMessage")) } diff --git a/message/ipldcbor/types.go b/message/ipldcbor/types.go index 3fc989e2..ba6ce69f 100644 --- a/message/ipldcbor/types.go +++ b/message/ipldcbor/types.go @@ -1,5 +1,7 @@ package ipldcbor +import "github.com/ipld/go-ipld-prime/datamodel" + type Map struct { Keys []string Values map[string]datamodel.Node From a6949b7b337d05d533441cb13f2aaec29bb31835 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Mon, 10 Jan 2022 16:27:57 +0000 Subject: [PATCH 04/21] schema is fully working now --- message/ipldcbor/gen_test.go | 1 + message/ipldcbor/schema.go | 3 +-- message/ipldcbor/schema.ipldsch | 10 ++++------ message/ipldcbor/types.go | 4 +++- 4 files changed, 9 insertions(+), 9 deletions(-) diff --git a/message/ipldcbor/gen_test.go b/message/ipldcbor/gen_test.go index e25c333b..e5051863 100644 --- a/message/ipldcbor/gen_test.go +++ b/message/ipldcbor/gen_test.go @@ -17,6 +17,7 @@ func TestGenerate(t *testing.T) { t.Fatal(err) } fmt.Fprintf(f, "package ipldcbor\n\n") + fmt.Fprintf(f, "import \"github.com/ipld/go-ipld-prime/datamodel\"\n\n") if err := bindnode.ProduceGoTypes(f, schemaTypeSystem); err != nil { t.Fatal(err) } diff --git a/message/ipldcbor/schema.go b/message/ipldcbor/schema.go index a2876c2a..2dd0499f 100644 --- a/message/ipldcbor/schema.go +++ b/message/ipldcbor/schema.go @@ -6,7 +6,6 @@ import ( _ "embed" "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/node/bindnode" "github.com/ipld/go-ipld-prime/schema" ) @@ -26,5 +25,5 @@ func init() { } schemaTypeSystem = ts - Prototype.Message = bindnode.Prototype((*GraphSyncMessage)(nil), ts.TypeByName("GraphSyncMessage")) + // Prototype.Message = bindnode.Prototype((*GraphSyncMessage)(nil), ts.TypeByName("GraphSyncMessage")) } diff --git a/message/ipldcbor/schema.ipldsch b/message/ipldcbor/schema.ipldsch index 73c3379c..89fc5a3d 100644 --- a/message/ipldcbor/schema.ipldsch +++ b/message/ipldcbor/schema.ipldsch @@ -3,13 +3,13 @@ type GraphSyncRequestID int type GraphSyncPriority int type GraphSyncMetadatum struct { - link Link # TODO(mvdan): support "&" links in the ipld-prime schema loader + link Link blockPresent Bool } representation tuple type GraphSyncMetadata [GraphSyncMetadatum] -type GraphSyncResponseCode enum { +type GraphSyncResponseStatusCode enum { # Informational Codes (request in progress) | RequestAcknowledged ("10") @@ -37,8 +37,7 @@ type GraphSyncResponseCode enum { type GraphSyncRequest struct { id GraphSyncRequestID (rename "ID") # unique id set on the requester side root Link (rename "Root") # a CID for the root node in the query - # TODO(mvdan): must we copy-paste the selector schema here? - # selector Selector (rename "Sel") # see https://github.com/ipld/specs/blob/master/selectors/selectors.md + selector Any (rename "Sel") # see https://github.com/ipld/specs/blob/master/selectors/selectors.md extensions GraphSyncExtensions (rename "Ext") # side channel information priority GraphSyncPriority (rename "Pri") # the priority (normalized). default to 1 cancel Bool (rename "Canc") # whether this cancels a request @@ -47,8 +46,7 @@ type GraphSyncRequest struct { type GraphSyncResponse struct { id GraphSyncRequestID (rename "ID") # the request id we are responding to - # TODO(mvdan): this type appears to be missing - # status GraphSyncResponseStatusCode (rename "Stat") # a status code. + status GraphSyncResponseStatusCode (rename "Stat") # a status code. metadata GraphSyncMetadata (rename "Meta") # metadata about response extensions GraphSyncExtensions (rename "Ext") # side channel information } representation map diff --git a/message/ipldcbor/types.go b/message/ipldcbor/types.go index ba6ce69f..ec6fd871 100644 --- a/message/ipldcbor/types.go +++ b/message/ipldcbor/types.go @@ -16,10 +16,11 @@ type GraphSyncMetadatum struct { BlockPresent bool } type GraphSyncMetadata []GraphSyncMetadatum -type GraphSyncResponseCode string +type GraphSyncResponseStatusCode string type GraphSyncRequest struct { Id int Root datamodel.Link + Selector datamodel.Node Extensions GraphSyncExtensions Priority int Cancel bool @@ -27,6 +28,7 @@ type GraphSyncRequest struct { } type GraphSyncResponse struct { Id int + Status GraphSyncResponseStatusCode Metadata GraphSyncMetadata Extensions GraphSyncExtensions } From da225d301b9c9344f44601b25fd84650bfd97c72 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Mon, 10 Jan 2022 16:33:44 +0000 Subject: [PATCH 05/21] remove the replace directive after the ipld merge --- go.mod | 4 +--- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index 3d33da9a..35bf72ab 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/ipfs/go-peertaskqueue v0.7.1 github.com/ipfs/go-unixfs v0.2.4 github.com/ipld/go-codec-dagpb v1.3.0 - github.com/ipld/go-ipld-prime v0.14.4-0.20211217152141-008fd70fc96f + github.com/ipld/go-ipld-prime v0.14.4-0.20220110161855-fc09d6b768e9 github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c github.com/libp2p/go-buffer-pool v0.0.2 github.com/libp2p/go-libp2p v0.16.0 @@ -44,5 +44,3 @@ require ( golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 google.golang.org/protobuf v1.27.1 ) - -replace github.com/ipld/go-ipld-prime => ../../src/ipld diff --git a/go.sum b/go.sum index 0567bc1c..7c422042 100644 --- a/go.sum +++ b/go.sum @@ -457,8 +457,8 @@ github.com/ipld/go-codec-dagpb v1.3.0 h1:czTcaoAuNNyIYWs6Qe01DJ+sEX7B+1Z0LcXjSat github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA= github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= -github.com/ipld/go-ipld-prime v0.14.4-0.20211217152141-008fd70fc96f h1:6ISKbCjgF2pR2W/YRNBeTV7XL0+d+r6h9vRoer0zFm8= -github.com/ipld/go-ipld-prime v0.14.4-0.20211217152141-008fd70fc96f/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= +github.com/ipld/go-ipld-prime v0.14.4-0.20220110161855-fc09d6b768e9 h1:fqQSvdPznhyE5jZoCwbvykHbK+QukxJcS1SFWbj9ig0= +github.com/ipld/go-ipld-prime v0.14.4-0.20220110161855-fc09d6b768e9/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= From 15e34dc41ac4d285e9385b62723207710ebad361 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Mon, 10 Jan 2022 21:21:19 +0000 Subject: [PATCH 06/21] start refactoring types --- message/message.go | 212 +++++++++++++++++++-------------------------- 1 file changed, 89 insertions(+), 123 deletions(-) diff --git a/message/message.go b/message/message.go index 6e818847..289d36e5 100644 --- a/message/message.go +++ b/message/message.go @@ -9,6 +9,7 @@ import ( blocks "github.com/ipfs/go-block-format" cid "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" pool "github.com/libp2p/go-buffer-pool" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-msgio" @@ -46,30 +47,48 @@ type Exportable interface { ToNet(w io.Writer) error } +type GraphSyncExtensions struct { + Keys []string + Values map[string]datamodel.Node +} + // GraphSyncRequest is a struct to capture data on a request contained in a // GraphSyncMessage. type GraphSyncRequest struct { - root cid.Cid - selector ipld.Node - priority graphsync.Priority - id graphsync.RequestID - extensions map[string][]byte - isCancel bool - isUpdate bool + ID graphsync.RequestID + + Root cid.Cid + Selector ipld.Node + Extensions GraphSyncExtensions + Priority graphsync.Priority + Cancel bool + Update bool +} + +type GraphSyncMetadatum struct { + Link datamodel.Link + BlockPresent bool } // GraphSyncResponse is an struct to capture data on a response sent back // in a GraphSyncMessage. type GraphSyncResponse struct { - requestID graphsync.RequestID - status graphsync.ResponseStatusCode - extensions map[string][]byte + ID graphsync.RequestID + + Status graphsync.ResponseStatusCode + Metadata []GraphSyncMetadatum + Extensions GraphSyncExtensions +} + +type GraphSyncBlock struct { + Prefix []byte + Data []byte } type GraphSyncMessage struct { - requests map[graphsync.RequestID]GraphSyncRequest - responses map[graphsync.RequestID]GraphSyncResponse - blocks map[cid.Cid]blocks.Block + Requests []GraphSyncRequest + Responses []GraphSyncResponse + Blocks []GraphSyncBlock } // NewRequest builds a new Graphsync request @@ -77,29 +96,37 @@ func NewRequest(id graphsync.RequestID, root cid.Cid, selector ipld.Node, priority graphsync.Priority, - extensions ...graphsync.ExtensionData) GraphSyncRequest { + extensions ...NamedExtension) GraphSyncRequest { return newRequest(id, root, selector, priority, false, false, toExtensionsMap(extensions)) } // CancelRequest request generates a request to cancel an in progress request func CancelRequest(id graphsync.RequestID) GraphSyncRequest { - return newRequest(id, cid.Cid{}, nil, 0, true, false, nil) + return newRequest(id, cid.Cid{}, nil, 0, true, false, GraphSyncExtensions{}) } // UpdateRequest generates a new request to update an in progress request with the given extensions -func UpdateRequest(id graphsync.RequestID, extensions ...graphsync.ExtensionData) GraphSyncRequest { +func UpdateRequest(id graphsync.RequestID, extensions ...NamedExtension) GraphSyncRequest { return newRequest(id, cid.Cid{}, nil, 0, false, true, toExtensionsMap(extensions)) } -func toExtensionsMap(extensions []graphsync.ExtensionData) (extensionsMap map[string][]byte) { +// NamedExtension exists just for the purpose of the constructors. +type NamedExtension struct { + Name string + Data ipld.Node +} + +func toExtensionsMap(extensions []NamedExtension) (m GraphSyncExtensions) { if len(extensions) > 0 { - extensionsMap = make(map[string][]byte, len(extensions)) - for _, extension := range extensions { - extensionsMap[string(extension.Name)] = extension.Data + m.Keys = make([]string, len(extensions)) + m.Values = make(map[string]ipld.Node, len(extensions)) + for i, ext := range extensions { + m.Keys[i] = ext.Name + m.Values[ext.Name] = ext.Data } } - return + return m } func newRequest(id graphsync.RequestID, @@ -108,37 +135,37 @@ func newRequest(id graphsync.RequestID, priority graphsync.Priority, isCancel bool, isUpdate bool, - extensions map[string][]byte) GraphSyncRequest { + extensions GraphSyncExtensions) GraphSyncRequest { return GraphSyncRequest{ - id: id, - root: root, - selector: selector, - priority: priority, - isCancel: isCancel, - isUpdate: isUpdate, - extensions: extensions, + ID: id, + Root: root, + Selector: selector, + Priority: priority, + Cancel: isCancel, + Update: isUpdate, + Extensions: extensions, } } // NewResponse builds a new Graphsync response func NewResponse(requestID graphsync.RequestID, status graphsync.ResponseStatusCode, - extensions ...graphsync.ExtensionData) GraphSyncResponse { + extensions ...NamedExtension) GraphSyncResponse { return newResponse(requestID, status, toExtensionsMap(extensions)) } func newResponse(requestID graphsync.RequestID, - status graphsync.ResponseStatusCode, extensions map[string][]byte) GraphSyncResponse { + status graphsync.ResponseStatusCode, extensions GraphSyncExtensions) GraphSyncResponse { return GraphSyncResponse{ - requestID: requestID, - status: status, - extensions: extensions, + ID: requestID, + Status: status, + Extensions: extensions, } } func newMessageFromProto(pbm *pb.Message) (GraphSyncMessage, error) { - requests := make(map[graphsync.RequestID]GraphSyncRequest, len(pbm.GetRequests())) - for _, req := range pbm.Requests { + requests := make([]GraphSyncRequest, len(pbm.GetRequests())) + for i, req := range pbm.Requests { if req == nil { return GraphSyncMessage{}, errors.New("request is nil") } @@ -162,11 +189,11 @@ func newMessageFromProto(pbm *pb.Message) (GraphSyncMessage, error) { if exts == nil { exts = make(map[string][]byte) } - requests[graphsync.RequestID(req.Id)] = newRequest(graphsync.RequestID(req.Id), root, selector, graphsync.Priority(req.Priority), req.Cancel, req.Update, exts) + requests[i] = newRequest(graphsync.RequestID(req.Id), root, selector, graphsync.Priority(req.Priority), req.Cancel, req.Update, exts) } - responses := make(map[graphsync.RequestID]GraphSyncResponse, len(pbm.GetResponses())) - for _, res := range pbm.Responses { + responses := make([]GraphSyncResponse, len(pbm.GetResponses())) + for i, res := range pbm.Responses { if res == nil { return GraphSyncMessage{}, errors.New("response is nil") } @@ -174,31 +201,18 @@ func newMessageFromProto(pbm *pb.Message) (GraphSyncMessage, error) { if exts == nil { exts = make(map[string][]byte) } - responses[graphsync.RequestID(res.Id)] = newResponse(graphsync.RequestID(res.Id), graphsync.ResponseStatusCode(res.Status), exts) + responses[i] = newResponse(graphsync.RequestID(res.Id), graphsync.ResponseStatusCode(res.Status), exts) } - blks := make(map[cid.Cid]blocks.Block, len(pbm.GetData())) - for _, b := range pbm.GetData() { + blks := make([]GraphSyncBlock, len(pbm.GetData())) + for i, b := range pbm.GetData() { if b == nil { return GraphSyncMessage{}, errors.New("block is nil") } - - pref, err := cid.PrefixFromBytes(b.GetPrefix()) - if err != nil { - return GraphSyncMessage{}, err - } - - c, err := pref.Sum(b.GetData()) - if err != nil { - return GraphSyncMessage{}, err - } - - blk, err := blocks.NewBlockWithCid(b.GetData(), c) - if err != nil { - return GraphSyncMessage{}, err + blks[i] = GraphSyncBlock{ + Prefix: b.GetPrefix(), + Data: b.GetData(), } - - blks[blk.Cid()] = blk } return GraphSyncMessage{ @@ -207,41 +221,17 @@ func newMessageFromProto(pbm *pb.Message) (GraphSyncMessage, error) { } func (gsm GraphSyncMessage) Empty() bool { - return len(gsm.blocks) == 0 && len(gsm.requests) == 0 && len(gsm.responses) == 0 -} - -func (gsm GraphSyncMessage) Requests() []GraphSyncRequest { - requests := make([]GraphSyncRequest, 0, len(gsm.requests)) - for _, request := range gsm.requests { - requests = append(requests, request) - } - return requests + return len(gsm.Blocks) == 0 && len(gsm.Requests) == 0 && len(gsm.Responses) == 0 } func (gsm GraphSyncMessage) ResponseCodes() map[graphsync.RequestID]graphsync.ResponseStatusCode { - codes := make(map[graphsync.RequestID]graphsync.ResponseStatusCode, len(gsm.responses)) - for id, response := range gsm.responses { + codes := make(map[graphsync.RequestID]graphsync.ResponseStatusCode, len(gsm.Responses)) + for id, response := range gsm.Responses { codes[id] = response.Status() } return codes } -func (gsm GraphSyncMessage) Responses() []GraphSyncResponse { - responses := make([]GraphSyncResponse, 0, len(gsm.responses)) - for _, response := range gsm.responses { - responses = append(responses, response) - } - return responses -} - -func (gsm GraphSyncMessage) Blocks() []blocks.Block { - bs := make([]blocks.Block, 0, len(gsm.blocks)) - for _, block := range gsm.blocks { - bs = append(bs, block) - } - return bs -} - // FromNet can read a network stream to deserialized a GraphSyncMessage func FromNet(r io.Reader) (GraphSyncMessage, error) { reader := msgio.NewVarintReaderSize(r, network.MessageSizeMax) @@ -267,8 +257,8 @@ func FromMsgReader(r msgio.Reader) (GraphSyncMessage, error) { func (gsm GraphSyncMessage) ToProto() (*pb.Message, error) { pbm := new(pb.Message) - pbm.Requests = make([]*pb.Message_Request, 0, len(gsm.requests)) - for _, request := range gsm.requests { + pbm.Requests = make([]*pb.Message_Request, 0, len(gsm.Requests)) + for _, request := range gsm.Requests { var selector []byte var err error if request.selector != nil { @@ -288,8 +278,8 @@ func (gsm GraphSyncMessage) ToProto() (*pb.Message, error) { }) } - pbm.Responses = make([]*pb.Message_Response, 0, len(gsm.responses)) - for _, response := range gsm.responses { + pbm.Responses = make([]*pb.Message_Response, 0, len(gsm.Responses)) + for _, response := range gsm.Responses { pbm.Responses = append(pbm.Responses, &pb.Message_Response{ Id: int32(response.requestID), Status: int32(response.status), @@ -328,13 +318,13 @@ func (gsm GraphSyncMessage) ToNet(w io.Writer) error { } func (gsm GraphSyncMessage) Loggable() map[string]interface{} { - requests := make([]string, 0, len(gsm.requests)) - for _, request := range gsm.requests { - requests = append(requests, fmt.Sprintf("%d", request.id)) + requests := make([]string, 0, len(gsm.Requests)) + for _, request := range gsm.Requests { + requests = append(requests, fmt.Sprintf("%d", request.ID)) } - responses := make([]string, 0, len(gsm.responses)) - for _, response := range gsm.responses { - responses = append(responses, fmt.Sprintf("%d", response.requestID)) + responses := make([]string, 0, len(gsm.Responses)) + for _, response := range gsm.Responses { + responses = append(responses, fmt.Sprintf("%d", response.ID)) } return map[string]interface{}{ "requests": requests, @@ -343,12 +333,12 @@ func (gsm GraphSyncMessage) Loggable() map[string]interface{} { } func (gsm GraphSyncMessage) Clone() GraphSyncMessage { - requests := make(map[graphsync.RequestID]GraphSyncRequest, len(gsm.requests)) - for id, request := range gsm.requests { + requests := make(map[graphsync.RequestID]GraphSyncRequest, len(gsm.Requests)) + for id, request := range gsm.Requests { requests[id] = request } - responses := make(map[graphsync.RequestID]GraphSyncResponse, len(gsm.responses)) - for id, response := range gsm.responses { + responses := make(map[graphsync.RequestID]GraphSyncResponse, len(gsm.Responses)) + for id, response := range gsm.Responses { responses[id] = response } blocks := make(map[cid.Cid]blocks.Block, len(gsm.blocks)) @@ -358,18 +348,6 @@ func (gsm GraphSyncMessage) Clone() GraphSyncMessage { return GraphSyncMessage{requests, responses, blocks} } -// ID Returns the request ID for this Request -func (gsr GraphSyncRequest) ID() graphsync.RequestID { return gsr.id } - -// Root returns the CID to the root block of this request -func (gsr GraphSyncRequest) Root() cid.Cid { return gsr.root } - -// Selector returns the byte representation of the selector for this request -func (gsr GraphSyncRequest) Selector() ipld.Node { return gsr.selector } - -// Priority returns the priority of this request -func (gsr GraphSyncRequest) Priority() graphsync.Priority { return gsr.priority } - // Extension returns the content for an extension on a response, or errors // if extension is not present func (gsr GraphSyncRequest) Extension(name graphsync.ExtensionName) ([]byte, bool) { @@ -392,18 +370,6 @@ func (gsr GraphSyncRequest) ExtensionNames() []string { return extNames } -// IsCancel returns true if this particular request is being cancelled -func (gsr GraphSyncRequest) IsCancel() bool { return gsr.isCancel } - -// IsUpdate returns true if this particular request is being updated -func (gsr GraphSyncRequest) IsUpdate() bool { return gsr.isUpdate } - -// RequestID returns the request ID for this response -func (gsr GraphSyncResponse) RequestID() graphsync.RequestID { return gsr.requestID } - -// Status returns the status for a response -func (gsr GraphSyncResponse) Status() graphsync.ResponseStatusCode { return gsr.status } - // Extension returns the content for an extension on a response, or errors // if extension is not present func (gsr GraphSyncResponse) Extension(name graphsync.ExtensionName) ([]byte, bool) { From 7e61e3cc11f07942071c89648cb3f0eef81acfda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Mon, 10 Jan 2022 23:07:33 +0000 Subject: [PATCH 07/21] some more transition work in builder --- message/builder.go | 31 +++++----- message/message.go | 140 ++++++++++++++++++++------------------------- 2 files changed, 79 insertions(+), 92 deletions(-) diff --git a/message/builder.go b/message/builder.go index 15017198..9fa2a31c 100644 --- a/message/builder.go +++ b/message/builder.go @@ -5,6 +5,7 @@ import ( "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime" cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipfs/go-graphsync" "github.com/ipfs/go-graphsync/metadata" @@ -14,38 +15,39 @@ import ( // requests for a given peer and then generates the corresponding // GraphSync message when ready to send type Builder struct { - outgoingBlocks map[cid.Cid]blocks.Block + outgoingBlocks []GraphSyncBlock blkSize uint64 completedResponses map[graphsync.RequestID]graphsync.ResponseStatusCode outgoingResponses map[graphsync.RequestID]metadata.Metadata - extensions map[graphsync.RequestID][]graphsync.ExtensionData - requests map[graphsync.RequestID]GraphSyncRequest + extensions map[graphsync.RequestID][]NamedExtension + requests []GraphSyncRequest } // NewBuilder generates a new Builder. func NewBuilder() *Builder { return &Builder{ - requests: make(map[graphsync.RequestID]GraphSyncRequest), - outgoingBlocks: make(map[cid.Cid]blocks.Block), completedResponses: make(map[graphsync.RequestID]graphsync.ResponseStatusCode), outgoingResponses: make(map[graphsync.RequestID]metadata.Metadata), - extensions: make(map[graphsync.RequestID][]graphsync.ExtensionData), + extensions: make(map[graphsync.RequestID][]NamedExtension), } } // AddRequest registers a new request to be added to the message. func (b *Builder) AddRequest(request GraphSyncRequest) { - b.requests[request.ID()] = request + b.requests = append(b.requests, request) } // AddBlock adds the given block to the message. func (b *Builder) AddBlock(block blocks.Block) { b.blkSize += uint64(len(block.RawData())) - b.outgoingBlocks[block.Cid()] = block + b.outgoingBlocks = append(b.outgoingBlocks, GraphSyncBlock{ + Prefix: block.Cid().Prefix().Bytes(), + Data: block.RawData(), + }) } // AddExtensionData adds the given extension data to to the message -func (b *Builder) AddExtensionData(requestID graphsync.RequestID, extension graphsync.ExtensionData) { +func (b *Builder) AddExtensionData(requestID graphsync.RequestID, extension NamedExtension) { b.extensions[requestID] = append(b.extensions[requestID], extension) // make sure this extension goes out in next response even if no links are sent _, ok := b.outgoingResponses[requestID] @@ -109,19 +111,20 @@ func (b *Builder) ScrubResponses(requestIDs []graphsync.RequestID) uint64 { // Build assembles and encodes message data from the added requests, links, and blocks. func (b *Builder) Build() (GraphSyncMessage, error) { - responses := make(map[graphsync.RequestID]GraphSyncResponse, len(b.outgoingResponses)) + responses := make([]GraphSyncResponse, 0, len(b.outgoingResponses)) for requestID, linkMap := range b.outgoingResponses { mdRaw, err := metadata.EncodeMetadata(linkMap) if err != nil { return GraphSyncMessage{}, err } - b.extensions[requestID] = append(b.extensions[requestID], graphsync.ExtensionData{ - Name: graphsync.ExtensionMetadata, - Data: mdRaw, + b.extensions[requestID] = append(b.extensions[requestID], NamedExtension{ + Name: string(graphsync.ExtensionMetadata), + Data: basicnode.NewBytes(mdRaw), // TODO: likely wrong }) status, isComplete := b.completedResponses[requestID] - responses[requestID] = NewResponse(requestID, responseCode(status, isComplete), b.extensions[requestID]...) + responses = append(responses, NewResponse(requestID, responseCode(status, isComplete), b.extensions[requestID]...)) } + // TODO: sort responses? return GraphSyncMessage{ b.requests, responses, b.outgoingBlocks, }, nil diff --git a/message/message.go b/message/message.go index 289d36e5..6167a0c6 100644 --- a/message/message.go +++ b/message/message.go @@ -5,11 +5,12 @@ import ( "errors" "fmt" "io" + "sort" - blocks "github.com/ipfs/go-block-format" cid "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/node/basicnode" pool "github.com/libp2p/go-buffer-pool" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-msgio" @@ -129,6 +130,17 @@ func toExtensionsMap(extensions []NamedExtension) (m GraphSyncExtensions) { return m } +func fromProtoExtensions(protoExts map[string][]byte) GraphSyncExtensions { + var exts []NamedExtension + for name, data := range protoExts { + exts = append(exts, NamedExtension{name, basicnode.NewBytes(data)}) + } + // Iterating over the map above is non-deterministic, + // so sort by the unique names to ensure determinism. + sort.Slice(exts, func(i, j int) bool { return exts[i].Name < exts[j].Name }) + return toExtensionsMap(exts) +} + func newRequest(id graphsync.RequestID, root cid.Cid, selector ipld.Node, @@ -185,11 +197,10 @@ func newMessageFromProto(pbm *pb.Message) (GraphSyncMessage, error) { return GraphSyncMessage{}, err } } - exts := req.GetExtensions() - if exts == nil { - exts = make(map[string][]byte) - } - requests[i] = newRequest(graphsync.RequestID(req.Id), root, selector, graphsync.Priority(req.Priority), req.Cancel, req.Update, exts) + // TODO: we likely need to turn some "core" extensions to fields, + // as some of those got moved to proper fields in the new protocol. + // Same for responses above, as well as the "to proto" funcs. + requests[i] = newRequest(graphsync.RequestID(req.Id), root, selector, graphsync.Priority(req.Priority), req.Cancel, req.Update, fromProtoExtensions(req.GetExtensions())) } responses := make([]GraphSyncResponse, len(pbm.GetResponses())) @@ -197,11 +208,7 @@ func newMessageFromProto(pbm *pb.Message) (GraphSyncMessage, error) { if res == nil { return GraphSyncMessage{}, errors.New("response is nil") } - exts := res.GetExtensions() - if exts == nil { - exts = make(map[string][]byte) - } - responses[i] = newResponse(graphsync.RequestID(res.Id), graphsync.ResponseStatusCode(res.Status), exts) + responses[i] = newResponse(graphsync.RequestID(res.Id), graphsync.ResponseStatusCode(res.Status), fromProtoExtensions(res.GetExtensions())) } blks := make([]GraphSyncBlock, len(pbm.GetData())) @@ -226,8 +233,8 @@ func (gsm GraphSyncMessage) Empty() bool { func (gsm GraphSyncMessage) ResponseCodes() map[graphsync.RequestID]graphsync.ResponseStatusCode { codes := make(map[graphsync.RequestID]graphsync.ResponseStatusCode, len(gsm.Responses)) - for id, response := range gsm.Responses { - codes[id] = response.Status() + for _, response := range gsm.Responses { + codes[response.ID] = response.Status } return codes } @@ -261,38 +268,37 @@ func (gsm GraphSyncMessage) ToProto() (*pb.Message, error) { for _, request := range gsm.Requests { var selector []byte var err error - if request.selector != nil { - selector, err = ipldutil.EncodeNode(request.selector) + if request.Selector != nil { + selector, err = ipldutil.EncodeNode(request.Selector) if err != nil { return nil, err } } pbm.Requests = append(pbm.Requests, &pb.Message_Request{ - Id: int32(request.id), - Root: request.root.Bytes(), - Selector: selector, - Priority: int32(request.priority), - Cancel: request.isCancel, - Update: request.isUpdate, - Extensions: request.extensions, + Id: int32(request.ID), + Root: request.Root.Bytes(), + Selector: selector, + Priority: int32(request.Priority), + Cancel: request.Cancel, + Update: request.Update, + // Extensions: request.Extensions, TODO }) } pbm.Responses = make([]*pb.Message_Response, 0, len(gsm.Responses)) for _, response := range gsm.Responses { pbm.Responses = append(pbm.Responses, &pb.Message_Response{ - Id: int32(response.requestID), - Status: int32(response.status), - Extensions: response.extensions, + Id: int32(response.ID), + Status: int32(response.Status), + // Extensions: response.Extensions, TODO }) } - blocks := gsm.Blocks() - pbm.Data = make([]*pb.Message_Block, 0, len(blocks)) - for _, b := range blocks { + pbm.Data = make([]*pb.Message_Block, 0, len(gsm.Blocks)) + for _, b := range gsm.Blocks { pbm.Data = append(pbm.Data, &pb.Message_Block{ - Data: b.RawData(), - Prefix: b.Cid().Prefix().Bytes(), + Prefix: b.Prefix, + Data: b.Data, }) } return pbm, nil @@ -333,28 +339,16 @@ func (gsm GraphSyncMessage) Loggable() map[string]interface{} { } func (gsm GraphSyncMessage) Clone() GraphSyncMessage { - requests := make(map[graphsync.RequestID]GraphSyncRequest, len(gsm.Requests)) - for id, request := range gsm.Requests { - requests[id] = request - } - responses := make(map[graphsync.RequestID]GraphSyncResponse, len(gsm.Responses)) - for id, response := range gsm.Responses { - responses[id] = response - } - blocks := make(map[cid.Cid]blocks.Block, len(gsm.blocks)) - for cid, block := range gsm.blocks { - blocks[cid] = block - } + requests := append([]GraphSyncRequest{}, gsm.Requests...) + responses := append([]GraphSyncResponse{}, gsm.Responses...) + blocks := append([]GraphSyncBlock{}, gsm.Blocks...) return GraphSyncMessage{requests, responses, blocks} } // Extension returns the content for an extension on a response, or errors // if extension is not present -func (gsr GraphSyncRequest) Extension(name graphsync.ExtensionName) ([]byte, bool) { - if gsr.extensions == nil { - return nil, false - } - val, ok := gsr.extensions[string(name)] +func (gsr GraphSyncRequest) Extension(name graphsync.ExtensionName) (ipld.Node, bool) { + val, ok := gsr.Extensions.Values[string(name)] if !ok { return nil, false } @@ -363,20 +357,13 @@ func (gsr GraphSyncRequest) Extension(name graphsync.ExtensionName) ([]byte, boo // ExtensionNames returns the names of the extensions included in this request func (gsr GraphSyncRequest) ExtensionNames() []string { - var extNames []string - for ext := range gsr.extensions { - extNames = append(extNames, ext) - } - return extNames + return gsr.Extensions.Keys } // Extension returns the content for an extension on a response, or errors // if extension is not present -func (gsr GraphSyncResponse) Extension(name graphsync.ExtensionName) ([]byte, bool) { - if gsr.extensions == nil { - return nil, false - } - val, ok := gsr.extensions[string(name)] +func (gsr GraphSyncResponse) Extension(name graphsync.ExtensionName) (ipld.Node, bool) { + val, ok := gsr.Extensions.Values[string(name)] if !ok { return nil, false } @@ -385,18 +372,14 @@ func (gsr GraphSyncResponse) Extension(name graphsync.ExtensionName) ([]byte, bo // ExtensionNames returns the names of the extensions included in this request func (gsr GraphSyncResponse) ExtensionNames() []string { - var extNames []string - for ext := range gsr.extensions { - extNames = append(extNames, ext) - } - return extNames + return gsr.Extensions.Keys } // ReplaceExtensions merges the extensions given extensions into the request to create a new request, // but always uses new data -func (gsr GraphSyncRequest) ReplaceExtensions(extensions []graphsync.ExtensionData) GraphSyncRequest { - req, _ := gsr.MergeExtensions(extensions, func(name graphsync.ExtensionName, oldData []byte, newData []byte) ([]byte, error) { - return newData, nil +func (gsr GraphSyncRequest) ReplaceExtensions(extensions []NamedExtension) GraphSyncRequest { + req, _ := gsr.MergeExtensions(extensions, func(name graphsync.ExtensionName, oldNode, newNode ipld.Node) (ipld.Node, error) { + return newNode, nil }) return req } @@ -404,31 +387,32 @@ func (gsr GraphSyncRequest) ReplaceExtensions(extensions []graphsync.ExtensionDa // MergeExtensions merges the given list of extensions to produce a new request with the combination of the old request // plus the new extensions. When an old extension and a new extension are both present, mergeFunc is called to produce // the result -func (gsr GraphSyncRequest) MergeExtensions(extensions []graphsync.ExtensionData, mergeFunc func(name graphsync.ExtensionName, oldData []byte, newData []byte) ([]byte, error)) (GraphSyncRequest, error) { - if gsr.extensions == nil { - return newRequest(gsr.id, gsr.root, gsr.selector, gsr.priority, gsr.isCancel, gsr.isUpdate, toExtensionsMap(extensions)), nil +func (gsr GraphSyncRequest) MergeExtensions(extensions []NamedExtension, mergeFunc func(name graphsync.ExtensionName, oldNode, newNode ipld.Node) (ipld.Node, error)) (GraphSyncRequest, error) { + if len(gsr.Extensions.Keys) == 0 { + return newRequest(gsr.ID, gsr.Root, gsr.Selector, gsr.Priority, gsr.Cancel, gsr.Update, toExtensionsMap(extensions)), nil } - newExtensionMap := toExtensionsMap(extensions) - combinedExtensions := make(map[string][]byte) - for name, newData := range newExtensionMap { - oldData, ok := gsr.extensions[name] + combinedExtensions := make(map[string]ipld.Node) + for _, newExt := range extensions { + oldNode, ok := gsr.Extensions.Values[newExt.Name] if !ok { - combinedExtensions[name] = newData + combinedExtensions[newExt.Name] = newExt.Data continue } - resultData, err := mergeFunc(graphsync.ExtensionName(name), oldData, newData) + resultNode, err := mergeFunc(graphsync.ExtensionName(newExt.Name), oldNode, newExt.Data) if err != nil { return GraphSyncRequest{}, err } - combinedExtensions[name] = resultData + combinedExtensions[newExt.Name] = resultNode } - for name, oldData := range gsr.extensions { + for name, oldNode := range gsr.Extensions.Values { _, ok := combinedExtensions[name] if ok { continue } - combinedExtensions[name] = oldData + combinedExtensions[name] = oldNode } - return newRequest(gsr.id, gsr.root, gsr.selector, gsr.priority, gsr.isCancel, gsr.isUpdate, combinedExtensions), nil + extNames := make([]string, len(combinedExtensions)) + sort.Strings(extNames) // for reproducibility + return newRequest(gsr.ID, gsr.Root, gsr.Selector, gsr.Priority, gsr.Cancel, gsr.Update, GraphSyncExtensions{extNames, combinedExtensions}), nil } From 37d113740a95fb3bb23ef9660330152100ede272 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Tue, 11 Jan 2022 11:07:17 +0000 Subject: [PATCH 08/21] current types now bind to the schema --- go.mod | 2 ++ message/builder.go | 4 +++- message/ipldcbor/schema.go | 5 ++++- 3 files changed, 9 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 35bf72ab..5bce5b54 100644 --- a/go.mod +++ b/go.mod @@ -44,3 +44,5 @@ require ( golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 google.golang.org/protobuf v1.27.1 ) + +replace github.com/ipld/go-ipld-prime => ../../src/ipld diff --git a/message/builder.go b/message/builder.go index 9fa2a31c..6a4b6264 100644 --- a/message/builder.go +++ b/message/builder.go @@ -2,7 +2,6 @@ package message import ( blocks "github.com/ipfs/go-block-format" - "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" @@ -86,6 +85,7 @@ func (b *Builder) Empty() bool { // ScrubResponse removes a response from a message and any blocks only referenced by that response func (b *Builder) ScrubResponses(requestIDs []graphsync.RequestID) uint64 { + /* TODO for _, requestID := range requestIDs { delete(b.completedResponses, requestID) delete(b.extensions, requestID) @@ -107,6 +107,8 @@ func (b *Builder) ScrubResponses(requestIDs []graphsync.RequestID) uint64 { b.blkSize = newBlkSize b.outgoingBlocks = savedBlocks return oldSize - newBlkSize + */ + return 0 } // Build assembles and encodes message data from the added requests, links, and blocks. diff --git a/message/ipldcbor/schema.go b/message/ipldcbor/schema.go index 2dd0499f..41b4e0f0 100644 --- a/message/ipldcbor/schema.go +++ b/message/ipldcbor/schema.go @@ -6,7 +6,10 @@ import ( _ "embed" "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/node/bindnode" "github.com/ipld/go-ipld-prime/schema" + + "github.com/ipfs/go-graphsync/message" ) //go:embed schema.ipldsch @@ -25,5 +28,5 @@ func init() { } schemaTypeSystem = ts - // Prototype.Message = bindnode.Prototype((*GraphSyncMessage)(nil), ts.TypeByName("GraphSyncMessage")) + Prototype.Message = bindnode.Prototype((*message.GraphSyncMessage)(nil), ts.TypeByName("GraphSyncMessage")) } From e41494b6e7b9d30dde9d48ef2128417c057efc3e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Tue, 11 Jan 2022 11:32:47 +0000 Subject: [PATCH 09/21] start adapting tests; custom type for extension names --- message/builder.go | 2 +- message/builder_test.go | 47 ++++++++++---------- message/message.go | 14 +++--- message/message_test.go | 96 ++++++++++++++++++++--------------------- 4 files changed, 80 insertions(+), 79 deletions(-) diff --git a/message/builder.go b/message/builder.go index 6a4b6264..50564376 100644 --- a/message/builder.go +++ b/message/builder.go @@ -120,7 +120,7 @@ func (b *Builder) Build() (GraphSyncMessage, error) { return GraphSyncMessage{}, err } b.extensions[requestID] = append(b.extensions[requestID], NamedExtension{ - Name: string(graphsync.ExtensionMetadata), + Name: graphsync.ExtensionMetadata, Data: basicnode.NewBytes(mdRaw), // TODO: likely wrong }) status, isComplete := b.completedResponses[requestID] diff --git a/message/builder_test.go b/message/builder_test.go index b9722c84..15b38f7b 100644 --- a/message/builder_test.go +++ b/message/builder_test.go @@ -7,6 +7,7 @@ import ( "github.com/ipld/go-ipld-prime" cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/stretchr/testify/require" "github.com/ipfs/go-graphsync" @@ -20,15 +21,15 @@ func TestMessageBuilding(t *testing.T) { for _, block := range blocks { links = append(links, cidlink.Link{Cid: block.Cid()}) } - extensionData1 := testutil.RandomBytes(100) + extensionData1 := basicnode.NewBytes(testutil.RandomBytes(100)) extensionName1 := graphsync.ExtensionName("AppleSauce/McGee") - extension1 := graphsync.ExtensionData{ + extension1 := NamedExtension{ Name: extensionName1, Data: extensionData1, } - extensionData2 := testutil.RandomBytes(100) + extensionData2 := basicnode.NewBytes(testutil.RandomBytes(100)) extensionName2 := graphsync.ExtensionName("HappyLand/Happenstance") - extension2 := graphsync.ExtensionData{ + extension2 := NamedExtension{ Name: extensionName2, Data: extensionData2, } @@ -75,12 +76,12 @@ func TestMessageBuilding(t *testing.T) { }, checkMsg: func(t *testing.T, message GraphSyncMessage) { - responses := message.Responses() - sentBlocks := message.Blocks() + responses := message.Responses + sentBlocks := message.Blocks require.Len(t, responses, 4, "did not assemble correct number of responses") response1 := findResponseForRequestID(t, responses, requestID1) - require.Equal(t, graphsync.RequestCompletedPartial, response1.Status(), "did not generate completed partial response") + require.Equal(t, graphsync.RequestCompletedPartial, response1.Status, "did not generate completed partial response") assertMetadata(t, response1, metadata.Metadata{ metadata.Item{Link: links[0].(cidlink.Link).Cid, BlockPresent: true}, metadata.Item{Link: links[1].(cidlink.Link).Cid, BlockPresent: false}, @@ -89,7 +90,7 @@ func TestMessageBuilding(t *testing.T) { assertExtension(t, response1, extension1) response2 := findResponseForRequestID(t, responses, requestID2) - require.Equal(t, graphsync.RequestCompletedFull, response2.Status(), "did not generate completed full response") + require.Equal(t, graphsync.RequestCompletedFull, response2.Status, "did not generate completed full response") assertMetadata(t, response2, metadata.Metadata{ metadata.Item{Link: links[1].(cidlink.Link).Cid, BlockPresent: true}, metadata.Item{Link: links[2].(cidlink.Link).Cid, BlockPresent: true}, @@ -97,7 +98,7 @@ func TestMessageBuilding(t *testing.T) { }) response3 := findResponseForRequestID(t, responses, requestID3) - require.Equal(t, graphsync.PartialResponse, response3.Status(), "did not generate partial response") + require.Equal(t, graphsync.PartialResponse, response3.Status, "did not generate partial response") assertMetadata(t, response3, metadata.Metadata{ metadata.Item{Link: links[0].(cidlink.Link).Cid, BlockPresent: true}, metadata.Item{Link: links[1].(cidlink.Link).Cid, BlockPresent: true}, @@ -105,7 +106,7 @@ func TestMessageBuilding(t *testing.T) { assertExtension(t, response3, extension2) response4 := findResponseForRequestID(t, responses, requestID4) - require.Equal(t, graphsync.RequestCompletedFull, response4.Status(), "did not generate completed full response") + require.Equal(t, graphsync.RequestCompletedFull, response4.Status, "did not generate completed full response") require.Equal(t, len(blocks), len(sentBlocks), "did not send all blocks") @@ -121,15 +122,15 @@ func TestMessageBuilding(t *testing.T) { }, expectedSize: 0, checkMsg: func(t *testing.T, message GraphSyncMessage) { - responses := message.Responses() + responses := message.Responses response1 := findResponseForRequestID(t, responses, requestID1) - require.Equal(t, graphsync.PartialResponse, response1.Status(), "did not generate partial response") + require.Equal(t, graphsync.PartialResponse, response1.Status, "did not generate partial response") assertMetadata(t, response1, nil) assertExtension(t, response1, extension1) response2 := findResponseForRequestID(t, responses, requestID2) - require.Equal(t, graphsync.PartialResponse, response2.Status(), "did not generate partial response") + require.Equal(t, graphsync.PartialResponse, response2.Status, "did not generate partial response") assertMetadata(t, response2, nil) assertExtension(t, response2, extension2) }, @@ -162,12 +163,12 @@ func TestMessageBuilding(t *testing.T) { expectedSize: 200, checkMsg: func(t *testing.T, message GraphSyncMessage) { - responses := message.Responses() - sentBlocks := message.Blocks() + responses := message.Responses + sentBlocks := message.Blocks require.Len(t, responses, 3, "did not assemble correct number of responses") response2 := findResponseForRequestID(t, responses, requestID2) - require.Equal(t, graphsync.RequestCompletedFull, response2.Status(), "did not generate completed full response") + require.Equal(t, graphsync.RequestCompletedFull, response2.Status, "did not generate completed full response") assertMetadata(t, response2, metadata.Metadata{ metadata.Item{Link: links[1].(cidlink.Link).Cid, BlockPresent: true}, metadata.Item{Link: links[2].(cidlink.Link).Cid, BlockPresent: true}, @@ -175,14 +176,14 @@ func TestMessageBuilding(t *testing.T) { }) response3 := findResponseForRequestID(t, responses, requestID3) - require.Equal(t, graphsync.PartialResponse, response3.Status(), "did not generate partial response") + require.Equal(t, graphsync.PartialResponse, response3.Status, "did not generate partial response") assertMetadata(t, response3, metadata.Metadata{ metadata.Item{Link: links[1].(cidlink.Link).Cid, BlockPresent: true}, }) assertExtension(t, response3, extension2) response4 := findResponseForRequestID(t, responses, requestID4) - require.Equal(t, graphsync.RequestCompletedFull, response4.Status(), "did not generate completed full response") + require.Equal(t, graphsync.RequestCompletedFull, response4.Status, "did not generate completed full response") require.Equal(t, len(blocks)-1, len(sentBlocks), "did not send all blocks") @@ -220,12 +221,12 @@ func TestMessageBuilding(t *testing.T) { expectedSize: 100, checkMsg: func(t *testing.T, message GraphSyncMessage) { - responses := message.Responses() - sentBlocks := message.Blocks() + responses := message.Responses + sentBlocks := message.Blocks require.Len(t, responses, 1, "did not assemble correct number of responses") response3 := findResponseForRequestID(t, responses, requestID3) - require.Equal(t, graphsync.PartialResponse, response3.Status(), "did not generate partial response") + require.Equal(t, graphsync.PartialResponse, response3.Status, "did not generate partial response") assertMetadata(t, response3, metadata.Metadata{ metadata.Item{Link: links[1].(cidlink.Link).Cid, BlockPresent: true}, }) @@ -251,7 +252,7 @@ func TestMessageBuilding(t *testing.T) { func findResponseForRequestID(t *testing.T, responses []GraphSyncResponse, requestID graphsync.RequestID) GraphSyncResponse { for _, response := range responses { - if response.RequestID() == requestID { + if response.ID == requestID { return response } } @@ -259,7 +260,7 @@ func findResponseForRequestID(t *testing.T, responses []GraphSyncResponse, reque return GraphSyncResponse{} } -func assertExtension(t *testing.T, response GraphSyncResponse, extension graphsync.ExtensionData) { +func assertExtension(t *testing.T, response GraphSyncResponse, extension NamedExtension) { returnedExtensionData, found := response.Extension(extension.Name) require.True(t, found) require.Equal(t, extension.Data, returnedExtensionData, "did not encode extension") diff --git a/message/message.go b/message/message.go index 6167a0c6..2f834fa9 100644 --- a/message/message.go +++ b/message/message.go @@ -114,7 +114,7 @@ func UpdateRequest(id graphsync.RequestID, extensions ...NamedExtension) GraphSy // NamedExtension exists just for the purpose of the constructors. type NamedExtension struct { - Name string + Name graphsync.ExtensionName Data ipld.Node } @@ -123,8 +123,8 @@ func toExtensionsMap(extensions []NamedExtension) (m GraphSyncExtensions) { m.Keys = make([]string, len(extensions)) m.Values = make(map[string]ipld.Node, len(extensions)) for i, ext := range extensions { - m.Keys[i] = ext.Name - m.Values[ext.Name] = ext.Data + m.Keys[i] = string(ext.Name) + m.Values[string(ext.Name)] = ext.Data } } return m @@ -133,7 +133,7 @@ func toExtensionsMap(extensions []NamedExtension) (m GraphSyncExtensions) { func fromProtoExtensions(protoExts map[string][]byte) GraphSyncExtensions { var exts []NamedExtension for name, data := range protoExts { - exts = append(exts, NamedExtension{name, basicnode.NewBytes(data)}) + exts = append(exts, NamedExtension{graphsync.ExtensionName(name), basicnode.NewBytes(data)}) } // Iterating over the map above is non-deterministic, // so sort by the unique names to ensure determinism. @@ -393,16 +393,16 @@ func (gsr GraphSyncRequest) MergeExtensions(extensions []NamedExtension, mergeFu } combinedExtensions := make(map[string]ipld.Node) for _, newExt := range extensions { - oldNode, ok := gsr.Extensions.Values[newExt.Name] + oldNode, ok := gsr.Extensions.Values[string(newExt.Name)] if !ok { - combinedExtensions[newExt.Name] = newExt.Data + combinedExtensions[string(newExt.Name)] = newExt.Data continue } resultNode, err := mergeFunc(graphsync.ExtensionName(newExt.Name), oldNode, newExt.Data) if err != nil { return GraphSyncRequest{}, err } - combinedExtensions[newExt.Name] = resultNode + combinedExtensions[string(newExt.Name)] = resultNode } for name, oldNode := range gsr.Extensions.Values { diff --git a/message/message_test.go b/message/message_test.go index 135342d3..e341c142 100644 --- a/message/message_test.go +++ b/message/message_test.go @@ -19,7 +19,7 @@ import ( func TestAppendingRequests(t *testing.T) { extensionName := graphsync.ExtensionName("graphsync/awesome") - extension := graphsync.ExtensionData{ + extension := NamedExtension{ Name: extensionName, Data: testutil.RandomBytes(100), } @@ -33,15 +33,15 @@ func TestAppendingRequests(t *testing.T) { builder.AddRequest(NewRequest(id, root, selector, priority, extension)) gsm, err := builder.Build() require.NoError(t, err) - requests := gsm.Requests() + requests := gsm.Requests require.Len(t, requests, 1, "did not add request to message") request := requests[0] extensionData, found := request.Extension(extensionName) - require.Equal(t, id, request.ID()) - require.False(t, request.IsCancel()) - require.Equal(t, priority, request.Priority()) - require.Equal(t, root.String(), request.Root().String()) - require.Equal(t, selector, request.Selector()) + require.Equal(t, id, request.ID) + require.False(t, request.Cancel) + require.Equal(t, priority, request.Priority) + require.Equal(t, root.String(), request.Root.String()) + require.Equal(t, selector, request.Selector) require.True(t, found) require.Equal(t, extension.Data, extensionData) @@ -61,24 +61,24 @@ func TestAppendingRequests(t *testing.T) { deserialized, err := newMessageFromProto(pbMessage) require.NoError(t, err, "deserializing protobuf message errored") - deserializedRequests := deserialized.Requests() + deserializedRequests := deserialized.Requests require.Len(t, deserializedRequests, 1, "did not add request to deserialized message") deserializedRequest := deserializedRequests[0] extensionData, found = deserializedRequest.Extension(extensionName) require.Equal(t, id, deserializedRequest.ID()) - require.False(t, deserializedRequest.IsCancel()) - require.False(t, deserializedRequest.IsUpdate()) - require.Equal(t, priority, deserializedRequest.Priority()) - require.Equal(t, root.String(), deserializedRequest.Root().String()) - require.Equal(t, selector, deserializedRequest.Selector()) + require.False(t, deserializedRequest.Cancel) + require.False(t, deserializedRequest.Update) + require.Equal(t, priority, deserializedRequest.Priority) + require.Equal(t, root.String(), deserializedRequest.Root.String()) + require.Equal(t, selector, deserializedRequest.Selector) require.True(t, found) require.Equal(t, extension.Data, extensionData) } func TestAppendingResponses(t *testing.T) { extensionName := graphsync.ExtensionName("graphsync/awesome") - extension := graphsync.ExtensionData{ + extension := NamedExtension{ Name: extensionName, Data: testutil.RandomBytes(100), } @@ -90,12 +90,12 @@ func TestAppendingResponses(t *testing.T) { builder.AddExtensionData(requestID, extension) gsm, err := builder.Build() require.NoError(t, err) - responses := gsm.Responses() + responses := gsm.Responses require.Len(t, responses, 1, "did not add response to message") response := responses[0] extensionData, found := response.Extension(extensionName) - require.Equal(t, requestID, response.RequestID()) - require.Equal(t, status, response.Status()) + require.Equal(t, requestID, response.ID) + require.Equal(t, status, response.Status) require.True(t, found) require.Equal(t, extension.Data, extensionData) @@ -108,12 +108,12 @@ func TestAppendingResponses(t *testing.T) { deserialized, err := newMessageFromProto(pbMessage) require.NoError(t, err, "deserializing protobuf message errored") - deserializedResponses := deserialized.Responses() + deserializedResponses := deserialized.Responses require.Len(t, deserializedResponses, 1, "did not add response to deserialized message") deserializedResponse := deserializedResponses[0] extensionData, found = deserializedResponse.Extension(extensionName) - require.Equal(t, response.RequestID(), deserializedResponse.RequestID()) - require.Equal(t, response.Status(), deserializedResponse.Status()) + require.Equal(t, response.ID, deserializedResponse.ID) + require.Equal(t, response.Status, deserializedResponse.Status) require.True(t, found) require.Equal(t, extension.Data, extensionData) } @@ -164,29 +164,29 @@ func TestRequestCancel(t *testing.T) { gsm, err := builder.Build() require.NoError(t, err) - requests := gsm.Requests() + requests := gsm.Requests require.Len(t, requests, 1, "did not add cancel request") request := requests[0] require.Equal(t, id, request.ID()) - require.True(t, request.IsCancel()) + require.True(t, request.Cancel) buf := new(bytes.Buffer) err = gsm.ToNet(buf) require.NoError(t, err, "did not serialize protobuf message") deserialized, err := FromNet(buf) require.NoError(t, err, "did not deserialize protobuf message") - deserializedRequests := deserialized.Requests() + deserializedRequests := deserialized.Requests require.Len(t, deserializedRequests, 1, "did not add request to deserialized message") deserializedRequest := deserializedRequests[0] require.Equal(t, request.ID(), deserializedRequest.ID()) - require.Equal(t, request.IsCancel(), deserializedRequest.IsCancel()) + require.Equal(t, request.Cancel, deserializedRequest.Cancel) } func TestRequestUpdate(t *testing.T) { id := graphsync.RequestID(rand.Int31()) extensionName := graphsync.ExtensionName("graphsync/awesome") - extension := graphsync.ExtensionData{ + extension := NamedExtension{ Name: extensionName, Data: testutil.RandomBytes(100), } @@ -196,12 +196,12 @@ func TestRequestUpdate(t *testing.T) { gsm, err := builder.Build() require.NoError(t, err) - requests := gsm.Requests() + requests := gsm.Requests require.Len(t, requests, 1, "did not add cancel request") request := requests[0] require.Equal(t, id, request.ID()) - require.True(t, request.IsUpdate()) - require.False(t, request.IsCancel()) + require.True(t, request.Update) + require.False(t, request.Cancel) extensionData, found := request.Extension(extensionName) require.True(t, found) require.Equal(t, extension.Data, extensionData) @@ -212,13 +212,13 @@ func TestRequestUpdate(t *testing.T) { deserialized, err := FromNet(buf) require.NoError(t, err, "did not deserialize protobuf message") - deserializedRequests := deserialized.Requests() + deserializedRequests := deserialized.Requests require.Len(t, deserializedRequests, 1, "did not add request to deserialized message") deserializedRequest := deserializedRequests[0] extensionData, found = deserializedRequest.Extension(extensionName) require.Equal(t, request.ID(), deserializedRequest.ID()) - require.Equal(t, request.IsCancel(), deserializedRequest.IsCancel()) - require.Equal(t, request.IsUpdate(), deserializedRequest.IsUpdate()) + require.Equal(t, request.Cancel, deserializedRequest.Cancel) + require.Equal(t, request.Update, deserializedRequest.Update) require.Equal(t, request.Priority(), deserializedRequest.Priority()) require.Equal(t, request.Root().String(), deserializedRequest.Root().String()) require.Equal(t, request.Selector(), deserializedRequest.Selector()) @@ -231,7 +231,7 @@ func TestToNetFromNetEquivalency(t *testing.T) { ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) selector := ssb.Matcher().Node() extensionName := graphsync.ExtensionName("graphsync/awesome") - extension := graphsync.ExtensionData{ + extension := NamedExtension{ Name: extensionName, Data: testutil.RandomBytes(100), } @@ -256,30 +256,30 @@ func TestToNetFromNetEquivalency(t *testing.T) { deserialized, err := FromNet(buf) require.NoError(t, err, "did not deserialize protobuf message") - requests := gsm.Requests() + requests := gsm.Requests require.Len(t, requests, 1, "did not add request to message") request := requests[0] - deserializedRequests := deserialized.Requests() + deserializedRequests := deserialized.Requests require.Len(t, deserializedRequests, 1, "did not add request to deserialized message") deserializedRequest := deserializedRequests[0] extensionData, found := deserializedRequest.Extension(extensionName) require.Equal(t, request.ID(), deserializedRequest.ID()) - require.False(t, deserializedRequest.IsCancel()) - require.False(t, deserializedRequest.IsUpdate()) + require.False(t, deserializedRequest.Cancel) + require.False(t, deserializedRequest.Update) require.Equal(t, request.Priority(), deserializedRequest.Priority()) require.Equal(t, request.Root().String(), deserializedRequest.Root().String()) require.Equal(t, request.Selector(), deserializedRequest.Selector()) require.True(t, found) require.Equal(t, extension.Data, extensionData) - responses := gsm.Responses() + responses := gsm.Responses require.Len(t, responses, 1, "did not add response to message") response := responses[0] - deserializedResponses := deserialized.Responses() + deserializedResponses := deserialized.Responses require.Len(t, deserializedResponses, 1, "did not add response to message") deserializedResponse := deserializedResponses[0] extensionData, found = deserializedResponse.Extension(extensionName) - require.Equal(t, response.RequestID(), deserializedResponse.RequestID()) + require.Equal(t, response.ID, deserializedResponse.ID) require.Equal(t, response.Status(), deserializedResponse.Status()) require.True(t, found) require.Equal(t, extension.Data, extensionData) @@ -299,7 +299,7 @@ func TestMergeExtensions(t *testing.T) { extensionName1 := graphsync.ExtensionName("graphsync/1") extensionName2 := graphsync.ExtensionName("graphsync/2") extensionName3 := graphsync.ExtensionName("graphsync/3") - initialExtensions := []graphsync.ExtensionData{ + initialExtensions := []NamedExtension{ { Name: extensionName1, Data: []byte("applesauce"), @@ -348,10 +348,10 @@ func TestMergeExtensions(t *testing.T) { t.Run("when merging two requests", func(t *testing.T) { resultRequest, err := defaultRequest.MergeExtensions(replacementExtensions, defaultMergeFunc) require.NoError(t, err) - require.Equal(t, defaultRequest.ID(), resultRequest.ID()) - require.Equal(t, defaultRequest.Priority(), resultRequest.Priority()) - require.Equal(t, defaultRequest.Root().String(), resultRequest.Root().String()) - require.Equal(t, defaultRequest.Selector(), resultRequest.Selector()) + require.Equal(t, defaultRequest.ID, resultRequest.ID) + require.Equal(t, defaultRequest.Priority, resultRequest.Priority) + require.Equal(t, defaultRequest.Root.String(), resultRequest.Root.String()) + require.Equal(t, defaultRequest.Selector, resultRequest.Selector) extData1, has := resultRequest.Extension(extensionName1) require.True(t, has) require.Equal(t, []byte("applesauce"), extData1) @@ -371,10 +371,10 @@ func TestMergeExtensions(t *testing.T) { }) t.Run("when merging with replace", func(t *testing.T) { resultRequest := defaultRequest.ReplaceExtensions(replacementExtensions) - require.Equal(t, defaultRequest.ID(), resultRequest.ID()) - require.Equal(t, defaultRequest.Priority(), resultRequest.Priority()) - require.Equal(t, defaultRequest.Root().String(), resultRequest.Root().String()) - require.Equal(t, defaultRequest.Selector(), resultRequest.Selector()) + require.Equal(t, defaultRequest.ID, resultRequest.ID) + require.Equal(t, defaultRequest.Priority, resultRequest.Priority) + require.Equal(t, defaultRequest.Root.String(), resultRequest.Root.String()) + require.Equal(t, defaultRequest.Selector, resultRequest.Selector) extData1, has := resultRequest.Extension(extensionName1) require.True(t, has) require.Equal(t, []byte("applesauce"), extData1) From ff6c3d951baa3c27e274c97980918abe2f48afee Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Tue, 11 Jan 2022 11:41:55 +0000 Subject: [PATCH 10/21] go back to blocks.Block for the message funcs --- message/builder.go | 22 +++++++++++++--------- 1 file changed, 13 insertions(+), 9 deletions(-) diff --git a/message/builder.go b/message/builder.go index 50564376..82fb3625 100644 --- a/message/builder.go +++ b/message/builder.go @@ -2,6 +2,7 @@ package message import ( blocks "github.com/ipfs/go-block-format" + cid "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime" cidlink "github.com/ipld/go-ipld-prime/linking/cid" "github.com/ipld/go-ipld-prime/node/basicnode" @@ -14,7 +15,7 @@ import ( // requests for a given peer and then generates the corresponding // GraphSync message when ready to send type Builder struct { - outgoingBlocks []GraphSyncBlock + outgoingBlocks map[cid.Cid]blocks.Block blkSize uint64 completedResponses map[graphsync.RequestID]graphsync.ResponseStatusCode outgoingResponses map[graphsync.RequestID]metadata.Metadata @@ -25,6 +26,7 @@ type Builder struct { // NewBuilder generates a new Builder. func NewBuilder() *Builder { return &Builder{ + outgoingBlocks: make(map[cid.Cid]blocks.Block), completedResponses: make(map[graphsync.RequestID]graphsync.ResponseStatusCode), outgoingResponses: make(map[graphsync.RequestID]metadata.Metadata), extensions: make(map[graphsync.RequestID][]NamedExtension), @@ -39,10 +41,7 @@ func (b *Builder) AddRequest(request GraphSyncRequest) { // AddBlock adds the given block to the message. func (b *Builder) AddBlock(block blocks.Block) { b.blkSize += uint64(len(block.RawData())) - b.outgoingBlocks = append(b.outgoingBlocks, GraphSyncBlock{ - Prefix: block.Cid().Prefix().Bytes(), - Data: block.RawData(), - }) + b.outgoingBlocks[block.Cid()] = block } // AddExtensionData adds the given extension data to to the message @@ -85,7 +84,6 @@ func (b *Builder) Empty() bool { // ScrubResponse removes a response from a message and any blocks only referenced by that response func (b *Builder) ScrubResponses(requestIDs []graphsync.RequestID) uint64 { - /* TODO for _, requestID := range requestIDs { delete(b.completedResponses, requestID) delete(b.extensions, requestID) @@ -107,8 +105,6 @@ func (b *Builder) ScrubResponses(requestIDs []graphsync.RequestID) uint64 { b.blkSize = newBlkSize b.outgoingBlocks = savedBlocks return oldSize - newBlkSize - */ - return 0 } // Build assembles and encodes message data from the added requests, links, and blocks. @@ -127,8 +123,16 @@ func (b *Builder) Build() (GraphSyncMessage, error) { responses = append(responses, NewResponse(requestID, responseCode(status, isComplete), b.extensions[requestID]...)) } // TODO: sort responses? + blocks := make([]GraphSyncBlock, len(b.outgoingBlocks)) + for cid, block := range b.outgoingBlocks { + blocks = append(blocks, GraphSyncBlock{ + Prefix: cid.Prefix().Bytes(), + Data: block.RawData(), + }) + } + // TODO: sort blocks? return GraphSyncMessage{ - b.requests, responses, b.outgoingBlocks, + b.requests, responses, blocks, }, nil } From 3d9529ae93e32658a3b6b607c278fb6a337fe15c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Tue, 11 Jan 2022 12:03:15 +0000 Subject: [PATCH 11/21] message tests build for now --- message/builder.go | 7 ++-- message/builder_test.go | 32 +++++++++++++++--- message/message.go | 34 +++++++++++++++++++ message/message_test.go | 73 ++++++++++++++++++++++------------------- 4 files changed, 103 insertions(+), 43 deletions(-) diff --git a/message/builder.go b/message/builder.go index 82fb3625..7faa10b9 100644 --- a/message/builder.go +++ b/message/builder.go @@ -124,11 +124,8 @@ func (b *Builder) Build() (GraphSyncMessage, error) { } // TODO: sort responses? blocks := make([]GraphSyncBlock, len(b.outgoingBlocks)) - for cid, block := range b.outgoingBlocks { - blocks = append(blocks, GraphSyncBlock{ - Prefix: cid.Prefix().Bytes(), - Data: block.RawData(), - }) + for _, block := range b.outgoingBlocks { + blocks = append(blocks, FromBlockFormat(block)) } // TODO: sort blocks? return GraphSyncMessage{ diff --git a/message/builder_test.go b/message/builder_test.go index 15b38f7b..4dac2132 100644 --- a/message/builder_test.go +++ b/message/builder_test.go @@ -1,6 +1,7 @@ package message import ( + "bytes" "io" "math/rand" "testing" @@ -15,6 +16,27 @@ import ( "github.com/ipfs/go-graphsync/testutil" ) +// Like the funcs in testutil above, but using blocks at the protocol level. +// We can't put them there right away, due to import cycles. +// We need to refactor these tests to be external, i.e. "package message_test". + +func ContainsGraphSyncBlock(blks []GraphSyncBlock, block GraphSyncBlock) bool { + for _, blk := range blks { + if bytes.Equal(blk.Prefix, block.Prefix) && bytes.Equal(blk.Data, block.Data) { + return true + } + } + return false +} +func AssertContainsGraphSyncBlock(t testing.TB, blks []GraphSyncBlock, block GraphSyncBlock) { + t.Helper() + require.True(t, ContainsGraphSyncBlock(blks, block), "given block should be in list") +} +func RefuteContainsGraphSyncBlock(t testing.TB, blks []GraphSyncBlock, block GraphSyncBlock) { + t.Helper() + require.False(t, ContainsGraphSyncBlock(blks, block), "given block should not be in list") +} + func TestMessageBuilding(t *testing.T) { blocks := testutil.GenerateBlocksOfSize(3, 100) links := make([]ipld.Link, 0, len(blocks)) @@ -77,7 +99,7 @@ func TestMessageBuilding(t *testing.T) { checkMsg: func(t *testing.T, message GraphSyncMessage) { responses := message.Responses - sentBlocks := message.Blocks + sentBlocks := BlockFormatSlice(message.Blocks) require.Len(t, responses, 4, "did not assemble correct number of responses") response1 := findResponseForRequestID(t, responses, requestID1) @@ -164,7 +186,7 @@ func TestMessageBuilding(t *testing.T) { checkMsg: func(t *testing.T, message GraphSyncMessage) { responses := message.Responses - sentBlocks := message.Blocks + sentBlocks := BlockFormatSlice(message.Blocks) require.Len(t, responses, 3, "did not assemble correct number of responses") response2 := findResponseForRequestID(t, responses, requestID2) @@ -222,7 +244,7 @@ func TestMessageBuilding(t *testing.T) { checkMsg: func(t *testing.T, message GraphSyncMessage) { responses := message.Responses - sentBlocks := message.Blocks + sentBlocks := BlockFormatSlice(message.Blocks) require.Len(t, responses, 1, "did not assemble correct number of responses") response3 := findResponseForRequestID(t, responses, requestID3) @@ -267,8 +289,10 @@ func assertExtension(t *testing.T, response GraphSyncResponse, extension NamedEx } func assertMetadata(t *testing.T, response GraphSyncResponse, expectedMetadata metadata.Metadata) { - responseMetadataRaw, found := response.Extension(graphsync.ExtensionMetadata) + responseMetadataNode, found := response.Extension(graphsync.ExtensionMetadata) require.True(t, found, "Metadata should be included in response") + responseMetadataRaw, err := responseMetadataNode.AsBytes() + require.NoError(t, err) responseMetadata, err := metadata.DecodeMetadata(responseMetadataRaw) require.NoError(t, err) require.Equal(t, expectedMetadata, responseMetadata, "incorrect metadata included in response") diff --git a/message/message.go b/message/message.go index 2f834fa9..24f3cd4d 100644 --- a/message/message.go +++ b/message/message.go @@ -7,6 +7,7 @@ import ( "io" "sort" + blocks "github.com/ipfs/go-block-format" cid "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/datamodel" @@ -86,6 +87,39 @@ type GraphSyncBlock struct { Data []byte } +func FromBlockFormat(block blocks.Block) GraphSyncBlock { + return GraphSyncBlock{ + Prefix: block.Cid().Prefix().Bytes(), + Data: block.RawData(), + } +} + +func (b GraphSyncBlock) BlockFormat() *blocks.BasicBlock { + pref, err := cid.PrefixFromBytes(b.Prefix) + if err != nil { + panic(err) // should never happen + } + + c, err := pref.Sum(b.Data) + if err != nil { + panic(err) // should never happen + } + + block, err := blocks.NewBlockWithCid(b.Data, c) + if err != nil { + panic(err) // should never happen + } + return block +} + +func BlockFormatSlice(bs []GraphSyncBlock) []blocks.Block { + blks := make([]blocks.Block, len(bs)) + for i, b := range bs { + blks[i] = b.BlockFormat() + } + return blks +} + type GraphSyncMessage struct { Requests []GraphSyncRequest Responses []GraphSyncResponse diff --git a/message/message_test.go b/message/message_test.go index e341c142..77be3d44 100644 --- a/message/message_test.go +++ b/message/message_test.go @@ -8,6 +8,7 @@ import ( blocks "github.com/ipfs/go-block-format" cid "github.com/ipfs/go-cid" + "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/traversal/selector/builder" "github.com/stretchr/testify/require" @@ -21,7 +22,7 @@ func TestAppendingRequests(t *testing.T) { extensionName := graphsync.ExtensionName("graphsync/awesome") extension := NamedExtension{ Name: extensionName, - Data: testutil.RandomBytes(100), + Data: basicnode.NewBytes(testutil.RandomBytes(100)), } root := testutil.GenerateCids(1)[0] ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) @@ -57,7 +58,7 @@ func TestAppendingRequests(t *testing.T) { require.False(t, pbRequest.Update) require.Equal(t, root.Bytes(), pbRequest.Root) require.Equal(t, selectorEncoded, pbRequest.Selector) - require.Equal(t, map[string][]byte{"graphsync/awesome": extension.Data}, pbRequest.Extensions) + require.Equal(t, map[string]ipld.Node{"graphsync/awesome": extension.Data}, pbRequest.Extensions) deserialized, err := newMessageFromProto(pbMessage) require.NoError(t, err, "deserializing protobuf message errored") @@ -66,7 +67,7 @@ func TestAppendingRequests(t *testing.T) { deserializedRequest := deserializedRequests[0] extensionData, found = deserializedRequest.Extension(extensionName) - require.Equal(t, id, deserializedRequest.ID()) + require.Equal(t, id, deserializedRequest.ID) require.False(t, deserializedRequest.Cancel) require.False(t, deserializedRequest.Update) require.Equal(t, priority, deserializedRequest.Priority) @@ -80,7 +81,7 @@ func TestAppendingResponses(t *testing.T) { extensionName := graphsync.ExtensionName("graphsync/awesome") extension := NamedExtension{ Name: extensionName, - Data: testutil.RandomBytes(100), + Data: basicnode.NewBytes(testutil.RandomBytes(100)), } requestID := graphsync.RequestID(rand.Int31()) status := graphsync.RequestAcknowledged @@ -167,7 +168,7 @@ func TestRequestCancel(t *testing.T) { requests := gsm.Requests require.Len(t, requests, 1, "did not add cancel request") request := requests[0] - require.Equal(t, id, request.ID()) + require.Equal(t, id, request.ID) require.True(t, request.Cancel) buf := new(bytes.Buffer) @@ -178,7 +179,7 @@ func TestRequestCancel(t *testing.T) { deserializedRequests := deserialized.Requests require.Len(t, deserializedRequests, 1, "did not add request to deserialized message") deserializedRequest := deserializedRequests[0] - require.Equal(t, request.ID(), deserializedRequest.ID()) + require.Equal(t, request.ID, deserializedRequest.ID) require.Equal(t, request.Cancel, deserializedRequest.Cancel) } @@ -188,7 +189,7 @@ func TestRequestUpdate(t *testing.T) { extensionName := graphsync.ExtensionName("graphsync/awesome") extension := NamedExtension{ Name: extensionName, - Data: testutil.RandomBytes(100), + Data: basicnode.NewBytes(testutil.RandomBytes(100)), } builder := NewBuilder() @@ -199,7 +200,7 @@ func TestRequestUpdate(t *testing.T) { requests := gsm.Requests require.Len(t, requests, 1, "did not add cancel request") request := requests[0] - require.Equal(t, id, request.ID()) + require.Equal(t, id, request.ID) require.True(t, request.Update) require.False(t, request.Cancel) extensionData, found := request.Extension(extensionName) @@ -216,12 +217,12 @@ func TestRequestUpdate(t *testing.T) { require.Len(t, deserializedRequests, 1, "did not add request to deserialized message") deserializedRequest := deserializedRequests[0] extensionData, found = deserializedRequest.Extension(extensionName) - require.Equal(t, request.ID(), deserializedRequest.ID()) + require.Equal(t, request.ID, deserializedRequest.ID) require.Equal(t, request.Cancel, deserializedRequest.Cancel) require.Equal(t, request.Update, deserializedRequest.Update) - require.Equal(t, request.Priority(), deserializedRequest.Priority()) - require.Equal(t, request.Root().String(), deserializedRequest.Root().String()) - require.Equal(t, request.Selector(), deserializedRequest.Selector()) + require.Equal(t, request.Priority, deserializedRequest.Priority) + require.Equal(t, request.Root.String(), deserializedRequest.Root.String()) + require.Equal(t, request.Selector, deserializedRequest.Selector) require.True(t, found) require.Equal(t, extension.Data, extensionData) } @@ -233,7 +234,7 @@ func TestToNetFromNetEquivalency(t *testing.T) { extensionName := graphsync.ExtensionName("graphsync/awesome") extension := NamedExtension{ Name: extensionName, - Data: testutil.RandomBytes(100), + Data: basicnode.NewBytes(testutil.RandomBytes(100)), } id := graphsync.RequestID(rand.Int31()) priority := graphsync.Priority(rand.Int31()) @@ -263,12 +264,12 @@ func TestToNetFromNetEquivalency(t *testing.T) { require.Len(t, deserializedRequests, 1, "did not add request to deserialized message") deserializedRequest := deserializedRequests[0] extensionData, found := deserializedRequest.Extension(extensionName) - require.Equal(t, request.ID(), deserializedRequest.ID()) + require.Equal(t, request.ID, deserializedRequest.ID) require.False(t, deserializedRequest.Cancel) require.False(t, deserializedRequest.Update) - require.Equal(t, request.Priority(), deserializedRequest.Priority()) - require.Equal(t, request.Root().String(), deserializedRequest.Root().String()) - require.Equal(t, request.Selector(), deserializedRequest.Selector()) + require.Equal(t, request.Priority, deserializedRequest.Priority) + require.Equal(t, request.Root.String(), deserializedRequest.Root.String()) + require.Equal(t, request.Selector, deserializedRequest.Selector) require.True(t, found) require.Equal(t, extension.Data, extensionData) @@ -280,17 +281,17 @@ func TestToNetFromNetEquivalency(t *testing.T) { deserializedResponse := deserializedResponses[0] extensionData, found = deserializedResponse.Extension(extensionName) require.Equal(t, response.ID, deserializedResponse.ID) - require.Equal(t, response.Status(), deserializedResponse.Status()) + require.Equal(t, response.Status, deserializedResponse.Status) require.True(t, found) require.Equal(t, extension.Data, extensionData) keys := make(map[cid.Cid]bool) - for _, b := range deserialized.Blocks() { - keys[b.Cid()] = true + for _, b := range deserialized.Blocks { + keys[b.BlockFormat().Cid()] = true } - for _, b := range gsm.Blocks() { - _, ok := keys[b.Cid()] + for _, b := range gsm.Blocks { + _, ok := keys[b.BlockFormat().Cid()] require.True(t, ok) } } @@ -302,25 +303,29 @@ func TestMergeExtensions(t *testing.T) { initialExtensions := []NamedExtension{ { Name: extensionName1, - Data: []byte("applesauce"), + Data: basicnode.NewBytes([]byte("applesauce")), }, { Name: extensionName2, - Data: []byte("hello"), + Data: basicnode.NewBytes([]byte("hello")), }, } - replacementExtensions := []graphsync.ExtensionData{ + replacementExtensions := []NamedExtension{ { Name: extensionName2, - Data: []byte("world"), + Data: basicnode.NewBytes([]byte("world")), }, { Name: extensionName3, - Data: []byte("cheese"), + Data: basicnode.NewBytes([]byte("cheese")), }, } - defaultMergeFunc := func(name graphsync.ExtensionName, oldData []byte, newData []byte) ([]byte, error) { - return []byte(string(oldData) + " " + string(newData)), nil + defaultMergeFunc := func(name graphsync.ExtensionName, oldNode, newNode ipld.Node) (ipld.Node, error) { + oldData, err := oldNode.AsBytes() + require.NoError(t, err) + newData, err := newNode.AsBytes() + require.NoError(t, err) + return basicnode.NewBytes([]byte(string(oldData) + " " + string(newData))), nil } root := testutil.GenerateCids(1)[0] ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) @@ -332,10 +337,10 @@ func TestMergeExtensions(t *testing.T) { emptyRequest := NewRequest(id, root, selector, priority) resultRequest, err := emptyRequest.MergeExtensions(replacementExtensions, defaultMergeFunc) require.NoError(t, err) - require.Equal(t, emptyRequest.ID(), resultRequest.ID()) - require.Equal(t, emptyRequest.Priority(), resultRequest.Priority()) - require.Equal(t, emptyRequest.Root().String(), resultRequest.Root().String()) - require.Equal(t, emptyRequest.Selector(), resultRequest.Selector()) + require.Equal(t, emptyRequest.ID, resultRequest.ID) + require.Equal(t, emptyRequest.Priority, resultRequest.Priority) + require.Equal(t, emptyRequest.Root.String(), resultRequest.Root.String()) + require.Equal(t, emptyRequest.Selector, resultRequest.Selector) _, has := resultRequest.Extension(extensionName1) require.False(t, has) extData2, has := resultRequest.Extension(extensionName2) @@ -363,7 +368,7 @@ func TestMergeExtensions(t *testing.T) { require.Equal(t, []byte("cheese"), extData3) }) t.Run("when merging errors", func(t *testing.T) { - errorMergeFunc := func(name graphsync.ExtensionName, oldData []byte, newData []byte) ([]byte, error) { + errorMergeFunc := func(name graphsync.ExtensionName, oldNode, newNode ipld.Node) (ipld.Node, error) { return nil, errors.New("something went wrong") } _, err := defaultRequest.MergeExtensions(replacementExtensions, errorMergeFunc) From 17d0b7a81895079ee06b84531de0335169a19059 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Tue, 11 Jan 2022 12:17:44 +0000 Subject: [PATCH 12/21] nearly all tests passing --- message/builder.go | 2 +- message/message.go | 37 +++++++++++++++++++++++++++---------- message/message_test.go | 16 ++++++++-------- 3 files changed, 36 insertions(+), 19 deletions(-) diff --git a/message/builder.go b/message/builder.go index 7faa10b9..2cb9002c 100644 --- a/message/builder.go +++ b/message/builder.go @@ -123,7 +123,7 @@ func (b *Builder) Build() (GraphSyncMessage, error) { responses = append(responses, NewResponse(requestID, responseCode(status, isComplete), b.extensions[requestID]...)) } // TODO: sort responses? - blocks := make([]GraphSyncBlock, len(b.outgoingBlocks)) + blocks := make([]GraphSyncBlock, 0, len(b.outgoingBlocks)) for _, block := range b.outgoingBlocks { blocks = append(blocks, FromBlockFormat(block)) } diff --git a/message/message.go b/message/message.go index 24f3cd4d..d1e7428b 100644 --- a/message/message.go +++ b/message/message.go @@ -296,6 +296,23 @@ func FromMsgReader(r msgio.Reader) (GraphSyncMessage, error) { return newMessageFromProto(&pb) } +func toProtoExtensions(m GraphSyncExtensions) map[string][]byte { + protoExts := make(map[string][]byte, len(m.Values)) + for name, node := range m.Values { + // Only keep those which are plain bytes, + // as those are the only ones that the older protocol clients understand. + if node.Kind() != ipld.Kind_Bytes { + continue + } + raw, err := node.AsBytes() + if err != nil { + panic(err) // shouldn't happen + } + protoExts[name] = raw + } + return protoExts +} + func (gsm GraphSyncMessage) ToProto() (*pb.Message, error) { pbm := new(pb.Message) pbm.Requests = make([]*pb.Message_Request, 0, len(gsm.Requests)) @@ -309,22 +326,22 @@ func (gsm GraphSyncMessage) ToProto() (*pb.Message, error) { } } pbm.Requests = append(pbm.Requests, &pb.Message_Request{ - Id: int32(request.ID), - Root: request.Root.Bytes(), - Selector: selector, - Priority: int32(request.Priority), - Cancel: request.Cancel, - Update: request.Update, - // Extensions: request.Extensions, TODO + Id: int32(request.ID), + Root: request.Root.Bytes(), + Selector: selector, + Priority: int32(request.Priority), + Cancel: request.Cancel, + Update: request.Update, + Extensions: toProtoExtensions(request.Extensions), }) } pbm.Responses = make([]*pb.Message_Response, 0, len(gsm.Responses)) for _, response := range gsm.Responses { pbm.Responses = append(pbm.Responses, &pb.Message_Response{ - Id: int32(response.ID), - Status: int32(response.Status), - // Extensions: response.Extensions, TODO + Id: int32(response.ID), + Status: int32(response.Status), + Extensions: toProtoExtensions(response.Extensions), }) } diff --git a/message/message_test.go b/message/message_test.go index 77be3d44..bc19f7cd 100644 --- a/message/message_test.go +++ b/message/message_test.go @@ -345,10 +345,10 @@ func TestMergeExtensions(t *testing.T) { require.False(t, has) extData2, has := resultRequest.Extension(extensionName2) require.True(t, has) - require.Equal(t, []byte("world"), extData2) + require.Equal(t, basicnode.NewBytes([]byte("world")), extData2) extData3, has := resultRequest.Extension(extensionName3) require.True(t, has) - require.Equal(t, []byte("cheese"), extData3) + require.Equal(t, basicnode.NewBytes([]byte("cheese")), extData3) }) t.Run("when merging two requests", func(t *testing.T) { resultRequest, err := defaultRequest.MergeExtensions(replacementExtensions, defaultMergeFunc) @@ -359,13 +359,13 @@ func TestMergeExtensions(t *testing.T) { require.Equal(t, defaultRequest.Selector, resultRequest.Selector) extData1, has := resultRequest.Extension(extensionName1) require.True(t, has) - require.Equal(t, []byte("applesauce"), extData1) + require.Equal(t, basicnode.NewBytes([]byte("applesauce")), extData1) extData2, has := resultRequest.Extension(extensionName2) require.True(t, has) - require.Equal(t, []byte("hello world"), extData2) + require.Equal(t, basicnode.NewBytes([]byte("hello world")), extData2) extData3, has := resultRequest.Extension(extensionName3) require.True(t, has) - require.Equal(t, []byte("cheese"), extData3) + require.Equal(t, basicnode.NewBytes([]byte("cheese")), extData3) }) t.Run("when merging errors", func(t *testing.T) { errorMergeFunc := func(name graphsync.ExtensionName, oldNode, newNode ipld.Node) (ipld.Node, error) { @@ -382,13 +382,13 @@ func TestMergeExtensions(t *testing.T) { require.Equal(t, defaultRequest.Selector, resultRequest.Selector) extData1, has := resultRequest.Extension(extensionName1) require.True(t, has) - require.Equal(t, []byte("applesauce"), extData1) + require.Equal(t, basicnode.NewBytes([]byte("applesauce")), extData1) extData2, has := resultRequest.Extension(extensionName2) require.True(t, has) - require.Equal(t, []byte("world"), extData2) + require.Equal(t, basicnode.NewBytes([]byte("world")), extData2) extData3, has := resultRequest.Extension(extensionName3) require.True(t, has) - require.Equal(t, []byte("cheese"), extData3) + require.Equal(t, basicnode.NewBytes([]byte("cheese")), extData3) }) } From 0186fd8333d12ab7400793f8030a14853f8858fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Tue, 11 Jan 2022 12:22:52 +0000 Subject: [PATCH 13/21] message tests passing --- message/builder.go | 14 +++++++++----- message/message_test.go | 10 ++++++---- 2 files changed, 15 insertions(+), 9 deletions(-) diff --git a/message/builder.go b/message/builder.go index 2cb9002c..8c92f72a 100644 --- a/message/builder.go +++ b/message/builder.go @@ -20,12 +20,13 @@ type Builder struct { completedResponses map[graphsync.RequestID]graphsync.ResponseStatusCode outgoingResponses map[graphsync.RequestID]metadata.Metadata extensions map[graphsync.RequestID][]NamedExtension - requests []GraphSyncRequest + requests map[graphsync.RequestID]GraphSyncRequest } // NewBuilder generates a new Builder. func NewBuilder() *Builder { return &Builder{ + requests: make(map[graphsync.RequestID]GraphSyncRequest), outgoingBlocks: make(map[cid.Cid]blocks.Block), completedResponses: make(map[graphsync.RequestID]graphsync.ResponseStatusCode), outgoingResponses: make(map[graphsync.RequestID]metadata.Metadata), @@ -35,7 +36,7 @@ func NewBuilder() *Builder { // AddRequest registers a new request to be added to the message. func (b *Builder) AddRequest(request GraphSyncRequest) { - b.requests = append(b.requests, request) + b.requests[request.ID] = request } // AddBlock adds the given block to the message. @@ -109,6 +110,10 @@ func (b *Builder) ScrubResponses(requestIDs []graphsync.RequestID) uint64 { // Build assembles and encodes message data from the added requests, links, and blocks. func (b *Builder) Build() (GraphSyncMessage, error) { + requests := make([]GraphSyncRequest, 0, len(b.requests)) + for _, request := range b.requests { + requests = append(requests, request) + } responses := make([]GraphSyncResponse, 0, len(b.outgoingResponses)) for requestID, linkMap := range b.outgoingResponses { mdRaw, err := metadata.EncodeMetadata(linkMap) @@ -122,14 +127,13 @@ func (b *Builder) Build() (GraphSyncMessage, error) { status, isComplete := b.completedResponses[requestID] responses = append(responses, NewResponse(requestID, responseCode(status, isComplete), b.extensions[requestID]...)) } - // TODO: sort responses? blocks := make([]GraphSyncBlock, 0, len(b.outgoingBlocks)) for _, block := range b.outgoingBlocks { blocks = append(blocks, FromBlockFormat(block)) } - // TODO: sort blocks? + // TODO: sort requests, responses, and blocks? map order is randomized return GraphSyncMessage{ - b.requests, responses, blocks, + requests, responses, blocks, }, nil } diff --git a/message/message_test.go b/message/message_test.go index bc19f7cd..052950f9 100644 --- a/message/message_test.go +++ b/message/message_test.go @@ -20,9 +20,10 @@ import ( func TestAppendingRequests(t *testing.T) { extensionName := graphsync.ExtensionName("graphsync/awesome") + extensionBytes := testutil.RandomBytes(100) extension := NamedExtension{ Name: extensionName, - Data: basicnode.NewBytes(testutil.RandomBytes(100)), + Data: basicnode.NewBytes(extensionBytes), } root := testutil.GenerateCids(1)[0] ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) @@ -58,7 +59,7 @@ func TestAppendingRequests(t *testing.T) { require.False(t, pbRequest.Update) require.Equal(t, root.Bytes(), pbRequest.Root) require.Equal(t, selectorEncoded, pbRequest.Selector) - require.Equal(t, map[string]ipld.Node{"graphsync/awesome": extension.Data}, pbRequest.Extensions) + require.Equal(t, map[string][]byte{"graphsync/awesome": extensionBytes}, pbRequest.Extensions) deserialized, err := newMessageFromProto(pbMessage) require.NoError(t, err, "deserializing protobuf message errored") @@ -79,9 +80,10 @@ func TestAppendingRequests(t *testing.T) { func TestAppendingResponses(t *testing.T) { extensionName := graphsync.ExtensionName("graphsync/awesome") + extensionBytes := testutil.RandomBytes(100) extension := NamedExtension{ Name: extensionName, - Data: basicnode.NewBytes(testutil.RandomBytes(100)), + Data: basicnode.NewBytes(extensionBytes), } requestID := graphsync.RequestID(rand.Int31()) status := graphsync.RequestAcknowledged @@ -105,7 +107,7 @@ func TestAppendingResponses(t *testing.T) { pbResponse := pbMessage.Responses[0] require.Equal(t, int32(requestID), pbResponse.Id) require.Equal(t, int32(status), pbResponse.Status) - require.Equal(t, extension.Data, pbResponse.Extensions["graphsync/awesome"]) + require.Equal(t, extensionBytes, pbResponse.Extensions["graphsync/awesome"]) deserialized, err := newMessageFromProto(pbMessage) require.NoError(t, err, "deserializing protobuf message errored") From 13b3ea7e28d22f96225d1d966ca86933cc45bde9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Tue, 11 Jan 2022 12:30:28 +0000 Subject: [PATCH 14/21] remove bindnode-generated types, as we use the native ones now --- message/ipldcbor/gen_test.go | 27 --------------------- message/ipldcbor/types.go | 46 ------------------------------------ 2 files changed, 73 deletions(-) delete mode 100644 message/ipldcbor/gen_test.go delete mode 100644 message/ipldcbor/types.go diff --git a/message/ipldcbor/gen_test.go b/message/ipldcbor/gen_test.go deleted file mode 100644 index e5051863..00000000 --- a/message/ipldcbor/gen_test.go +++ /dev/null @@ -1,27 +0,0 @@ -//go:build bindnodegen -// +build bindnodegen - -package ipldcbor - -import ( - "fmt" - "os" - "testing" - - "github.com/ipld/go-ipld-prime/node/bindnode" -) - -func TestGenerate(t *testing.T) { - f, err := os.Create("types.go") - if err != nil { - t.Fatal(err) - } - fmt.Fprintf(f, "package ipldcbor\n\n") - fmt.Fprintf(f, "import \"github.com/ipld/go-ipld-prime/datamodel\"\n\n") - if err := bindnode.ProduceGoTypes(f, schemaTypeSystem); err != nil { - t.Fatal(err) - } - if err := f.Close(); err != nil { - t.Fatal(err) - } -} diff --git a/message/ipldcbor/types.go b/message/ipldcbor/types.go deleted file mode 100644 index ec6fd871..00000000 --- a/message/ipldcbor/types.go +++ /dev/null @@ -1,46 +0,0 @@ -package ipldcbor - -import "github.com/ipld/go-ipld-prime/datamodel" - -type Map struct { - Keys []string - Values map[string]datamodel.Node -} -type List []datamodel.Node -type GraphSyncExtensions struct { - Keys []string - Values map[string]datamodel.Node -} -type GraphSyncMetadatum struct { - Link datamodel.Link - BlockPresent bool -} -type GraphSyncMetadata []GraphSyncMetadatum -type GraphSyncResponseStatusCode string -type GraphSyncRequest struct { - Id int - Root datamodel.Link - Selector datamodel.Node - Extensions GraphSyncExtensions - Priority int - Cancel bool - Update bool -} -type GraphSyncResponse struct { - Id int - Status GraphSyncResponseStatusCode - Metadata GraphSyncMetadata - Extensions GraphSyncExtensions -} -type GraphSyncBlock struct { - Prefix []uint8 - Data []uint8 -} -type List__GraphSyncRequest []GraphSyncRequest -type List__GraphSyncResponse []GraphSyncResponse -type List__GraphSyncBlock []GraphSyncBlock -type GraphSyncMessage struct { - Requests List__GraphSyncRequest - Responses List__GraphSyncResponse - Blocks List__GraphSyncBlock -} From c0d1a3653a40b984f8ba6914d3b2413571452786 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Tue, 11 Jan 2022 13:56:52 +0000 Subject: [PATCH 15/21] join ipldcbor package, no longer needed --- message/ipldcbor/schema_test.go | 6 ------ message/{ipldcbor => }/schema.go | 8 ++------ message/{ipldcbor => }/schema.ipldsch | 0 3 files changed, 2 insertions(+), 12 deletions(-) delete mode 100644 message/ipldcbor/schema_test.go rename message/{ipldcbor => }/schema.go (64%) rename message/{ipldcbor => }/schema.ipldsch (100%) diff --git a/message/ipldcbor/schema_test.go b/message/ipldcbor/schema_test.go deleted file mode 100644 index f6f5eeb3..00000000 --- a/message/ipldcbor/schema_test.go +++ /dev/null @@ -1,6 +0,0 @@ -package ipldcbor_test - -import "testing" - -func TestSchema(t *testing.T) { -} diff --git a/message/ipldcbor/schema.go b/message/schema.go similarity index 64% rename from message/ipldcbor/schema.go rename to message/schema.go index 41b4e0f0..970b801e 100644 --- a/message/ipldcbor/schema.go +++ b/message/schema.go @@ -1,6 +1,4 @@ -package ipldcbor - -//go:generate go test -run=Generate -vet=off -tags=bindnodegen +package message import ( _ "embed" @@ -8,8 +6,6 @@ import ( "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/node/bindnode" "github.com/ipld/go-ipld-prime/schema" - - "github.com/ipfs/go-graphsync/message" ) //go:embed schema.ipldsch @@ -28,5 +24,5 @@ func init() { } schemaTypeSystem = ts - Prototype.Message = bindnode.Prototype((*message.GraphSyncMessage)(nil), ts.TypeByName("GraphSyncMessage")) + Prototype.Message = bindnode.Prototype((*GraphSyncMessage)(nil), ts.TypeByName("GraphSyncMessage")) } diff --git a/message/ipldcbor/schema.ipldsch b/message/schema.ipldsch similarity index 100% rename from message/ipldcbor/schema.ipldsch rename to message/schema.ipldsch From be55f9463d969d2688d276a23f0d7b54e55c0beb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Tue, 11 Jan 2022 14:24:59 +0000 Subject: [PATCH 16/21] add first roundtrip benchmark --- message/bench_test.go | 81 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 81 insertions(+) create mode 100644 message/bench_test.go diff --git a/message/bench_test.go b/message/bench_test.go new file mode 100644 index 00000000..0c2d34cf --- /dev/null +++ b/message/bench_test.go @@ -0,0 +1,81 @@ +package message_test + +import ( + "bytes" + "math/rand" + "testing" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-graphsync" + "github.com/ipfs/go-graphsync/message" + "github.com/ipfs/go-graphsync/testutil" + "github.com/ipld/go-ipld-prime/codec/dagcbor" + "github.com/ipld/go-ipld-prime/node/basicnode" + "github.com/ipld/go-ipld-prime/node/bindnode" + "github.com/ipld/go-ipld-prime/traversal/selector/builder" + "github.com/stretchr/testify/require" +) + +func BenchmarkMessageEncodingRoundtrip(b *testing.B) { + root := testutil.GenerateCids(1)[0] + ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) + selector := ssb.Matcher().Node() + extensionName := graphsync.ExtensionName("graphsync/awesome") + extension := message.NamedExtension{ + Name: extensionName, + Data: basicnode.NewBytes(testutil.RandomBytes(100)), + } + id := graphsync.RequestID(rand.Int31()) + priority := graphsync.Priority(rand.Int31()) + status := graphsync.RequestAcknowledged + + builder := message.NewBuilder() + builder.AddRequest(message.NewRequest(id, root, selector, priority, extension)) + builder.AddResponseCode(id, status) + builder.AddExtensionData(id, extension) + builder.AddBlock(blocks.NewBlock([]byte("W"))) + builder.AddBlock(blocks.NewBlock([]byte("E"))) + builder.AddBlock(blocks.NewBlock([]byte("F"))) + builder.AddBlock(blocks.NewBlock([]byte("M"))) + + gsm, err := builder.Build() + require.NoError(b, err) + + b.Run("Protobuf", func(b *testing.B) { + b.ReportAllocs() + b.RunParallel(func(pb *testing.PB) { + buf := new(bytes.Buffer) + for pb.Next() { + buf.Reset() + + err := gsm.ToNet(buf) + require.NoError(b, err) + + gsm2, err := message.FromNet(buf) + require.NoError(b, err) + require.Equal(b, gsm, gsm2) + } + }) + }) + + b.Run("DagCbor", func(b *testing.B) { + b.ReportAllocs() + b.RunParallel(func(pb *testing.PB) { + buf := new(bytes.Buffer) + for pb.Next() { + buf.Reset() + + node := bindnode.Wrap(&gsm, message.Prototype.Message.Type()) + err := dagcbor.Encode(node.Representation(), buf) + require.NoError(b, err) + + builder := message.Prototype.Message.Representation().NewBuilder() + err = dagcbor.Decode(builder, buf) + require.NoError(b, err) + node2 := builder.Build() + gsm2 := *bindnode.Unwrap(node2).(*message.GraphSyncMessage) + require.Equal(b, gsm, gsm2) + } + }) + }) +} From 4800f1c478ae24c06e5a9424a56edd891ea67a41 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Thu, 13 Jan 2022 15:04:41 +0000 Subject: [PATCH 17/21] move ipld cbor out of the message package again --- message/bench_test.go | 7 +- message/ipldbind/message.go | 469 ++++++++++++++++++++++++++ message/{ => ipldbind}/schema.go | 2 +- message/{ => ipldbind}/schema.ipldsch | 0 4 files changed, 474 insertions(+), 4 deletions(-) create mode 100644 message/ipldbind/message.go rename message/{ => ipldbind}/schema.go (96%) rename message/{ => ipldbind}/schema.ipldsch (100%) diff --git a/message/bench_test.go b/message/bench_test.go index 0c2d34cf..efd4b434 100644 --- a/message/bench_test.go +++ b/message/bench_test.go @@ -8,6 +8,7 @@ import ( blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-graphsync" "github.com/ipfs/go-graphsync/message" + "github.com/ipfs/go-graphsync/message/ipldbind" "github.com/ipfs/go-graphsync/testutil" "github.com/ipld/go-ipld-prime/codec/dagcbor" "github.com/ipld/go-ipld-prime/node/basicnode" @@ -65,15 +66,15 @@ func BenchmarkMessageEncodingRoundtrip(b *testing.B) { for pb.Next() { buf.Reset() - node := bindnode.Wrap(&gsm, message.Prototype.Message.Type()) + node := bindnode.Wrap(&gsm, ipldbind.Prototype.Message.Type()) err := dagcbor.Encode(node.Representation(), buf) require.NoError(b, err) - builder := message.Prototype.Message.Representation().NewBuilder() + builder := ipldbind.Prototype.Message.Representation().NewBuilder() err = dagcbor.Decode(builder, buf) require.NoError(b, err) node2 := builder.Build() - gsm2 := *bindnode.Unwrap(node2).(*message.GraphSyncMessage) + gsm2 := *bindnode.Unwrap(node2).(*ipldbind.GraphSyncMessage) require.Equal(b, gsm, gsm2) } }) diff --git a/message/ipldbind/message.go b/message/ipldbind/message.go new file mode 100644 index 00000000..d0dc5cce --- /dev/null +++ b/message/ipldbind/message.go @@ -0,0 +1,469 @@ +package ipldbind + +import ( + "encoding/binary" + "errors" + "fmt" + "io" + "sort" + + blocks "github.com/ipfs/go-block-format" + cid "github.com/ipfs/go-cid" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/datamodel" + "github.com/ipld/go-ipld-prime/node/basicnode" + pool "github.com/libp2p/go-buffer-pool" + "github.com/libp2p/go-libp2p-core/network" + "github.com/libp2p/go-msgio" + "google.golang.org/protobuf/proto" + + "github.com/ipfs/go-graphsync" + "github.com/ipfs/go-graphsync/ipldutil" + pb "github.com/ipfs/go-graphsync/message/pb" +) + +// IsTerminalSuccessCode returns true if the response code indicates the +// request terminated successfully. +// DEPRECATED: use status.IsSuccess() +func IsTerminalSuccessCode(status graphsync.ResponseStatusCode) bool { + return status.IsSuccess() +} + +// IsTerminalFailureCode returns true if the response code indicates the +// request terminated in failure. +// DEPRECATED: use status.IsFailure() +func IsTerminalFailureCode(status graphsync.ResponseStatusCode) bool { + return status.IsFailure() +} + +// IsTerminalResponseCode returns true if the response code signals +// the end of the request +// DEPRECATED: use status.IsTerminal() +func IsTerminalResponseCode(status graphsync.ResponseStatusCode) bool { + return status.IsTerminal() +} + +// Exportable is an interface that can serialize to a protobuf +type Exportable interface { + ToProto() (*pb.Message, error) + ToNet(w io.Writer) error +} + +type GraphSyncExtensions struct { + Keys []string + Values map[string]datamodel.Node +} + +// GraphSyncRequest is a struct to capture data on a request contained in a +// GraphSyncMessage. +type GraphSyncRequest struct { + ID graphsync.RequestID + + Root cid.Cid + Selector ipld.Node + Extensions GraphSyncExtensions + Priority graphsync.Priority + Cancel bool + Update bool +} + +type GraphSyncMetadatum struct { + Link datamodel.Link + BlockPresent bool +} + +// GraphSyncResponse is an struct to capture data on a response sent back +// in a GraphSyncMessage. +type GraphSyncResponse struct { + ID graphsync.RequestID + + Status graphsync.ResponseStatusCode + Metadata []GraphSyncMetadatum + Extensions GraphSyncExtensions +} + +type GraphSyncBlock struct { + Prefix []byte + Data []byte +} + +func FromBlockFormat(block blocks.Block) GraphSyncBlock { + return GraphSyncBlock{ + Prefix: block.Cid().Prefix().Bytes(), + Data: block.RawData(), + } +} + +func (b GraphSyncBlock) BlockFormat() *blocks.BasicBlock { + pref, err := cid.PrefixFromBytes(b.Prefix) + if err != nil { + panic(err) // should never happen + } + + c, err := pref.Sum(b.Data) + if err != nil { + panic(err) // should never happen + } + + block, err := blocks.NewBlockWithCid(b.Data, c) + if err != nil { + panic(err) // should never happen + } + return block +} + +func BlockFormatSlice(bs []GraphSyncBlock) []blocks.Block { + blks := make([]blocks.Block, len(bs)) + for i, b := range bs { + blks[i] = b.BlockFormat() + } + return blks +} + +type GraphSyncMessage struct { + Requests []GraphSyncRequest + Responses []GraphSyncResponse + Blocks []GraphSyncBlock +} + +// NewRequest builds a new Graphsync request +func NewRequest(id graphsync.RequestID, + root cid.Cid, + selector ipld.Node, + priority graphsync.Priority, + extensions ...NamedExtension) GraphSyncRequest { + + return newRequest(id, root, selector, priority, false, false, toExtensionsMap(extensions)) +} + +// CancelRequest request generates a request to cancel an in progress request +func CancelRequest(id graphsync.RequestID) GraphSyncRequest { + return newRequest(id, cid.Cid{}, nil, 0, true, false, GraphSyncExtensions{}) +} + +// UpdateRequest generates a new request to update an in progress request with the given extensions +func UpdateRequest(id graphsync.RequestID, extensions ...NamedExtension) GraphSyncRequest { + return newRequest(id, cid.Cid{}, nil, 0, false, true, toExtensionsMap(extensions)) +} + +// NamedExtension exists just for the purpose of the constructors. +type NamedExtension struct { + Name graphsync.ExtensionName + Data ipld.Node +} + +func toExtensionsMap(extensions []NamedExtension) (m GraphSyncExtensions) { + if len(extensions) > 0 { + m.Keys = make([]string, len(extensions)) + m.Values = make(map[string]ipld.Node, len(extensions)) + for i, ext := range extensions { + m.Keys[i] = string(ext.Name) + m.Values[string(ext.Name)] = ext.Data + } + } + return m +} + +func fromProtoExtensions(protoExts map[string][]byte) GraphSyncExtensions { + var exts []NamedExtension + for name, data := range protoExts { + exts = append(exts, NamedExtension{graphsync.ExtensionName(name), basicnode.NewBytes(data)}) + } + // Iterating over the map above is non-deterministic, + // so sort by the unique names to ensure determinism. + sort.Slice(exts, func(i, j int) bool { return exts[i].Name < exts[j].Name }) + return toExtensionsMap(exts) +} + +func newRequest(id graphsync.RequestID, + root cid.Cid, + selector ipld.Node, + priority graphsync.Priority, + isCancel bool, + isUpdate bool, + extensions GraphSyncExtensions) GraphSyncRequest { + return GraphSyncRequest{ + ID: id, + Root: root, + Selector: selector, + Priority: priority, + Cancel: isCancel, + Update: isUpdate, + Extensions: extensions, + } +} + +// NewResponse builds a new Graphsync response +func NewResponse(requestID graphsync.RequestID, + status graphsync.ResponseStatusCode, + extensions ...NamedExtension) GraphSyncResponse { + return newResponse(requestID, status, toExtensionsMap(extensions)) +} + +func newResponse(requestID graphsync.RequestID, + status graphsync.ResponseStatusCode, extensions GraphSyncExtensions) GraphSyncResponse { + return GraphSyncResponse{ + ID: requestID, + Status: status, + Extensions: extensions, + } +} + +func newMessageFromProto(pbm *pb.Message) (GraphSyncMessage, error) { + requests := make([]GraphSyncRequest, len(pbm.GetRequests())) + for i, req := range pbm.Requests { + if req == nil { + return GraphSyncMessage{}, errors.New("request is nil") + } + var root cid.Cid + var err error + if !req.Cancel && !req.Update { + root, err = cid.Cast(req.Root) + if err != nil { + return GraphSyncMessage{}, err + } + } + + var selector ipld.Node + if !req.Cancel && !req.Update { + selector, err = ipldutil.DecodeNode(req.Selector) + if err != nil { + return GraphSyncMessage{}, err + } + } + // TODO: we likely need to turn some "core" extensions to fields, + // as some of those got moved to proper fields in the new protocol. + // Same for responses above, as well as the "to proto" funcs. + requests[i] = newRequest(graphsync.RequestID(req.Id), root, selector, graphsync.Priority(req.Priority), req.Cancel, req.Update, fromProtoExtensions(req.GetExtensions())) + } + + responses := make([]GraphSyncResponse, len(pbm.GetResponses())) + for i, res := range pbm.Responses { + if res == nil { + return GraphSyncMessage{}, errors.New("response is nil") + } + responses[i] = newResponse(graphsync.RequestID(res.Id), graphsync.ResponseStatusCode(res.Status), fromProtoExtensions(res.GetExtensions())) + } + + blks := make([]GraphSyncBlock, len(pbm.GetData())) + for i, b := range pbm.GetData() { + if b == nil { + return GraphSyncMessage{}, errors.New("block is nil") + } + blks[i] = GraphSyncBlock{ + Prefix: b.GetPrefix(), + Data: b.GetData(), + } + } + + return GraphSyncMessage{ + requests, responses, blks, + }, nil +} + +func (gsm GraphSyncMessage) Empty() bool { + return len(gsm.Blocks) == 0 && len(gsm.Requests) == 0 && len(gsm.Responses) == 0 +} + +func (gsm GraphSyncMessage) ResponseCodes() map[graphsync.RequestID]graphsync.ResponseStatusCode { + codes := make(map[graphsync.RequestID]graphsync.ResponseStatusCode, len(gsm.Responses)) + for _, response := range gsm.Responses { + codes[response.ID] = response.Status + } + return codes +} + +// FromNet can read a network stream to deserialized a GraphSyncMessage +func FromNet(r io.Reader) (GraphSyncMessage, error) { + reader := msgio.NewVarintReaderSize(r, network.MessageSizeMax) + return FromMsgReader(reader) +} + +// FromMsgReader can deserialize a protobuf message into a GraphySyncMessage. +func FromMsgReader(r msgio.Reader) (GraphSyncMessage, error) { + msg, err := r.ReadMsg() + if err != nil { + return GraphSyncMessage{}, err + } + + var pb pb.Message + err = proto.Unmarshal(msg, &pb) + r.ReleaseMsg(msg) + if err != nil { + return GraphSyncMessage{}, err + } + + return newMessageFromProto(&pb) +} + +func toProtoExtensions(m GraphSyncExtensions) map[string][]byte { + protoExts := make(map[string][]byte, len(m.Values)) + for name, node := range m.Values { + // Only keep those which are plain bytes, + // as those are the only ones that the older protocol clients understand. + if node.Kind() != ipld.Kind_Bytes { + continue + } + raw, err := node.AsBytes() + if err != nil { + panic(err) // shouldn't happen + } + protoExts[name] = raw + } + return protoExts +} + +func (gsm GraphSyncMessage) ToProto() (*pb.Message, error) { + pbm := new(pb.Message) + pbm.Requests = make([]*pb.Message_Request, 0, len(gsm.Requests)) + for _, request := range gsm.Requests { + var selector []byte + var err error + if request.Selector != nil { + selector, err = ipldutil.EncodeNode(request.Selector) + if err != nil { + return nil, err + } + } + pbm.Requests = append(pbm.Requests, &pb.Message_Request{ + Id: int32(request.ID), + Root: request.Root.Bytes(), + Selector: selector, + Priority: int32(request.Priority), + Cancel: request.Cancel, + Update: request.Update, + Extensions: toProtoExtensions(request.Extensions), + }) + } + + pbm.Responses = make([]*pb.Message_Response, 0, len(gsm.Responses)) + for _, response := range gsm.Responses { + pbm.Responses = append(pbm.Responses, &pb.Message_Response{ + Id: int32(response.ID), + Status: int32(response.Status), + Extensions: toProtoExtensions(response.Extensions), + }) + } + + pbm.Data = make([]*pb.Message_Block, 0, len(gsm.Blocks)) + for _, b := range gsm.Blocks { + pbm.Data = append(pbm.Data, &pb.Message_Block{ + Prefix: b.Prefix, + Data: b.Data, + }) + } + return pbm, nil +} + +func (gsm GraphSyncMessage) ToNet(w io.Writer) error { + msg, err := gsm.ToProto() + if err != nil { + return err + } + size := proto.Size(msg) + buf := pool.Get(size + binary.MaxVarintLen64) + defer pool.Put(buf) + + n := binary.PutUvarint(buf, uint64(size)) + + out, err := proto.MarshalOptions{}.MarshalAppend(buf[:n], msg) + if err != nil { + return err + } + _, err = w.Write(out) + return err +} + +func (gsm GraphSyncMessage) Loggable() map[string]interface{} { + requests := make([]string, 0, len(gsm.Requests)) + for _, request := range gsm.Requests { + requests = append(requests, fmt.Sprintf("%d", request.ID)) + } + responses := make([]string, 0, len(gsm.Responses)) + for _, response := range gsm.Responses { + responses = append(responses, fmt.Sprintf("%d", response.ID)) + } + return map[string]interface{}{ + "requests": requests, + "responses": responses, + } +} + +func (gsm GraphSyncMessage) Clone() GraphSyncMessage { + requests := append([]GraphSyncRequest{}, gsm.Requests...) + responses := append([]GraphSyncResponse{}, gsm.Responses...) + blocks := append([]GraphSyncBlock{}, gsm.Blocks...) + return GraphSyncMessage{requests, responses, blocks} +} + +// Extension returns the content for an extension on a response, or errors +// if extension is not present +func (gsr GraphSyncRequest) Extension(name graphsync.ExtensionName) (ipld.Node, bool) { + val, ok := gsr.Extensions.Values[string(name)] + if !ok { + return nil, false + } + return val, true +} + +// ExtensionNames returns the names of the extensions included in this request +func (gsr GraphSyncRequest) ExtensionNames() []string { + return gsr.Extensions.Keys +} + +// Extension returns the content for an extension on a response, or errors +// if extension is not present +func (gsr GraphSyncResponse) Extension(name graphsync.ExtensionName) (ipld.Node, bool) { + val, ok := gsr.Extensions.Values[string(name)] + if !ok { + return nil, false + } + return val, true +} + +// ExtensionNames returns the names of the extensions included in this request +func (gsr GraphSyncResponse) ExtensionNames() []string { + return gsr.Extensions.Keys +} + +// ReplaceExtensions merges the extensions given extensions into the request to create a new request, +// but always uses new data +func (gsr GraphSyncRequest) ReplaceExtensions(extensions []NamedExtension) GraphSyncRequest { + req, _ := gsr.MergeExtensions(extensions, func(name graphsync.ExtensionName, oldNode, newNode ipld.Node) (ipld.Node, error) { + return newNode, nil + }) + return req +} + +// MergeExtensions merges the given list of extensions to produce a new request with the combination of the old request +// plus the new extensions. When an old extension and a new extension are both present, mergeFunc is called to produce +// the result +func (gsr GraphSyncRequest) MergeExtensions(extensions []NamedExtension, mergeFunc func(name graphsync.ExtensionName, oldNode, newNode ipld.Node) (ipld.Node, error)) (GraphSyncRequest, error) { + if len(gsr.Extensions.Keys) == 0 { + return newRequest(gsr.ID, gsr.Root, gsr.Selector, gsr.Priority, gsr.Cancel, gsr.Update, toExtensionsMap(extensions)), nil + } + combinedExtensions := make(map[string]ipld.Node) + for _, newExt := range extensions { + oldNode, ok := gsr.Extensions.Values[string(newExt.Name)] + if !ok { + combinedExtensions[string(newExt.Name)] = newExt.Data + continue + } + resultNode, err := mergeFunc(graphsync.ExtensionName(newExt.Name), oldNode, newExt.Data) + if err != nil { + return GraphSyncRequest{}, err + } + combinedExtensions[string(newExt.Name)] = resultNode + } + + for name, oldNode := range gsr.Extensions.Values { + _, ok := combinedExtensions[name] + if ok { + continue + } + combinedExtensions[name] = oldNode + } + extNames := make([]string, len(combinedExtensions)) + sort.Strings(extNames) // for reproducibility + return newRequest(gsr.ID, gsr.Root, gsr.Selector, gsr.Priority, gsr.Cancel, gsr.Update, GraphSyncExtensions{extNames, combinedExtensions}), nil +} diff --git a/message/schema.go b/message/ipldbind/schema.go similarity index 96% rename from message/schema.go rename to message/ipldbind/schema.go index 970b801e..ec7be58a 100644 --- a/message/schema.go +++ b/message/ipldbind/schema.go @@ -1,4 +1,4 @@ -package message +package ipldbind import ( _ "embed" diff --git a/message/schema.ipldsch b/message/ipldbind/schema.ipldsch similarity index 100% rename from message/schema.ipldsch rename to message/ipldbind/schema.ipldsch From 01ecf662e685111ea4a7f8cc0d0332e34bd13f7c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Thu, 13 Jan 2022 15:23:47 +0000 Subject: [PATCH 18/21] add first version of to/from ipld --- message/bench_test.go | 37 +-- message/builder.go | 30 +-- message/builder_test.go | 73 ++---- message/message.go | 489 ++++++++++++++++++++++++---------------- message/message_test.go | 189 ++++++++-------- 5 files changed, 443 insertions(+), 375 deletions(-) diff --git a/message/bench_test.go b/message/bench_test.go index efd4b434..970f3a22 100644 --- a/message/bench_test.go +++ b/message/bench_test.go @@ -1,4 +1,4 @@ -package message_test +package message import ( "bytes" @@ -7,7 +7,6 @@ import ( blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-graphsync" - "github.com/ipfs/go-graphsync/message" "github.com/ipfs/go-graphsync/message/ipldbind" "github.com/ipfs/go-graphsync/testutil" "github.com/ipld/go-ipld-prime/codec/dagcbor" @@ -21,19 +20,20 @@ func BenchmarkMessageEncodingRoundtrip(b *testing.B) { root := testutil.GenerateCids(1)[0] ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) selector := ssb.Matcher().Node() - extensionName := graphsync.ExtensionName("graphsync/awesome") - extension := message.NamedExtension{ - Name: extensionName, - Data: basicnode.NewBytes(testutil.RandomBytes(100)), - } + // extensionName := graphsync.ExtensionName("graphsync/awesome") + // extension := NamedExtension{ + // Name: extensionName, + // Data: basicnode.NewBytes(testutil.RandomBytes(100)), + // } id := graphsync.RequestID(rand.Int31()) priority := graphsync.Priority(rand.Int31()) status := graphsync.RequestAcknowledged - builder := message.NewBuilder() - builder.AddRequest(message.NewRequest(id, root, selector, priority, extension)) + builder := NewBuilder() + // builder.AddRequest(NewRequest(id, root, selector, priority, extension)) + builder.AddRequest(NewRequest(id, root, selector, priority)) builder.AddResponseCode(id, status) - builder.AddExtensionData(id, extension) + // builder.AddExtensionData(id, extension) builder.AddBlock(blocks.NewBlock([]byte("W"))) builder.AddBlock(blocks.NewBlock([]byte("E"))) builder.AddBlock(blocks.NewBlock([]byte("F"))) @@ -52,9 +52,11 @@ func BenchmarkMessageEncodingRoundtrip(b *testing.B) { err := gsm.ToNet(buf) require.NoError(b, err) - gsm2, err := message.FromNet(buf) + gsm2, err := FromNet(buf) require.NoError(b, err) - require.Equal(b, gsm, gsm2) + // Unfortunately, protobuf encoding isn't canonical nor stable. + _ = gsm2 + // require.Equal(b, gsm, gsm2) } }) }) @@ -66,15 +68,20 @@ func BenchmarkMessageEncodingRoundtrip(b *testing.B) { for pb.Next() { buf.Reset() - node := bindnode.Wrap(&gsm, ipldbind.Prototype.Message.Type()) - err := dagcbor.Encode(node.Representation(), buf) + ipldGSM, err := gsm.ToIPLD() + require.NoError(b, err) + node := bindnode.Wrap(ipldGSM, ipldbind.Prototype.Message.Type()) + err = dagcbor.Encode(node.Representation(), buf) require.NoError(b, err) builder := ipldbind.Prototype.Message.Representation().NewBuilder() err = dagcbor.Decode(builder, buf) require.NoError(b, err) node2 := builder.Build() - gsm2 := *bindnode.Unwrap(node2).(*ipldbind.GraphSyncMessage) + ipldGSM2 := bindnode.Unwrap(node2).(*ipldbind.GraphSyncMessage) + gsm2, err := messageFromIPLD(ipldGSM2) + require.NoError(b, err) + require.Equal(b, gsm, gsm2) } }) diff --git a/message/builder.go b/message/builder.go index 8c92f72a..15017198 100644 --- a/message/builder.go +++ b/message/builder.go @@ -2,10 +2,9 @@ package message import ( blocks "github.com/ipfs/go-block-format" - cid "github.com/ipfs/go-cid" + "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime" cidlink "github.com/ipld/go-ipld-prime/linking/cid" - "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipfs/go-graphsync" "github.com/ipfs/go-graphsync/metadata" @@ -19,7 +18,7 @@ type Builder struct { blkSize uint64 completedResponses map[graphsync.RequestID]graphsync.ResponseStatusCode outgoingResponses map[graphsync.RequestID]metadata.Metadata - extensions map[graphsync.RequestID][]NamedExtension + extensions map[graphsync.RequestID][]graphsync.ExtensionData requests map[graphsync.RequestID]GraphSyncRequest } @@ -30,13 +29,13 @@ func NewBuilder() *Builder { outgoingBlocks: make(map[cid.Cid]blocks.Block), completedResponses: make(map[graphsync.RequestID]graphsync.ResponseStatusCode), outgoingResponses: make(map[graphsync.RequestID]metadata.Metadata), - extensions: make(map[graphsync.RequestID][]NamedExtension), + extensions: make(map[graphsync.RequestID][]graphsync.ExtensionData), } } // AddRequest registers a new request to be added to the message. func (b *Builder) AddRequest(request GraphSyncRequest) { - b.requests[request.ID] = request + b.requests[request.ID()] = request } // AddBlock adds the given block to the message. @@ -46,7 +45,7 @@ func (b *Builder) AddBlock(block blocks.Block) { } // AddExtensionData adds the given extension data to to the message -func (b *Builder) AddExtensionData(requestID graphsync.RequestID, extension NamedExtension) { +func (b *Builder) AddExtensionData(requestID graphsync.RequestID, extension graphsync.ExtensionData) { b.extensions[requestID] = append(b.extensions[requestID], extension) // make sure this extension goes out in next response even if no links are sent _, ok := b.outgoingResponses[requestID] @@ -110,30 +109,21 @@ func (b *Builder) ScrubResponses(requestIDs []graphsync.RequestID) uint64 { // Build assembles and encodes message data from the added requests, links, and blocks. func (b *Builder) Build() (GraphSyncMessage, error) { - requests := make([]GraphSyncRequest, 0, len(b.requests)) - for _, request := range b.requests { - requests = append(requests, request) - } - responses := make([]GraphSyncResponse, 0, len(b.outgoingResponses)) + responses := make(map[graphsync.RequestID]GraphSyncResponse, len(b.outgoingResponses)) for requestID, linkMap := range b.outgoingResponses { mdRaw, err := metadata.EncodeMetadata(linkMap) if err != nil { return GraphSyncMessage{}, err } - b.extensions[requestID] = append(b.extensions[requestID], NamedExtension{ + b.extensions[requestID] = append(b.extensions[requestID], graphsync.ExtensionData{ Name: graphsync.ExtensionMetadata, - Data: basicnode.NewBytes(mdRaw), // TODO: likely wrong + Data: mdRaw, }) status, isComplete := b.completedResponses[requestID] - responses = append(responses, NewResponse(requestID, responseCode(status, isComplete), b.extensions[requestID]...)) - } - blocks := make([]GraphSyncBlock, 0, len(b.outgoingBlocks)) - for _, block := range b.outgoingBlocks { - blocks = append(blocks, FromBlockFormat(block)) + responses[requestID] = NewResponse(requestID, responseCode(status, isComplete), b.extensions[requestID]...) } - // TODO: sort requests, responses, and blocks? map order is randomized return GraphSyncMessage{ - requests, responses, blocks, + b.requests, responses, b.outgoingBlocks, }, nil } diff --git a/message/builder_test.go b/message/builder_test.go index 4dac2132..b9722c84 100644 --- a/message/builder_test.go +++ b/message/builder_test.go @@ -1,14 +1,12 @@ package message import ( - "bytes" "io" "math/rand" "testing" "github.com/ipld/go-ipld-prime" cidlink "github.com/ipld/go-ipld-prime/linking/cid" - "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/stretchr/testify/require" "github.com/ipfs/go-graphsync" @@ -16,42 +14,21 @@ import ( "github.com/ipfs/go-graphsync/testutil" ) -// Like the funcs in testutil above, but using blocks at the protocol level. -// We can't put them there right away, due to import cycles. -// We need to refactor these tests to be external, i.e. "package message_test". - -func ContainsGraphSyncBlock(blks []GraphSyncBlock, block GraphSyncBlock) bool { - for _, blk := range blks { - if bytes.Equal(blk.Prefix, block.Prefix) && bytes.Equal(blk.Data, block.Data) { - return true - } - } - return false -} -func AssertContainsGraphSyncBlock(t testing.TB, blks []GraphSyncBlock, block GraphSyncBlock) { - t.Helper() - require.True(t, ContainsGraphSyncBlock(blks, block), "given block should be in list") -} -func RefuteContainsGraphSyncBlock(t testing.TB, blks []GraphSyncBlock, block GraphSyncBlock) { - t.Helper() - require.False(t, ContainsGraphSyncBlock(blks, block), "given block should not be in list") -} - func TestMessageBuilding(t *testing.T) { blocks := testutil.GenerateBlocksOfSize(3, 100) links := make([]ipld.Link, 0, len(blocks)) for _, block := range blocks { links = append(links, cidlink.Link{Cid: block.Cid()}) } - extensionData1 := basicnode.NewBytes(testutil.RandomBytes(100)) + extensionData1 := testutil.RandomBytes(100) extensionName1 := graphsync.ExtensionName("AppleSauce/McGee") - extension1 := NamedExtension{ + extension1 := graphsync.ExtensionData{ Name: extensionName1, Data: extensionData1, } - extensionData2 := basicnode.NewBytes(testutil.RandomBytes(100)) + extensionData2 := testutil.RandomBytes(100) extensionName2 := graphsync.ExtensionName("HappyLand/Happenstance") - extension2 := NamedExtension{ + extension2 := graphsync.ExtensionData{ Name: extensionName2, Data: extensionData2, } @@ -98,12 +75,12 @@ func TestMessageBuilding(t *testing.T) { }, checkMsg: func(t *testing.T, message GraphSyncMessage) { - responses := message.Responses - sentBlocks := BlockFormatSlice(message.Blocks) + responses := message.Responses() + sentBlocks := message.Blocks() require.Len(t, responses, 4, "did not assemble correct number of responses") response1 := findResponseForRequestID(t, responses, requestID1) - require.Equal(t, graphsync.RequestCompletedPartial, response1.Status, "did not generate completed partial response") + require.Equal(t, graphsync.RequestCompletedPartial, response1.Status(), "did not generate completed partial response") assertMetadata(t, response1, metadata.Metadata{ metadata.Item{Link: links[0].(cidlink.Link).Cid, BlockPresent: true}, metadata.Item{Link: links[1].(cidlink.Link).Cid, BlockPresent: false}, @@ -112,7 +89,7 @@ func TestMessageBuilding(t *testing.T) { assertExtension(t, response1, extension1) response2 := findResponseForRequestID(t, responses, requestID2) - require.Equal(t, graphsync.RequestCompletedFull, response2.Status, "did not generate completed full response") + require.Equal(t, graphsync.RequestCompletedFull, response2.Status(), "did not generate completed full response") assertMetadata(t, response2, metadata.Metadata{ metadata.Item{Link: links[1].(cidlink.Link).Cid, BlockPresent: true}, metadata.Item{Link: links[2].(cidlink.Link).Cid, BlockPresent: true}, @@ -120,7 +97,7 @@ func TestMessageBuilding(t *testing.T) { }) response3 := findResponseForRequestID(t, responses, requestID3) - require.Equal(t, graphsync.PartialResponse, response3.Status, "did not generate partial response") + require.Equal(t, graphsync.PartialResponse, response3.Status(), "did not generate partial response") assertMetadata(t, response3, metadata.Metadata{ metadata.Item{Link: links[0].(cidlink.Link).Cid, BlockPresent: true}, metadata.Item{Link: links[1].(cidlink.Link).Cid, BlockPresent: true}, @@ -128,7 +105,7 @@ func TestMessageBuilding(t *testing.T) { assertExtension(t, response3, extension2) response4 := findResponseForRequestID(t, responses, requestID4) - require.Equal(t, graphsync.RequestCompletedFull, response4.Status, "did not generate completed full response") + require.Equal(t, graphsync.RequestCompletedFull, response4.Status(), "did not generate completed full response") require.Equal(t, len(blocks), len(sentBlocks), "did not send all blocks") @@ -144,15 +121,15 @@ func TestMessageBuilding(t *testing.T) { }, expectedSize: 0, checkMsg: func(t *testing.T, message GraphSyncMessage) { - responses := message.Responses + responses := message.Responses() response1 := findResponseForRequestID(t, responses, requestID1) - require.Equal(t, graphsync.PartialResponse, response1.Status, "did not generate partial response") + require.Equal(t, graphsync.PartialResponse, response1.Status(), "did not generate partial response") assertMetadata(t, response1, nil) assertExtension(t, response1, extension1) response2 := findResponseForRequestID(t, responses, requestID2) - require.Equal(t, graphsync.PartialResponse, response2.Status, "did not generate partial response") + require.Equal(t, graphsync.PartialResponse, response2.Status(), "did not generate partial response") assertMetadata(t, response2, nil) assertExtension(t, response2, extension2) }, @@ -185,12 +162,12 @@ func TestMessageBuilding(t *testing.T) { expectedSize: 200, checkMsg: func(t *testing.T, message GraphSyncMessage) { - responses := message.Responses - sentBlocks := BlockFormatSlice(message.Blocks) + responses := message.Responses() + sentBlocks := message.Blocks() require.Len(t, responses, 3, "did not assemble correct number of responses") response2 := findResponseForRequestID(t, responses, requestID2) - require.Equal(t, graphsync.RequestCompletedFull, response2.Status, "did not generate completed full response") + require.Equal(t, graphsync.RequestCompletedFull, response2.Status(), "did not generate completed full response") assertMetadata(t, response2, metadata.Metadata{ metadata.Item{Link: links[1].(cidlink.Link).Cid, BlockPresent: true}, metadata.Item{Link: links[2].(cidlink.Link).Cid, BlockPresent: true}, @@ -198,14 +175,14 @@ func TestMessageBuilding(t *testing.T) { }) response3 := findResponseForRequestID(t, responses, requestID3) - require.Equal(t, graphsync.PartialResponse, response3.Status, "did not generate partial response") + require.Equal(t, graphsync.PartialResponse, response3.Status(), "did not generate partial response") assertMetadata(t, response3, metadata.Metadata{ metadata.Item{Link: links[1].(cidlink.Link).Cid, BlockPresent: true}, }) assertExtension(t, response3, extension2) response4 := findResponseForRequestID(t, responses, requestID4) - require.Equal(t, graphsync.RequestCompletedFull, response4.Status, "did not generate completed full response") + require.Equal(t, graphsync.RequestCompletedFull, response4.Status(), "did not generate completed full response") require.Equal(t, len(blocks)-1, len(sentBlocks), "did not send all blocks") @@ -243,12 +220,12 @@ func TestMessageBuilding(t *testing.T) { expectedSize: 100, checkMsg: func(t *testing.T, message GraphSyncMessage) { - responses := message.Responses - sentBlocks := BlockFormatSlice(message.Blocks) + responses := message.Responses() + sentBlocks := message.Blocks() require.Len(t, responses, 1, "did not assemble correct number of responses") response3 := findResponseForRequestID(t, responses, requestID3) - require.Equal(t, graphsync.PartialResponse, response3.Status, "did not generate partial response") + require.Equal(t, graphsync.PartialResponse, response3.Status(), "did not generate partial response") assertMetadata(t, response3, metadata.Metadata{ metadata.Item{Link: links[1].(cidlink.Link).Cid, BlockPresent: true}, }) @@ -274,7 +251,7 @@ func TestMessageBuilding(t *testing.T) { func findResponseForRequestID(t *testing.T, responses []GraphSyncResponse, requestID graphsync.RequestID) GraphSyncResponse { for _, response := range responses { - if response.ID == requestID { + if response.RequestID() == requestID { return response } } @@ -282,17 +259,15 @@ func findResponseForRequestID(t *testing.T, responses []GraphSyncResponse, reque return GraphSyncResponse{} } -func assertExtension(t *testing.T, response GraphSyncResponse, extension NamedExtension) { +func assertExtension(t *testing.T, response GraphSyncResponse, extension graphsync.ExtensionData) { returnedExtensionData, found := response.Extension(extension.Name) require.True(t, found) require.Equal(t, extension.Data, returnedExtensionData, "did not encode extension") } func assertMetadata(t *testing.T, response GraphSyncResponse, expectedMetadata metadata.Metadata) { - responseMetadataNode, found := response.Extension(graphsync.ExtensionMetadata) + responseMetadataRaw, found := response.Extension(graphsync.ExtensionMetadata) require.True(t, found, "Metadata should be included in response") - responseMetadataRaw, err := responseMetadataNode.AsBytes() - require.NoError(t, err) responseMetadata, err := metadata.DecodeMetadata(responseMetadataRaw) require.NoError(t, err) require.Equal(t, expectedMetadata, responseMetadata, "incorrect metadata included in response") diff --git a/message/message.go b/message/message.go index d1e7428b..1e02816a 100644 --- a/message/message.go +++ b/message/message.go @@ -5,13 +5,10 @@ import ( "errors" "fmt" "io" - "sort" blocks "github.com/ipfs/go-block-format" cid "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/datamodel" - "github.com/ipld/go-ipld-prime/node/basicnode" pool "github.com/libp2p/go-buffer-pool" "github.com/libp2p/go-libp2p-core/network" "github.com/libp2p/go-msgio" @@ -19,6 +16,7 @@ import ( "github.com/ipfs/go-graphsync" "github.com/ipfs/go-graphsync/ipldutil" + "github.com/ipfs/go-graphsync/message/ipldbind" pb "github.com/ipfs/go-graphsync/message/pb" ) @@ -49,81 +47,30 @@ type Exportable interface { ToNet(w io.Writer) error } -type GraphSyncExtensions struct { - Keys []string - Values map[string]datamodel.Node -} - // GraphSyncRequest is a struct to capture data on a request contained in a // GraphSyncMessage. type GraphSyncRequest struct { - ID graphsync.RequestID - - Root cid.Cid - Selector ipld.Node - Extensions GraphSyncExtensions - Priority graphsync.Priority - Cancel bool - Update bool -} - -type GraphSyncMetadatum struct { - Link datamodel.Link - BlockPresent bool + root cid.Cid + selector ipld.Node + priority graphsync.Priority + id graphsync.RequestID + extensions map[string][]byte + isCancel bool + isUpdate bool } // GraphSyncResponse is an struct to capture data on a response sent back // in a GraphSyncMessage. type GraphSyncResponse struct { - ID graphsync.RequestID - - Status graphsync.ResponseStatusCode - Metadata []GraphSyncMetadatum - Extensions GraphSyncExtensions -} - -type GraphSyncBlock struct { - Prefix []byte - Data []byte -} - -func FromBlockFormat(block blocks.Block) GraphSyncBlock { - return GraphSyncBlock{ - Prefix: block.Cid().Prefix().Bytes(), - Data: block.RawData(), - } -} - -func (b GraphSyncBlock) BlockFormat() *blocks.BasicBlock { - pref, err := cid.PrefixFromBytes(b.Prefix) - if err != nil { - panic(err) // should never happen - } - - c, err := pref.Sum(b.Data) - if err != nil { - panic(err) // should never happen - } - - block, err := blocks.NewBlockWithCid(b.Data, c) - if err != nil { - panic(err) // should never happen - } - return block -} - -func BlockFormatSlice(bs []GraphSyncBlock) []blocks.Block { - blks := make([]blocks.Block, len(bs)) - for i, b := range bs { - blks[i] = b.BlockFormat() - } - return blks + requestID graphsync.RequestID + status graphsync.ResponseStatusCode + extensions map[string][]byte } type GraphSyncMessage struct { - Requests []GraphSyncRequest - Responses []GraphSyncResponse - Blocks []GraphSyncBlock + requests map[graphsync.RequestID]GraphSyncRequest + responses map[graphsync.RequestID]GraphSyncResponse + blocks map[cid.Cid]blocks.Block } // NewRequest builds a new Graphsync request @@ -131,48 +78,29 @@ func NewRequest(id graphsync.RequestID, root cid.Cid, selector ipld.Node, priority graphsync.Priority, - extensions ...NamedExtension) GraphSyncRequest { + extensions ...graphsync.ExtensionData) GraphSyncRequest { return newRequest(id, root, selector, priority, false, false, toExtensionsMap(extensions)) } // CancelRequest request generates a request to cancel an in progress request func CancelRequest(id graphsync.RequestID) GraphSyncRequest { - return newRequest(id, cid.Cid{}, nil, 0, true, false, GraphSyncExtensions{}) + return newRequest(id, cid.Cid{}, nil, 0, true, false, nil) } // UpdateRequest generates a new request to update an in progress request with the given extensions -func UpdateRequest(id graphsync.RequestID, extensions ...NamedExtension) GraphSyncRequest { +func UpdateRequest(id graphsync.RequestID, extensions ...graphsync.ExtensionData) GraphSyncRequest { return newRequest(id, cid.Cid{}, nil, 0, false, true, toExtensionsMap(extensions)) } -// NamedExtension exists just for the purpose of the constructors. -type NamedExtension struct { - Name graphsync.ExtensionName - Data ipld.Node -} - -func toExtensionsMap(extensions []NamedExtension) (m GraphSyncExtensions) { +func toExtensionsMap(extensions []graphsync.ExtensionData) (extensionsMap map[string][]byte) { if len(extensions) > 0 { - m.Keys = make([]string, len(extensions)) - m.Values = make(map[string]ipld.Node, len(extensions)) - for i, ext := range extensions { - m.Keys[i] = string(ext.Name) - m.Values[string(ext.Name)] = ext.Data + extensionsMap = make(map[string][]byte, len(extensions)) + for _, extension := range extensions { + extensionsMap[string(extension.Name)] = extension.Data } } - return m -} - -func fromProtoExtensions(protoExts map[string][]byte) GraphSyncExtensions { - var exts []NamedExtension - for name, data := range protoExts { - exts = append(exts, NamedExtension{graphsync.ExtensionName(name), basicnode.NewBytes(data)}) - } - // Iterating over the map above is non-deterministic, - // so sort by the unique names to ensure determinism. - sort.Slice(exts, func(i, j int) bool { return exts[i].Name < exts[j].Name }) - return toExtensionsMap(exts) + return } func newRequest(id graphsync.RequestID, @@ -181,37 +109,37 @@ func newRequest(id graphsync.RequestID, priority graphsync.Priority, isCancel bool, isUpdate bool, - extensions GraphSyncExtensions) GraphSyncRequest { + extensions map[string][]byte) GraphSyncRequest { return GraphSyncRequest{ - ID: id, - Root: root, - Selector: selector, - Priority: priority, - Cancel: isCancel, - Update: isUpdate, - Extensions: extensions, + id: id, + root: root, + selector: selector, + priority: priority, + isCancel: isCancel, + isUpdate: isUpdate, + extensions: extensions, } } // NewResponse builds a new Graphsync response func NewResponse(requestID graphsync.RequestID, status graphsync.ResponseStatusCode, - extensions ...NamedExtension) GraphSyncResponse { + extensions ...graphsync.ExtensionData) GraphSyncResponse { return newResponse(requestID, status, toExtensionsMap(extensions)) } func newResponse(requestID graphsync.RequestID, - status graphsync.ResponseStatusCode, extensions GraphSyncExtensions) GraphSyncResponse { + status graphsync.ResponseStatusCode, extensions map[string][]byte) GraphSyncResponse { return GraphSyncResponse{ - ID: requestID, - Status: status, - Extensions: extensions, + requestID: requestID, + status: status, + extensions: extensions, } } func newMessageFromProto(pbm *pb.Message) (GraphSyncMessage, error) { - requests := make([]GraphSyncRequest, len(pbm.GetRequests())) - for i, req := range pbm.Requests { + requests := make(map[graphsync.RequestID]GraphSyncRequest, len(pbm.GetRequests())) + for _, req := range pbm.Requests { if req == nil { return GraphSyncMessage{}, errors.New("request is nil") } @@ -231,29 +159,47 @@ func newMessageFromProto(pbm *pb.Message) (GraphSyncMessage, error) { return GraphSyncMessage{}, err } } - // TODO: we likely need to turn some "core" extensions to fields, - // as some of those got moved to proper fields in the new protocol. - // Same for responses above, as well as the "to proto" funcs. - requests[i] = newRequest(graphsync.RequestID(req.Id), root, selector, graphsync.Priority(req.Priority), req.Cancel, req.Update, fromProtoExtensions(req.GetExtensions())) + exts := req.GetExtensions() + if exts == nil { + exts = make(map[string][]byte) + } + requests[graphsync.RequestID(req.Id)] = newRequest(graphsync.RequestID(req.Id), root, selector, graphsync.Priority(req.Priority), req.Cancel, req.Update, exts) } - responses := make([]GraphSyncResponse, len(pbm.GetResponses())) - for i, res := range pbm.Responses { + responses := make(map[graphsync.RequestID]GraphSyncResponse, len(pbm.GetResponses())) + for _, res := range pbm.Responses { if res == nil { return GraphSyncMessage{}, errors.New("response is nil") } - responses[i] = newResponse(graphsync.RequestID(res.Id), graphsync.ResponseStatusCode(res.Status), fromProtoExtensions(res.GetExtensions())) + exts := res.GetExtensions() + if exts == nil { + exts = make(map[string][]byte) + } + responses[graphsync.RequestID(res.Id)] = newResponse(graphsync.RequestID(res.Id), graphsync.ResponseStatusCode(res.Status), exts) } - blks := make([]GraphSyncBlock, len(pbm.GetData())) - for i, b := range pbm.GetData() { + blks := make(map[cid.Cid]blocks.Block, len(pbm.GetData())) + for _, b := range pbm.GetData() { if b == nil { return GraphSyncMessage{}, errors.New("block is nil") } - blks[i] = GraphSyncBlock{ - Prefix: b.GetPrefix(), - Data: b.GetData(), + + pref, err := cid.PrefixFromBytes(b.GetPrefix()) + if err != nil { + return GraphSyncMessage{}, err + } + + c, err := pref.Sum(b.GetData()) + if err != nil { + return GraphSyncMessage{}, err + } + + blk, err := blocks.NewBlockWithCid(b.GetData(), c) + if err != nil { + return GraphSyncMessage{}, err } + + blks[blk.Cid()] = blk } return GraphSyncMessage{ @@ -262,17 +208,41 @@ func newMessageFromProto(pbm *pb.Message) (GraphSyncMessage, error) { } func (gsm GraphSyncMessage) Empty() bool { - return len(gsm.Blocks) == 0 && len(gsm.Requests) == 0 && len(gsm.Responses) == 0 + return len(gsm.blocks) == 0 && len(gsm.requests) == 0 && len(gsm.responses) == 0 +} + +func (gsm GraphSyncMessage) Requests() []GraphSyncRequest { + requests := make([]GraphSyncRequest, 0, len(gsm.requests)) + for _, request := range gsm.requests { + requests = append(requests, request) + } + return requests } func (gsm GraphSyncMessage) ResponseCodes() map[graphsync.RequestID]graphsync.ResponseStatusCode { - codes := make(map[graphsync.RequestID]graphsync.ResponseStatusCode, len(gsm.Responses)) - for _, response := range gsm.Responses { - codes[response.ID] = response.Status + codes := make(map[graphsync.RequestID]graphsync.ResponseStatusCode, len(gsm.responses)) + for id, response := range gsm.responses { + codes[id] = response.Status() } return codes } +func (gsm GraphSyncMessage) Responses() []GraphSyncResponse { + responses := make([]GraphSyncResponse, 0, len(gsm.responses)) + for _, response := range gsm.responses { + responses = append(responses, response) + } + return responses +} + +func (gsm GraphSyncMessage) Blocks() []blocks.Block { + bs := make([]blocks.Block, 0, len(gsm.blocks)) + for _, block := range gsm.blocks { + bs = append(bs, block) + } + return bs +} + // FromNet can read a network stream to deserialized a GraphSyncMessage func FromNet(r io.Reader) (GraphSyncMessage, error) { reader := msgio.NewVarintReaderSize(r, network.MessageSizeMax) @@ -296,65 +266,152 @@ func FromMsgReader(r msgio.Reader) (GraphSyncMessage, error) { return newMessageFromProto(&pb) } -func toProtoExtensions(m GraphSyncExtensions) map[string][]byte { - protoExts := make(map[string][]byte, len(m.Values)) - for name, node := range m.Values { - // Only keep those which are plain bytes, - // as those are the only ones that the older protocol clients understand. - if node.Kind() != ipld.Kind_Bytes { - continue - } - raw, err := node.AsBytes() - if err != nil { - panic(err) // shouldn't happen - } - protoExts[name] = raw - } - return protoExts -} - func (gsm GraphSyncMessage) ToProto() (*pb.Message, error) { pbm := new(pb.Message) - pbm.Requests = make([]*pb.Message_Request, 0, len(gsm.Requests)) - for _, request := range gsm.Requests { + pbm.Requests = make([]*pb.Message_Request, 0, len(gsm.requests)) + for _, request := range gsm.requests { var selector []byte var err error - if request.Selector != nil { - selector, err = ipldutil.EncodeNode(request.Selector) + if request.selector != nil { + selector, err = ipldutil.EncodeNode(request.selector) if err != nil { return nil, err } } pbm.Requests = append(pbm.Requests, &pb.Message_Request{ - Id: int32(request.ID), - Root: request.Root.Bytes(), + Id: int32(request.id), + Root: request.root.Bytes(), Selector: selector, - Priority: int32(request.Priority), - Cancel: request.Cancel, - Update: request.Update, - Extensions: toProtoExtensions(request.Extensions), + Priority: int32(request.priority), + Cancel: request.isCancel, + Update: request.isUpdate, + Extensions: request.extensions, }) } - pbm.Responses = make([]*pb.Message_Response, 0, len(gsm.Responses)) - for _, response := range gsm.Responses { + pbm.Responses = make([]*pb.Message_Response, 0, len(gsm.responses)) + for _, response := range gsm.responses { pbm.Responses = append(pbm.Responses, &pb.Message_Response{ - Id: int32(response.ID), - Status: int32(response.Status), - Extensions: toProtoExtensions(response.Extensions), + Id: int32(response.requestID), + Status: int32(response.status), + Extensions: response.extensions, }) } - pbm.Data = make([]*pb.Message_Block, 0, len(gsm.Blocks)) - for _, b := range gsm.Blocks { + blocks := gsm.Blocks() + pbm.Data = make([]*pb.Message_Block, 0, len(blocks)) + for _, b := range blocks { pbm.Data = append(pbm.Data, &pb.Message_Block{ - Prefix: b.Prefix, - Data: b.Data, + Data: b.RawData(), + Prefix: b.Cid().Prefix().Bytes(), }) } return pbm, nil } +func (gsm GraphSyncMessage) ToIPLD() (*ipldbind.GraphSyncMessage, error) { + ibm := new(ipldbind.GraphSyncMessage) + ibm.Requests = make([]ipldbind.GraphSyncRequest, 0, len(gsm.requests)) + for _, request := range gsm.requests { + ibm.Requests = append(ibm.Requests, ipldbind.GraphSyncRequest{ + ID: request.id, + Root: request.root, + Selector: request.selector, + Priority: request.priority, + Cancel: request.isCancel, + Update: request.isUpdate, + // Extensions: request.extensions, + }) + } + + ibm.Responses = make([]ipldbind.GraphSyncResponse, 0, len(gsm.responses)) + for _, response := range gsm.responses { + ibm.Responses = append(ibm.Responses, ipldbind.GraphSyncResponse{ + ID: response.requestID, + Status: response.status, + // Extensions: response.extensions, + }) + } + + blocks := gsm.Blocks() + ibm.Blocks = make([]ipldbind.GraphSyncBlock, 0, len(blocks)) + for _, b := range blocks { + ibm.Blocks = append(ibm.Blocks, ipldbind.GraphSyncBlock{ + Data: b.RawData(), + Prefix: b.Cid().Prefix().Bytes(), + }) + } + return ibm, nil +} + +func messageFromIPLD(ibm *ipldbind.GraphSyncMessage) (GraphSyncMessage, error) { + requests := make(map[graphsync.RequestID]GraphSyncRequest, len(ibm.Requests)) + for _, req := range ibm.Requests { + // exts := req.Extensions + requests[graphsync.RequestID(req.ID)] = newRequest(graphsync.RequestID(req.ID), req.Root, req.Selector, graphsync.Priority(req.Priority), req.Cancel, req.Update, nil) + } + + responses := make(map[graphsync.RequestID]GraphSyncResponse, len(ibm.Responses)) + for _, res := range ibm.Responses { + // exts := res.Extensions + responses[graphsync.RequestID(res.ID)] = newResponse(graphsync.RequestID(res.ID), graphsync.ResponseStatusCode(res.Status), nil) + } + + blks := make(map[cid.Cid]blocks.Block, len(ibm.Blocks)) + for _, b := range ibm.Blocks { + pref, err := cid.PrefixFromBytes(b.Prefix) + if err != nil { + return GraphSyncMessage{}, err + } + + c, err := pref.Sum(b.Data) + if err != nil { + return GraphSyncMessage{}, err + } + + blk, err := blocks.NewBlockWithCid(b.Data, c) + if err != nil { + return GraphSyncMessage{}, err + } + + blks[blk.Cid()] = blk + } + + return GraphSyncMessage{ + requests, responses, blks, + }, nil +} + +/* +func fromProtoExtensions(protoExts map[string][]byte) GraphSyncExtensions { + var exts []NamedExtension + for name, data := range protoExts { + exts = append(exts, NamedExtension{graphsync.ExtensionName(name), basicnode.NewBytes(data)}) + } + // Iterating over the map above is non-deterministic, + // so sort by the unique names to ensure determinism. + sort.Slice(exts, func(i, j int) bool { return exts[i].Name < exts[j].Name }) + return toExtensionsMap(exts) +} + +func toProtoExtensions(m GraphSyncExtensions) map[string][]byte { + protoExts := make(map[string][]byte, len(m.Values)) + for name, node := range m.Values { + // Only keep those which are plain bytes, + // as those are the only ones that the older protocol clients understand. + if node.Kind() != ipld.Kind_Bytes { + continue + } + raw, err := node.AsBytes() + if err != nil { + panic(err) // shouldn't happen + } + protoExts[name] = raw + } + return protoExts +} +*/ + func (gsm GraphSyncMessage) ToNet(w io.Writer) error { msg, err := gsm.ToProto() if err != nil { @@ -375,13 +432,13 @@ func (gsm GraphSyncMessage) ToNet(w io.Writer) error { } func (gsm GraphSyncMessage) Loggable() map[string]interface{} { - requests := make([]string, 0, len(gsm.Requests)) - for _, request := range gsm.Requests { - requests = append(requests, fmt.Sprintf("%d", request.ID)) + requests := make([]string, 0, len(gsm.requests)) + for _, request := range gsm.requests { + requests = append(requests, fmt.Sprintf("%d", request.id)) } - responses := make([]string, 0, len(gsm.Responses)) - for _, response := range gsm.Responses { - responses = append(responses, fmt.Sprintf("%d", response.ID)) + responses := make([]string, 0, len(gsm.responses)) + for _, response := range gsm.responses { + responses = append(responses, fmt.Sprintf("%d", response.requestID)) } return map[string]interface{}{ "requests": requests, @@ -390,16 +447,40 @@ func (gsm GraphSyncMessage) Loggable() map[string]interface{} { } func (gsm GraphSyncMessage) Clone() GraphSyncMessage { - requests := append([]GraphSyncRequest{}, gsm.Requests...) - responses := append([]GraphSyncResponse{}, gsm.Responses...) - blocks := append([]GraphSyncBlock{}, gsm.Blocks...) + requests := make(map[graphsync.RequestID]GraphSyncRequest, len(gsm.requests)) + for id, request := range gsm.requests { + requests[id] = request + } + responses := make(map[graphsync.RequestID]GraphSyncResponse, len(gsm.responses)) + for id, response := range gsm.responses { + responses[id] = response + } + blocks := make(map[cid.Cid]blocks.Block, len(gsm.blocks)) + for cid, block := range gsm.blocks { + blocks[cid] = block + } return GraphSyncMessage{requests, responses, blocks} } +// ID Returns the request ID for this Request +func (gsr GraphSyncRequest) ID() graphsync.RequestID { return gsr.id } + +// Root returns the CID to the root block of this request +func (gsr GraphSyncRequest) Root() cid.Cid { return gsr.root } + +// Selector returns the byte representation of the selector for this request +func (gsr GraphSyncRequest) Selector() ipld.Node { return gsr.selector } + +// Priority returns the priority of this request +func (gsr GraphSyncRequest) Priority() graphsync.Priority { return gsr.priority } + // Extension returns the content for an extension on a response, or errors // if extension is not present -func (gsr GraphSyncRequest) Extension(name graphsync.ExtensionName) (ipld.Node, bool) { - val, ok := gsr.Extensions.Values[string(name)] +func (gsr GraphSyncRequest) Extension(name graphsync.ExtensionName) ([]byte, bool) { + if gsr.extensions == nil { + return nil, false + } + val, ok := gsr.extensions[string(name)] if !ok { return nil, false } @@ -408,13 +489,32 @@ func (gsr GraphSyncRequest) Extension(name graphsync.ExtensionName) (ipld.Node, // ExtensionNames returns the names of the extensions included in this request func (gsr GraphSyncRequest) ExtensionNames() []string { - return gsr.Extensions.Keys + var extNames []string + for ext := range gsr.extensions { + extNames = append(extNames, ext) + } + return extNames } +// IsCancel returns true if this particular request is being cancelled +func (gsr GraphSyncRequest) IsCancel() bool { return gsr.isCancel } + +// IsUpdate returns true if this particular request is being updated +func (gsr GraphSyncRequest) IsUpdate() bool { return gsr.isUpdate } + +// RequestID returns the request ID for this response +func (gsr GraphSyncResponse) RequestID() graphsync.RequestID { return gsr.requestID } + +// Status returns the status for a response +func (gsr GraphSyncResponse) Status() graphsync.ResponseStatusCode { return gsr.status } + // Extension returns the content for an extension on a response, or errors // if extension is not present -func (gsr GraphSyncResponse) Extension(name graphsync.ExtensionName) (ipld.Node, bool) { - val, ok := gsr.Extensions.Values[string(name)] +func (gsr GraphSyncResponse) Extension(name graphsync.ExtensionName) ([]byte, bool) { + if gsr.extensions == nil { + return nil, false + } + val, ok := gsr.extensions[string(name)] if !ok { return nil, false } @@ -423,14 +523,18 @@ func (gsr GraphSyncResponse) Extension(name graphsync.ExtensionName) (ipld.Node, // ExtensionNames returns the names of the extensions included in this request func (gsr GraphSyncResponse) ExtensionNames() []string { - return gsr.Extensions.Keys + var extNames []string + for ext := range gsr.extensions { + extNames = append(extNames, ext) + } + return extNames } // ReplaceExtensions merges the extensions given extensions into the request to create a new request, // but always uses new data -func (gsr GraphSyncRequest) ReplaceExtensions(extensions []NamedExtension) GraphSyncRequest { - req, _ := gsr.MergeExtensions(extensions, func(name graphsync.ExtensionName, oldNode, newNode ipld.Node) (ipld.Node, error) { - return newNode, nil +func (gsr GraphSyncRequest) ReplaceExtensions(extensions []graphsync.ExtensionData) GraphSyncRequest { + req, _ := gsr.MergeExtensions(extensions, func(name graphsync.ExtensionName, oldData []byte, newData []byte) ([]byte, error) { + return newData, nil }) return req } @@ -438,32 +542,31 @@ func (gsr GraphSyncRequest) ReplaceExtensions(extensions []NamedExtension) Graph // MergeExtensions merges the given list of extensions to produce a new request with the combination of the old request // plus the new extensions. When an old extension and a new extension are both present, mergeFunc is called to produce // the result -func (gsr GraphSyncRequest) MergeExtensions(extensions []NamedExtension, mergeFunc func(name graphsync.ExtensionName, oldNode, newNode ipld.Node) (ipld.Node, error)) (GraphSyncRequest, error) { - if len(gsr.Extensions.Keys) == 0 { - return newRequest(gsr.ID, gsr.Root, gsr.Selector, gsr.Priority, gsr.Cancel, gsr.Update, toExtensionsMap(extensions)), nil +func (gsr GraphSyncRequest) MergeExtensions(extensions []graphsync.ExtensionData, mergeFunc func(name graphsync.ExtensionName, oldData []byte, newData []byte) ([]byte, error)) (GraphSyncRequest, error) { + if gsr.extensions == nil { + return newRequest(gsr.id, gsr.root, gsr.selector, gsr.priority, gsr.isCancel, gsr.isUpdate, toExtensionsMap(extensions)), nil } - combinedExtensions := make(map[string]ipld.Node) - for _, newExt := range extensions { - oldNode, ok := gsr.Extensions.Values[string(newExt.Name)] + newExtensionMap := toExtensionsMap(extensions) + combinedExtensions := make(map[string][]byte) + for name, newData := range newExtensionMap { + oldData, ok := gsr.extensions[name] if !ok { - combinedExtensions[string(newExt.Name)] = newExt.Data + combinedExtensions[name] = newData continue } - resultNode, err := mergeFunc(graphsync.ExtensionName(newExt.Name), oldNode, newExt.Data) + resultData, err := mergeFunc(graphsync.ExtensionName(name), oldData, newData) if err != nil { return GraphSyncRequest{}, err } - combinedExtensions[string(newExt.Name)] = resultNode + combinedExtensions[name] = resultData } - for name, oldNode := range gsr.Extensions.Values { + for name, oldData := range gsr.extensions { _, ok := combinedExtensions[name] if ok { continue } - combinedExtensions[name] = oldNode + combinedExtensions[name] = oldData } - extNames := make([]string, len(combinedExtensions)) - sort.Strings(extNames) // for reproducibility - return newRequest(gsr.ID, gsr.Root, gsr.Selector, gsr.Priority, gsr.Cancel, gsr.Update, GraphSyncExtensions{extNames, combinedExtensions}), nil + return newRequest(gsr.id, gsr.root, gsr.selector, gsr.priority, gsr.isCancel, gsr.isUpdate, combinedExtensions), nil } diff --git a/message/message_test.go b/message/message_test.go index 052950f9..135342d3 100644 --- a/message/message_test.go +++ b/message/message_test.go @@ -8,7 +8,6 @@ import ( blocks "github.com/ipfs/go-block-format" cid "github.com/ipfs/go-cid" - "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/node/basicnode" "github.com/ipld/go-ipld-prime/traversal/selector/builder" "github.com/stretchr/testify/require" @@ -20,10 +19,9 @@ import ( func TestAppendingRequests(t *testing.T) { extensionName := graphsync.ExtensionName("graphsync/awesome") - extensionBytes := testutil.RandomBytes(100) - extension := NamedExtension{ + extension := graphsync.ExtensionData{ Name: extensionName, - Data: basicnode.NewBytes(extensionBytes), + Data: testutil.RandomBytes(100), } root := testutil.GenerateCids(1)[0] ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) @@ -35,15 +33,15 @@ func TestAppendingRequests(t *testing.T) { builder.AddRequest(NewRequest(id, root, selector, priority, extension)) gsm, err := builder.Build() require.NoError(t, err) - requests := gsm.Requests + requests := gsm.Requests() require.Len(t, requests, 1, "did not add request to message") request := requests[0] extensionData, found := request.Extension(extensionName) - require.Equal(t, id, request.ID) - require.False(t, request.Cancel) - require.Equal(t, priority, request.Priority) - require.Equal(t, root.String(), request.Root.String()) - require.Equal(t, selector, request.Selector) + require.Equal(t, id, request.ID()) + require.False(t, request.IsCancel()) + require.Equal(t, priority, request.Priority()) + require.Equal(t, root.String(), request.Root().String()) + require.Equal(t, selector, request.Selector()) require.True(t, found) require.Equal(t, extension.Data, extensionData) @@ -59,31 +57,30 @@ func TestAppendingRequests(t *testing.T) { require.False(t, pbRequest.Update) require.Equal(t, root.Bytes(), pbRequest.Root) require.Equal(t, selectorEncoded, pbRequest.Selector) - require.Equal(t, map[string][]byte{"graphsync/awesome": extensionBytes}, pbRequest.Extensions) + require.Equal(t, map[string][]byte{"graphsync/awesome": extension.Data}, pbRequest.Extensions) deserialized, err := newMessageFromProto(pbMessage) require.NoError(t, err, "deserializing protobuf message errored") - deserializedRequests := deserialized.Requests + deserializedRequests := deserialized.Requests() require.Len(t, deserializedRequests, 1, "did not add request to deserialized message") deserializedRequest := deserializedRequests[0] extensionData, found = deserializedRequest.Extension(extensionName) - require.Equal(t, id, deserializedRequest.ID) - require.False(t, deserializedRequest.Cancel) - require.False(t, deserializedRequest.Update) - require.Equal(t, priority, deserializedRequest.Priority) - require.Equal(t, root.String(), deserializedRequest.Root.String()) - require.Equal(t, selector, deserializedRequest.Selector) + require.Equal(t, id, deserializedRequest.ID()) + require.False(t, deserializedRequest.IsCancel()) + require.False(t, deserializedRequest.IsUpdate()) + require.Equal(t, priority, deserializedRequest.Priority()) + require.Equal(t, root.String(), deserializedRequest.Root().String()) + require.Equal(t, selector, deserializedRequest.Selector()) require.True(t, found) require.Equal(t, extension.Data, extensionData) } func TestAppendingResponses(t *testing.T) { extensionName := graphsync.ExtensionName("graphsync/awesome") - extensionBytes := testutil.RandomBytes(100) - extension := NamedExtension{ + extension := graphsync.ExtensionData{ Name: extensionName, - Data: basicnode.NewBytes(extensionBytes), + Data: testutil.RandomBytes(100), } requestID := graphsync.RequestID(rand.Int31()) status := graphsync.RequestAcknowledged @@ -93,12 +90,12 @@ func TestAppendingResponses(t *testing.T) { builder.AddExtensionData(requestID, extension) gsm, err := builder.Build() require.NoError(t, err) - responses := gsm.Responses + responses := gsm.Responses() require.Len(t, responses, 1, "did not add response to message") response := responses[0] extensionData, found := response.Extension(extensionName) - require.Equal(t, requestID, response.ID) - require.Equal(t, status, response.Status) + require.Equal(t, requestID, response.RequestID()) + require.Equal(t, status, response.Status()) require.True(t, found) require.Equal(t, extension.Data, extensionData) @@ -107,16 +104,16 @@ func TestAppendingResponses(t *testing.T) { pbResponse := pbMessage.Responses[0] require.Equal(t, int32(requestID), pbResponse.Id) require.Equal(t, int32(status), pbResponse.Status) - require.Equal(t, extensionBytes, pbResponse.Extensions["graphsync/awesome"]) + require.Equal(t, extension.Data, pbResponse.Extensions["graphsync/awesome"]) deserialized, err := newMessageFromProto(pbMessage) require.NoError(t, err, "deserializing protobuf message errored") - deserializedResponses := deserialized.Responses + deserializedResponses := deserialized.Responses() require.Len(t, deserializedResponses, 1, "did not add response to deserialized message") deserializedResponse := deserializedResponses[0] extensionData, found = deserializedResponse.Extension(extensionName) - require.Equal(t, response.ID, deserializedResponse.ID) - require.Equal(t, response.Status, deserializedResponse.Status) + require.Equal(t, response.RequestID(), deserializedResponse.RequestID()) + require.Equal(t, response.Status(), deserializedResponse.Status()) require.True(t, found) require.Equal(t, extension.Data, extensionData) } @@ -167,31 +164,31 @@ func TestRequestCancel(t *testing.T) { gsm, err := builder.Build() require.NoError(t, err) - requests := gsm.Requests + requests := gsm.Requests() require.Len(t, requests, 1, "did not add cancel request") request := requests[0] - require.Equal(t, id, request.ID) - require.True(t, request.Cancel) + require.Equal(t, id, request.ID()) + require.True(t, request.IsCancel()) buf := new(bytes.Buffer) err = gsm.ToNet(buf) require.NoError(t, err, "did not serialize protobuf message") deserialized, err := FromNet(buf) require.NoError(t, err, "did not deserialize protobuf message") - deserializedRequests := deserialized.Requests + deserializedRequests := deserialized.Requests() require.Len(t, deserializedRequests, 1, "did not add request to deserialized message") deserializedRequest := deserializedRequests[0] - require.Equal(t, request.ID, deserializedRequest.ID) - require.Equal(t, request.Cancel, deserializedRequest.Cancel) + require.Equal(t, request.ID(), deserializedRequest.ID()) + require.Equal(t, request.IsCancel(), deserializedRequest.IsCancel()) } func TestRequestUpdate(t *testing.T) { id := graphsync.RequestID(rand.Int31()) extensionName := graphsync.ExtensionName("graphsync/awesome") - extension := NamedExtension{ + extension := graphsync.ExtensionData{ Name: extensionName, - Data: basicnode.NewBytes(testutil.RandomBytes(100)), + Data: testutil.RandomBytes(100), } builder := NewBuilder() @@ -199,12 +196,12 @@ func TestRequestUpdate(t *testing.T) { gsm, err := builder.Build() require.NoError(t, err) - requests := gsm.Requests + requests := gsm.Requests() require.Len(t, requests, 1, "did not add cancel request") request := requests[0] - require.Equal(t, id, request.ID) - require.True(t, request.Update) - require.False(t, request.Cancel) + require.Equal(t, id, request.ID()) + require.True(t, request.IsUpdate()) + require.False(t, request.IsCancel()) extensionData, found := request.Extension(extensionName) require.True(t, found) require.Equal(t, extension.Data, extensionData) @@ -215,16 +212,16 @@ func TestRequestUpdate(t *testing.T) { deserialized, err := FromNet(buf) require.NoError(t, err, "did not deserialize protobuf message") - deserializedRequests := deserialized.Requests + deserializedRequests := deserialized.Requests() require.Len(t, deserializedRequests, 1, "did not add request to deserialized message") deserializedRequest := deserializedRequests[0] extensionData, found = deserializedRequest.Extension(extensionName) - require.Equal(t, request.ID, deserializedRequest.ID) - require.Equal(t, request.Cancel, deserializedRequest.Cancel) - require.Equal(t, request.Update, deserializedRequest.Update) - require.Equal(t, request.Priority, deserializedRequest.Priority) - require.Equal(t, request.Root.String(), deserializedRequest.Root.String()) - require.Equal(t, request.Selector, deserializedRequest.Selector) + require.Equal(t, request.ID(), deserializedRequest.ID()) + require.Equal(t, request.IsCancel(), deserializedRequest.IsCancel()) + require.Equal(t, request.IsUpdate(), deserializedRequest.IsUpdate()) + require.Equal(t, request.Priority(), deserializedRequest.Priority()) + require.Equal(t, request.Root().String(), deserializedRequest.Root().String()) + require.Equal(t, request.Selector(), deserializedRequest.Selector()) require.True(t, found) require.Equal(t, extension.Data, extensionData) } @@ -234,9 +231,9 @@ func TestToNetFromNetEquivalency(t *testing.T) { ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) selector := ssb.Matcher().Node() extensionName := graphsync.ExtensionName("graphsync/awesome") - extension := NamedExtension{ + extension := graphsync.ExtensionData{ Name: extensionName, - Data: basicnode.NewBytes(testutil.RandomBytes(100)), + Data: testutil.RandomBytes(100), } id := graphsync.RequestID(rand.Int31()) priority := graphsync.Priority(rand.Int31()) @@ -259,41 +256,41 @@ func TestToNetFromNetEquivalency(t *testing.T) { deserialized, err := FromNet(buf) require.NoError(t, err, "did not deserialize protobuf message") - requests := gsm.Requests + requests := gsm.Requests() require.Len(t, requests, 1, "did not add request to message") request := requests[0] - deserializedRequests := deserialized.Requests + deserializedRequests := deserialized.Requests() require.Len(t, deserializedRequests, 1, "did not add request to deserialized message") deserializedRequest := deserializedRequests[0] extensionData, found := deserializedRequest.Extension(extensionName) - require.Equal(t, request.ID, deserializedRequest.ID) - require.False(t, deserializedRequest.Cancel) - require.False(t, deserializedRequest.Update) - require.Equal(t, request.Priority, deserializedRequest.Priority) - require.Equal(t, request.Root.String(), deserializedRequest.Root.String()) - require.Equal(t, request.Selector, deserializedRequest.Selector) + require.Equal(t, request.ID(), deserializedRequest.ID()) + require.False(t, deserializedRequest.IsCancel()) + require.False(t, deserializedRequest.IsUpdate()) + require.Equal(t, request.Priority(), deserializedRequest.Priority()) + require.Equal(t, request.Root().String(), deserializedRequest.Root().String()) + require.Equal(t, request.Selector(), deserializedRequest.Selector()) require.True(t, found) require.Equal(t, extension.Data, extensionData) - responses := gsm.Responses + responses := gsm.Responses() require.Len(t, responses, 1, "did not add response to message") response := responses[0] - deserializedResponses := deserialized.Responses + deserializedResponses := deserialized.Responses() require.Len(t, deserializedResponses, 1, "did not add response to message") deserializedResponse := deserializedResponses[0] extensionData, found = deserializedResponse.Extension(extensionName) - require.Equal(t, response.ID, deserializedResponse.ID) - require.Equal(t, response.Status, deserializedResponse.Status) + require.Equal(t, response.RequestID(), deserializedResponse.RequestID()) + require.Equal(t, response.Status(), deserializedResponse.Status()) require.True(t, found) require.Equal(t, extension.Data, extensionData) keys := make(map[cid.Cid]bool) - for _, b := range deserialized.Blocks { - keys[b.BlockFormat().Cid()] = true + for _, b := range deserialized.Blocks() { + keys[b.Cid()] = true } - for _, b := range gsm.Blocks { - _, ok := keys[b.BlockFormat().Cid()] + for _, b := range gsm.Blocks() { + _, ok := keys[b.Cid()] require.True(t, ok) } } @@ -302,32 +299,28 @@ func TestMergeExtensions(t *testing.T) { extensionName1 := graphsync.ExtensionName("graphsync/1") extensionName2 := graphsync.ExtensionName("graphsync/2") extensionName3 := graphsync.ExtensionName("graphsync/3") - initialExtensions := []NamedExtension{ + initialExtensions := []graphsync.ExtensionData{ { Name: extensionName1, - Data: basicnode.NewBytes([]byte("applesauce")), + Data: []byte("applesauce"), }, { Name: extensionName2, - Data: basicnode.NewBytes([]byte("hello")), + Data: []byte("hello"), }, } - replacementExtensions := []NamedExtension{ + replacementExtensions := []graphsync.ExtensionData{ { Name: extensionName2, - Data: basicnode.NewBytes([]byte("world")), + Data: []byte("world"), }, { Name: extensionName3, - Data: basicnode.NewBytes([]byte("cheese")), + Data: []byte("cheese"), }, } - defaultMergeFunc := func(name graphsync.ExtensionName, oldNode, newNode ipld.Node) (ipld.Node, error) { - oldData, err := oldNode.AsBytes() - require.NoError(t, err) - newData, err := newNode.AsBytes() - require.NoError(t, err) - return basicnode.NewBytes([]byte(string(oldData) + " " + string(newData))), nil + defaultMergeFunc := func(name graphsync.ExtensionName, oldData []byte, newData []byte) ([]byte, error) { + return []byte(string(oldData) + " " + string(newData)), nil } root := testutil.GenerateCids(1)[0] ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) @@ -339,38 +332,38 @@ func TestMergeExtensions(t *testing.T) { emptyRequest := NewRequest(id, root, selector, priority) resultRequest, err := emptyRequest.MergeExtensions(replacementExtensions, defaultMergeFunc) require.NoError(t, err) - require.Equal(t, emptyRequest.ID, resultRequest.ID) - require.Equal(t, emptyRequest.Priority, resultRequest.Priority) - require.Equal(t, emptyRequest.Root.String(), resultRequest.Root.String()) - require.Equal(t, emptyRequest.Selector, resultRequest.Selector) + require.Equal(t, emptyRequest.ID(), resultRequest.ID()) + require.Equal(t, emptyRequest.Priority(), resultRequest.Priority()) + require.Equal(t, emptyRequest.Root().String(), resultRequest.Root().String()) + require.Equal(t, emptyRequest.Selector(), resultRequest.Selector()) _, has := resultRequest.Extension(extensionName1) require.False(t, has) extData2, has := resultRequest.Extension(extensionName2) require.True(t, has) - require.Equal(t, basicnode.NewBytes([]byte("world")), extData2) + require.Equal(t, []byte("world"), extData2) extData3, has := resultRequest.Extension(extensionName3) require.True(t, has) - require.Equal(t, basicnode.NewBytes([]byte("cheese")), extData3) + require.Equal(t, []byte("cheese"), extData3) }) t.Run("when merging two requests", func(t *testing.T) { resultRequest, err := defaultRequest.MergeExtensions(replacementExtensions, defaultMergeFunc) require.NoError(t, err) - require.Equal(t, defaultRequest.ID, resultRequest.ID) - require.Equal(t, defaultRequest.Priority, resultRequest.Priority) - require.Equal(t, defaultRequest.Root.String(), resultRequest.Root.String()) - require.Equal(t, defaultRequest.Selector, resultRequest.Selector) + require.Equal(t, defaultRequest.ID(), resultRequest.ID()) + require.Equal(t, defaultRequest.Priority(), resultRequest.Priority()) + require.Equal(t, defaultRequest.Root().String(), resultRequest.Root().String()) + require.Equal(t, defaultRequest.Selector(), resultRequest.Selector()) extData1, has := resultRequest.Extension(extensionName1) require.True(t, has) - require.Equal(t, basicnode.NewBytes([]byte("applesauce")), extData1) + require.Equal(t, []byte("applesauce"), extData1) extData2, has := resultRequest.Extension(extensionName2) require.True(t, has) - require.Equal(t, basicnode.NewBytes([]byte("hello world")), extData2) + require.Equal(t, []byte("hello world"), extData2) extData3, has := resultRequest.Extension(extensionName3) require.True(t, has) - require.Equal(t, basicnode.NewBytes([]byte("cheese")), extData3) + require.Equal(t, []byte("cheese"), extData3) }) t.Run("when merging errors", func(t *testing.T) { - errorMergeFunc := func(name graphsync.ExtensionName, oldNode, newNode ipld.Node) (ipld.Node, error) { + errorMergeFunc := func(name graphsync.ExtensionName, oldData []byte, newData []byte) ([]byte, error) { return nil, errors.New("something went wrong") } _, err := defaultRequest.MergeExtensions(replacementExtensions, errorMergeFunc) @@ -378,19 +371,19 @@ func TestMergeExtensions(t *testing.T) { }) t.Run("when merging with replace", func(t *testing.T) { resultRequest := defaultRequest.ReplaceExtensions(replacementExtensions) - require.Equal(t, defaultRequest.ID, resultRequest.ID) - require.Equal(t, defaultRequest.Priority, resultRequest.Priority) - require.Equal(t, defaultRequest.Root.String(), resultRequest.Root.String()) - require.Equal(t, defaultRequest.Selector, resultRequest.Selector) + require.Equal(t, defaultRequest.ID(), resultRequest.ID()) + require.Equal(t, defaultRequest.Priority(), resultRequest.Priority()) + require.Equal(t, defaultRequest.Root().String(), resultRequest.Root().String()) + require.Equal(t, defaultRequest.Selector(), resultRequest.Selector()) extData1, has := resultRequest.Extension(extensionName1) require.True(t, has) - require.Equal(t, basicnode.NewBytes([]byte("applesauce")), extData1) + require.Equal(t, []byte("applesauce"), extData1) extData2, has := resultRequest.Extension(extensionName2) require.True(t, has) - require.Equal(t, basicnode.NewBytes([]byte("world")), extData2) + require.Equal(t, []byte("world"), extData2) extData3, has := resultRequest.Extension(extensionName3) require.True(t, has) - require.Equal(t, basicnode.NewBytes([]byte("cheese")), extData3) + require.Equal(t, []byte("cheese"), extData3) }) } From 16475996f9f460ec5f161961ec86e536338f292e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Thu, 13 Jan 2022 15:50:17 +0000 Subject: [PATCH 19/21] wrap up the deep comparisons --- go.mod | 1 + go.sum | 6 ------ message/bench_test.go | 30 +++++++++++++++++++----------- message/message.go | 6 ------ 4 files changed, 20 insertions(+), 23 deletions(-) diff --git a/go.mod b/go.mod index 5bce5b54..3c62de84 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/ipfs/go-graphsync go 1.16 require ( + github.com/google/go-cmp v0.5.6 github.com/hannahhoward/cbor-gen-for v0.0.0-20200817222906-ea96cece81f1 github.com/hannahhoward/go-pubsub v0.0.0-20200423002714-8d62886cc36e github.com/ipfs/go-block-format v0.0.3 diff --git a/go.sum b/go.sum index 7c422042..0794f5aa 100644 --- a/go.sum +++ b/go.sum @@ -180,7 +180,6 @@ github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJn github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= -github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -455,10 +454,6 @@ github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2 github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= github.com/ipld/go-codec-dagpb v1.3.0 h1:czTcaoAuNNyIYWs6Qe01DJ+sEX7B+1Z0LcXjSatMGe8= github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA= -github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= -github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= -github.com/ipld/go-ipld-prime v0.14.4-0.20220110161855-fc09d6b768e9 h1:fqQSvdPznhyE5jZoCwbvykHbK+QukxJcS1SFWbj9ig0= -github.com/ipld/go-ipld-prime v0.14.4-0.20220110161855-fc09d6b768e9/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= @@ -952,7 +947,6 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polydawn/refmt v0.0.0-20190408063855-01bf1e26dd14/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= -github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e h1:ZOcivgkkFRnjfoTcGsDq3UQYiBmekwLA+qg0OjyB/ls= github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= diff --git a/message/bench_test.go b/message/bench_test.go index 970f3a22..1b346e67 100644 --- a/message/bench_test.go +++ b/message/bench_test.go @@ -3,8 +3,10 @@ package message import ( "bytes" "math/rand" + "reflect" "testing" + "github.com/google/go-cmp/cmp" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-graphsync" "github.com/ipfs/go-graphsync/message/ipldbind" @@ -20,20 +22,20 @@ func BenchmarkMessageEncodingRoundtrip(b *testing.B) { root := testutil.GenerateCids(1)[0] ssb := builder.NewSelectorSpecBuilder(basicnode.Prototype.Any) selector := ssb.Matcher().Node() - // extensionName := graphsync.ExtensionName("graphsync/awesome") - // extension := NamedExtension{ - // Name: extensionName, - // Data: basicnode.NewBytes(testutil.RandomBytes(100)), - // } + extensionName := graphsync.ExtensionName("graphsync/awesome") + extension := graphsync.ExtensionData{ + Name: extensionName, + Data: testutil.RandomBytes(100), + } id := graphsync.RequestID(rand.Int31()) priority := graphsync.Priority(rand.Int31()) status := graphsync.RequestAcknowledged builder := NewBuilder() - // builder.AddRequest(NewRequest(id, root, selector, priority, extension)) + builder.AddRequest(NewRequest(id, root, selector, priority, extension)) builder.AddRequest(NewRequest(id, root, selector, priority)) builder.AddResponseCode(id, status) - // builder.AddExtensionData(id, extension) + builder.AddExtensionData(id, extension) builder.AddBlock(blocks.NewBlock([]byte("W"))) builder.AddBlock(blocks.NewBlock([]byte("E"))) builder.AddBlock(blocks.NewBlock([]byte("F"))) @@ -54,9 +56,12 @@ func BenchmarkMessageEncodingRoundtrip(b *testing.B) { gsm2, err := FromNet(buf) require.NoError(b, err) - // Unfortunately, protobuf encoding isn't canonical nor stable. - _ = gsm2 - // require.Equal(b, gsm, gsm2) + + // Note that require.Equal doesn't seem to handle maps well. + // It says they are non-equal simply because their order isn't deterministic. + if diff := cmp.Diff(gsm, gsm2, cmp.Exporter(func(reflect.Type) bool { return true })); diff != "" { + b.Fatal(diff) + } } }) }) @@ -82,7 +87,10 @@ func BenchmarkMessageEncodingRoundtrip(b *testing.B) { gsm2, err := messageFromIPLD(ipldGSM2) require.NoError(b, err) - require.Equal(b, gsm, gsm2) + // same as above. + if diff := cmp.Diff(gsm, gsm2, cmp.Exporter(func(reflect.Type) bool { return true })); diff != "" { + b.Fatal(diff) + } } }) }) diff --git a/message/message.go b/message/message.go index 1e02816a..02ff14d8 100644 --- a/message/message.go +++ b/message/message.go @@ -160,9 +160,6 @@ func newMessageFromProto(pbm *pb.Message) (GraphSyncMessage, error) { } } exts := req.GetExtensions() - if exts == nil { - exts = make(map[string][]byte) - } requests[graphsync.RequestID(req.Id)] = newRequest(graphsync.RequestID(req.Id), root, selector, graphsync.Priority(req.Priority), req.Cancel, req.Update, exts) } @@ -172,9 +169,6 @@ func newMessageFromProto(pbm *pb.Message) (GraphSyncMessage, error) { return GraphSyncMessage{}, errors.New("response is nil") } exts := res.GetExtensions() - if exts == nil { - exts = make(map[string][]byte) - } responses[graphsync.RequestID(res.Id)] = newResponse(graphsync.RequestID(res.Id), graphsync.ResponseStatusCode(res.Status), exts) } From ed7254b25e60c0e39ca14599700928db4a5c056a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Thu, 13 Jan 2022 15:56:06 +0000 Subject: [PATCH 20/21] drop the ipld-prime replace --- go.mod | 4 +--- go.sum | 6 ++++++ 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3c62de84..a23ef5b4 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/ipfs/go-peertaskqueue v0.7.1 github.com/ipfs/go-unixfs v0.2.4 github.com/ipld/go-codec-dagpb v1.3.0 - github.com/ipld/go-ipld-prime v0.14.4-0.20220110161855-fc09d6b768e9 + github.com/ipld/go-ipld-prime v0.14.4 github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c github.com/libp2p/go-buffer-pool v0.0.2 github.com/libp2p/go-libp2p v0.16.0 @@ -45,5 +45,3 @@ require ( golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 google.golang.org/protobuf v1.27.1 ) - -replace github.com/ipld/go-ipld-prime => ../../src/ipld diff --git a/go.sum b/go.sum index 0794f5aa..3cc56de9 100644 --- a/go.sum +++ b/go.sum @@ -180,6 +180,7 @@ github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJn github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= +github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -454,6 +455,10 @@ github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2 github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= github.com/ipld/go-codec-dagpb v1.3.0 h1:czTcaoAuNNyIYWs6Qe01DJ+sEX7B+1Z0LcXjSatMGe8= github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA= +github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= +github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= +github.com/ipld/go-ipld-prime v0.14.4 h1:bqhmume8+nbNsX4/+J6eohktfZHAI8GKrF3rQ0xgOyc= +github.com/ipld/go-ipld-prime v0.14.4/go.mod h1:QcE4Y9n/ZZr8Ijg5bGPT0GqYWgZ1704nH0RDcQtgTP0= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= @@ -947,6 +952,7 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polydawn/refmt v0.0.0-20190408063855-01bf1e26dd14/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e h1:ZOcivgkkFRnjfoTcGsDq3UQYiBmekwLA+qg0OjyB/ls= github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI= From 6619a078d0d7c75638f93c811afb6ea4a9a3cca9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Daniel=20Mart=C3=AD?= Date: Thu, 13 Jan 2022 23:32:56 +0000 Subject: [PATCH 21/21] fix up ID vs Id, remove unintentionally copied code --- message/ipldbind/message.go | 350 +----------------------------------- message/message.go | 8 +- 2 files changed, 6 insertions(+), 352 deletions(-) diff --git a/message/ipldbind/message.go b/message/ipldbind/message.go index d0dc5cce..c78ef556 100644 --- a/message/ipldbind/message.go +++ b/message/ipldbind/message.go @@ -1,24 +1,14 @@ package ipldbind import ( - "encoding/binary" - "errors" - "fmt" "io" - "sort" blocks "github.com/ipfs/go-block-format" cid "github.com/ipfs/go-cid" "github.com/ipld/go-ipld-prime" "github.com/ipld/go-ipld-prime/datamodel" - "github.com/ipld/go-ipld-prime/node/basicnode" - pool "github.com/libp2p/go-buffer-pool" - "github.com/libp2p/go-libp2p-core/network" - "github.com/libp2p/go-msgio" - "google.golang.org/protobuf/proto" "github.com/ipfs/go-graphsync" - "github.com/ipfs/go-graphsync/ipldutil" pb "github.com/ipfs/go-graphsync/message/pb" ) @@ -57,7 +47,7 @@ type GraphSyncExtensions struct { // GraphSyncRequest is a struct to capture data on a request contained in a // GraphSyncMessage. type GraphSyncRequest struct { - ID graphsync.RequestID + Id graphsync.RequestID Root cid.Cid Selector ipld.Node @@ -75,7 +65,7 @@ type GraphSyncMetadatum struct { // GraphSyncResponse is an struct to capture data on a response sent back // in a GraphSyncMessage. type GraphSyncResponse struct { - ID graphsync.RequestID + Id graphsync.RequestID Status graphsync.ResponseStatusCode Metadata []GraphSyncMetadatum @@ -126,344 +116,8 @@ type GraphSyncMessage struct { Blocks []GraphSyncBlock } -// NewRequest builds a new Graphsync request -func NewRequest(id graphsync.RequestID, - root cid.Cid, - selector ipld.Node, - priority graphsync.Priority, - extensions ...NamedExtension) GraphSyncRequest { - - return newRequest(id, root, selector, priority, false, false, toExtensionsMap(extensions)) -} - -// CancelRequest request generates a request to cancel an in progress request -func CancelRequest(id graphsync.RequestID) GraphSyncRequest { - return newRequest(id, cid.Cid{}, nil, 0, true, false, GraphSyncExtensions{}) -} - -// UpdateRequest generates a new request to update an in progress request with the given extensions -func UpdateRequest(id graphsync.RequestID, extensions ...NamedExtension) GraphSyncRequest { - return newRequest(id, cid.Cid{}, nil, 0, false, true, toExtensionsMap(extensions)) -} - // NamedExtension exists just for the purpose of the constructors. type NamedExtension struct { Name graphsync.ExtensionName Data ipld.Node } - -func toExtensionsMap(extensions []NamedExtension) (m GraphSyncExtensions) { - if len(extensions) > 0 { - m.Keys = make([]string, len(extensions)) - m.Values = make(map[string]ipld.Node, len(extensions)) - for i, ext := range extensions { - m.Keys[i] = string(ext.Name) - m.Values[string(ext.Name)] = ext.Data - } - } - return m -} - -func fromProtoExtensions(protoExts map[string][]byte) GraphSyncExtensions { - var exts []NamedExtension - for name, data := range protoExts { - exts = append(exts, NamedExtension{graphsync.ExtensionName(name), basicnode.NewBytes(data)}) - } - // Iterating over the map above is non-deterministic, - // so sort by the unique names to ensure determinism. - sort.Slice(exts, func(i, j int) bool { return exts[i].Name < exts[j].Name }) - return toExtensionsMap(exts) -} - -func newRequest(id graphsync.RequestID, - root cid.Cid, - selector ipld.Node, - priority graphsync.Priority, - isCancel bool, - isUpdate bool, - extensions GraphSyncExtensions) GraphSyncRequest { - return GraphSyncRequest{ - ID: id, - Root: root, - Selector: selector, - Priority: priority, - Cancel: isCancel, - Update: isUpdate, - Extensions: extensions, - } -} - -// NewResponse builds a new Graphsync response -func NewResponse(requestID graphsync.RequestID, - status graphsync.ResponseStatusCode, - extensions ...NamedExtension) GraphSyncResponse { - return newResponse(requestID, status, toExtensionsMap(extensions)) -} - -func newResponse(requestID graphsync.RequestID, - status graphsync.ResponseStatusCode, extensions GraphSyncExtensions) GraphSyncResponse { - return GraphSyncResponse{ - ID: requestID, - Status: status, - Extensions: extensions, - } -} - -func newMessageFromProto(pbm *pb.Message) (GraphSyncMessage, error) { - requests := make([]GraphSyncRequest, len(pbm.GetRequests())) - for i, req := range pbm.Requests { - if req == nil { - return GraphSyncMessage{}, errors.New("request is nil") - } - var root cid.Cid - var err error - if !req.Cancel && !req.Update { - root, err = cid.Cast(req.Root) - if err != nil { - return GraphSyncMessage{}, err - } - } - - var selector ipld.Node - if !req.Cancel && !req.Update { - selector, err = ipldutil.DecodeNode(req.Selector) - if err != nil { - return GraphSyncMessage{}, err - } - } - // TODO: we likely need to turn some "core" extensions to fields, - // as some of those got moved to proper fields in the new protocol. - // Same for responses above, as well as the "to proto" funcs. - requests[i] = newRequest(graphsync.RequestID(req.Id), root, selector, graphsync.Priority(req.Priority), req.Cancel, req.Update, fromProtoExtensions(req.GetExtensions())) - } - - responses := make([]GraphSyncResponse, len(pbm.GetResponses())) - for i, res := range pbm.Responses { - if res == nil { - return GraphSyncMessage{}, errors.New("response is nil") - } - responses[i] = newResponse(graphsync.RequestID(res.Id), graphsync.ResponseStatusCode(res.Status), fromProtoExtensions(res.GetExtensions())) - } - - blks := make([]GraphSyncBlock, len(pbm.GetData())) - for i, b := range pbm.GetData() { - if b == nil { - return GraphSyncMessage{}, errors.New("block is nil") - } - blks[i] = GraphSyncBlock{ - Prefix: b.GetPrefix(), - Data: b.GetData(), - } - } - - return GraphSyncMessage{ - requests, responses, blks, - }, nil -} - -func (gsm GraphSyncMessage) Empty() bool { - return len(gsm.Blocks) == 0 && len(gsm.Requests) == 0 && len(gsm.Responses) == 0 -} - -func (gsm GraphSyncMessage) ResponseCodes() map[graphsync.RequestID]graphsync.ResponseStatusCode { - codes := make(map[graphsync.RequestID]graphsync.ResponseStatusCode, len(gsm.Responses)) - for _, response := range gsm.Responses { - codes[response.ID] = response.Status - } - return codes -} - -// FromNet can read a network stream to deserialized a GraphSyncMessage -func FromNet(r io.Reader) (GraphSyncMessage, error) { - reader := msgio.NewVarintReaderSize(r, network.MessageSizeMax) - return FromMsgReader(reader) -} - -// FromMsgReader can deserialize a protobuf message into a GraphySyncMessage. -func FromMsgReader(r msgio.Reader) (GraphSyncMessage, error) { - msg, err := r.ReadMsg() - if err != nil { - return GraphSyncMessage{}, err - } - - var pb pb.Message - err = proto.Unmarshal(msg, &pb) - r.ReleaseMsg(msg) - if err != nil { - return GraphSyncMessage{}, err - } - - return newMessageFromProto(&pb) -} - -func toProtoExtensions(m GraphSyncExtensions) map[string][]byte { - protoExts := make(map[string][]byte, len(m.Values)) - for name, node := range m.Values { - // Only keep those which are plain bytes, - // as those are the only ones that the older protocol clients understand. - if node.Kind() != ipld.Kind_Bytes { - continue - } - raw, err := node.AsBytes() - if err != nil { - panic(err) // shouldn't happen - } - protoExts[name] = raw - } - return protoExts -} - -func (gsm GraphSyncMessage) ToProto() (*pb.Message, error) { - pbm := new(pb.Message) - pbm.Requests = make([]*pb.Message_Request, 0, len(gsm.Requests)) - for _, request := range gsm.Requests { - var selector []byte - var err error - if request.Selector != nil { - selector, err = ipldutil.EncodeNode(request.Selector) - if err != nil { - return nil, err - } - } - pbm.Requests = append(pbm.Requests, &pb.Message_Request{ - Id: int32(request.ID), - Root: request.Root.Bytes(), - Selector: selector, - Priority: int32(request.Priority), - Cancel: request.Cancel, - Update: request.Update, - Extensions: toProtoExtensions(request.Extensions), - }) - } - - pbm.Responses = make([]*pb.Message_Response, 0, len(gsm.Responses)) - for _, response := range gsm.Responses { - pbm.Responses = append(pbm.Responses, &pb.Message_Response{ - Id: int32(response.ID), - Status: int32(response.Status), - Extensions: toProtoExtensions(response.Extensions), - }) - } - - pbm.Data = make([]*pb.Message_Block, 0, len(gsm.Blocks)) - for _, b := range gsm.Blocks { - pbm.Data = append(pbm.Data, &pb.Message_Block{ - Prefix: b.Prefix, - Data: b.Data, - }) - } - return pbm, nil -} - -func (gsm GraphSyncMessage) ToNet(w io.Writer) error { - msg, err := gsm.ToProto() - if err != nil { - return err - } - size := proto.Size(msg) - buf := pool.Get(size + binary.MaxVarintLen64) - defer pool.Put(buf) - - n := binary.PutUvarint(buf, uint64(size)) - - out, err := proto.MarshalOptions{}.MarshalAppend(buf[:n], msg) - if err != nil { - return err - } - _, err = w.Write(out) - return err -} - -func (gsm GraphSyncMessage) Loggable() map[string]interface{} { - requests := make([]string, 0, len(gsm.Requests)) - for _, request := range gsm.Requests { - requests = append(requests, fmt.Sprintf("%d", request.ID)) - } - responses := make([]string, 0, len(gsm.Responses)) - for _, response := range gsm.Responses { - responses = append(responses, fmt.Sprintf("%d", response.ID)) - } - return map[string]interface{}{ - "requests": requests, - "responses": responses, - } -} - -func (gsm GraphSyncMessage) Clone() GraphSyncMessage { - requests := append([]GraphSyncRequest{}, gsm.Requests...) - responses := append([]GraphSyncResponse{}, gsm.Responses...) - blocks := append([]GraphSyncBlock{}, gsm.Blocks...) - return GraphSyncMessage{requests, responses, blocks} -} - -// Extension returns the content for an extension on a response, or errors -// if extension is not present -func (gsr GraphSyncRequest) Extension(name graphsync.ExtensionName) (ipld.Node, bool) { - val, ok := gsr.Extensions.Values[string(name)] - if !ok { - return nil, false - } - return val, true -} - -// ExtensionNames returns the names of the extensions included in this request -func (gsr GraphSyncRequest) ExtensionNames() []string { - return gsr.Extensions.Keys -} - -// Extension returns the content for an extension on a response, or errors -// if extension is not present -func (gsr GraphSyncResponse) Extension(name graphsync.ExtensionName) (ipld.Node, bool) { - val, ok := gsr.Extensions.Values[string(name)] - if !ok { - return nil, false - } - return val, true -} - -// ExtensionNames returns the names of the extensions included in this request -func (gsr GraphSyncResponse) ExtensionNames() []string { - return gsr.Extensions.Keys -} - -// ReplaceExtensions merges the extensions given extensions into the request to create a new request, -// but always uses new data -func (gsr GraphSyncRequest) ReplaceExtensions(extensions []NamedExtension) GraphSyncRequest { - req, _ := gsr.MergeExtensions(extensions, func(name graphsync.ExtensionName, oldNode, newNode ipld.Node) (ipld.Node, error) { - return newNode, nil - }) - return req -} - -// MergeExtensions merges the given list of extensions to produce a new request with the combination of the old request -// plus the new extensions. When an old extension and a new extension are both present, mergeFunc is called to produce -// the result -func (gsr GraphSyncRequest) MergeExtensions(extensions []NamedExtension, mergeFunc func(name graphsync.ExtensionName, oldNode, newNode ipld.Node) (ipld.Node, error)) (GraphSyncRequest, error) { - if len(gsr.Extensions.Keys) == 0 { - return newRequest(gsr.ID, gsr.Root, gsr.Selector, gsr.Priority, gsr.Cancel, gsr.Update, toExtensionsMap(extensions)), nil - } - combinedExtensions := make(map[string]ipld.Node) - for _, newExt := range extensions { - oldNode, ok := gsr.Extensions.Values[string(newExt.Name)] - if !ok { - combinedExtensions[string(newExt.Name)] = newExt.Data - continue - } - resultNode, err := mergeFunc(graphsync.ExtensionName(newExt.Name), oldNode, newExt.Data) - if err != nil { - return GraphSyncRequest{}, err - } - combinedExtensions[string(newExt.Name)] = resultNode - } - - for name, oldNode := range gsr.Extensions.Values { - _, ok := combinedExtensions[name] - if ok { - continue - } - combinedExtensions[name] = oldNode - } - extNames := make([]string, len(combinedExtensions)) - sort.Strings(extNames) // for reproducibility - return newRequest(gsr.ID, gsr.Root, gsr.Selector, gsr.Priority, gsr.Cancel, gsr.Update, GraphSyncExtensions{extNames, combinedExtensions}), nil -} diff --git a/message/message.go b/message/message.go index 02ff14d8..3a08949e 100644 --- a/message/message.go +++ b/message/message.go @@ -308,7 +308,7 @@ func (gsm GraphSyncMessage) ToIPLD() (*ipldbind.GraphSyncMessage, error) { ibm.Requests = make([]ipldbind.GraphSyncRequest, 0, len(gsm.requests)) for _, request := range gsm.requests { ibm.Requests = append(ibm.Requests, ipldbind.GraphSyncRequest{ - ID: request.id, + Id: request.id, Root: request.root, Selector: request.selector, Priority: request.priority, @@ -321,7 +321,7 @@ func (gsm GraphSyncMessage) ToIPLD() (*ipldbind.GraphSyncMessage, error) { ibm.Responses = make([]ipldbind.GraphSyncResponse, 0, len(gsm.responses)) for _, response := range gsm.responses { ibm.Responses = append(ibm.Responses, ipldbind.GraphSyncResponse{ - ID: response.requestID, + Id: response.requestID, Status: response.status, // Extensions: response.extensions, }) @@ -342,13 +342,13 @@ func messageFromIPLD(ibm *ipldbind.GraphSyncMessage) (GraphSyncMessage, error) { requests := make(map[graphsync.RequestID]GraphSyncRequest, len(ibm.Requests)) for _, req := range ibm.Requests { // exts := req.Extensions - requests[graphsync.RequestID(req.ID)] = newRequest(graphsync.RequestID(req.ID), req.Root, req.Selector, graphsync.Priority(req.Priority), req.Cancel, req.Update, nil) + requests[graphsync.RequestID(req.Id)] = newRequest(graphsync.RequestID(req.Id), req.Root, req.Selector, graphsync.Priority(req.Priority), req.Cancel, req.Update, nil) } responses := make(map[graphsync.RequestID]GraphSyncResponse, len(ibm.Responses)) for _, res := range ibm.Responses { // exts := res.Extensions - responses[graphsync.RequestID(res.ID)] = newResponse(graphsync.RequestID(res.ID), graphsync.ResponseStatusCode(res.Status), nil) + responses[graphsync.RequestID(res.Id)] = newResponse(graphsync.RequestID(res.Id), graphsync.ResponseStatusCode(res.Status), nil) } blks := make(map[cid.Cid]blocks.Block, len(ibm.Blocks))