diff --git a/go.mod b/go.mod index 2ec6877c..0556056e 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/ipfs/go-graphsync go 1.12 require ( + github.com/filecoin-project/go-fil-markets v0.0.0-20200408062434-d92f329a6428 github.com/gogo/protobuf v1.3.1 github.com/ipfs/go-block-format v0.0.2 github.com/ipfs/go-blockservice v0.1.3 @@ -10,20 +11,21 @@ require ( github.com/ipfs/go-datastore v0.4.4 github.com/ipfs/go-ipfs-blockstore v0.1.4 github.com/ipfs/go-ipfs-blocksutil v0.0.1 - github.com/ipfs/go-ipfs-chunker v0.0.1 + github.com/ipfs/go-ipfs-chunker v0.0.5 github.com/ipfs/go-ipfs-exchange-offline v0.0.1 - github.com/ipfs/go-ipfs-files v0.0.4 + github.com/ipfs/go-ipfs-files v0.0.8 github.com/ipfs/go-ipfs-util v0.0.1 - github.com/ipfs/go-ipld-format v0.0.2 + github.com/ipfs/go-ipld-format v0.2.0 github.com/ipfs/go-log v1.0.2 - github.com/ipfs/go-merkledag v0.2.4 + github.com/ipfs/go-merkledag v0.3.1 github.com/ipfs/go-peertaskqueue v0.2.0 - github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb - github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785 - github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5 + github.com/ipfs/go-unixfs v0.2.4 + github.com/ipld/go-ipld-prime v0.0.2-0.20200229094926-eb71617f4aeb + github.com/ipld/go-ipld-prime-proto v0.0.0-20200409003434-8cf97d9cb362 github.com/jbenet/go-random v0.0.0-20190219211222-123a90aedc0c github.com/libp2p/go-libp2p v0.6.0 github.com/libp2p/go-libp2p-core v0.5.0 + github.com/libp2p/go-libp2p-peer v0.2.0 github.com/multiformats/go-multiaddr v0.2.1 github.com/multiformats/go-multihash v0.0.13 github.com/stretchr/testify v1.4.0 diff --git a/go.sum b/go.sum index 4d9216a2..1ec6e071 100644 --- a/go.sum +++ b/go.sum @@ -32,6 +32,9 @@ github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3Ee github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= +github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= +github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= github.com/cskr/pubsub v1.0.2 h1:vlOzMhl6PFn60gRlTQQsIfVwaPB/B/8MziK8FhEPt/0= github.com/cskr/pubsub v1.0.2/go.mod h1:/8MzYXk/NJAz782G8RPkFzXTZVu63VotefPnR9TIRis= github.com/davecgh/go-spew v0.0.0-20171005155431-ecdeabc65495/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -46,6 +49,27 @@ github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6ps github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/fatih/color v1.8.0/go.mod h1:3l45GVGkyrnYNl9HoIjnp2NnNWvh6hLAqD8yTfGjnw8= +github.com/filecoin-project/filecoin-ffi v0.0.0-20200304181354-4446ff8a1bb9/go.mod h1:PtH9YP0rURHUKHrKeEBeWg/BqIBMQOz8wtlXlVGREBE= +github.com/filecoin-project/go-address v0.0.0-20200107215422-da8eea2842b5/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= +github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be h1:TooKBwR/g8jG0hZ3lqe9S5sy2vTUcLOZLlz3M5wGn2E= +github.com/filecoin-project/go-address v0.0.2-0.20200218010043-eb9bb40ed5be/go.mod h1:SAOwJoakQ8EPjwNIsiakIQKsoKdkcbx8U3IapgCg9R0= +github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200131012142-05d80eeccc5e h1:IOoff6yAZSJ5zHCPY2jzGNwQYQU6ygsRVe/cSnJrY+o= +github.com/filecoin-project/go-amt-ipld/v2 v2.0.1-0.20200131012142-05d80eeccc5e/go.mod h1:boRtQhzmxNocrMxOXo1NYn4oUc1NGvR8tEa79wApNXg= +github.com/filecoin-project/go-cbor-util v0.0.0-20191219014500-08c40a1e63a2/go.mod h1:pqTiPHobNkOVM5thSRsHYjyQfq7O5QSCMhvuu9JoDlg= +github.com/filecoin-project/go-crypto v0.0.0-20191218222705-effae4ea9f03/go.mod h1:+viYnvGtUTgJRdy6oaeF4MTFKAfatX071MPDPBL11EQ= +github.com/filecoin-project/go-data-transfer v0.0.0-20200408061858-82c58b423ca6/go.mod h1:7b5/sG9Jj33aWqft8XsH8yIdxZBACqS5tx9hv4uj2Ck= +github.com/filecoin-project/go-fil-commcid v0.0.0-20200208005934-2b8bd03caca5/go.mod h1:JbkIgFF/Z9BDlvrJO1FuKkaWsH673/UdFaiVS6uIHlA= +github.com/filecoin-project/go-fil-markets v0.0.0-20200408062434-d92f329a6428 h1:y8P10ZwfmsKMVHrqcU1L8Bgj2q42O6LzaySI+XaogXE= +github.com/filecoin-project/go-fil-markets v0.0.0-20200408062434-d92f329a6428/go.mod h1:NmuTIqaivdyUzmvHOUCsTDGEtNjOZQwC1cgW3W/02m4= +github.com/filecoin-project/go-padreader v0.0.0-20200210211231-548257017ca6/go.mod h1:0HgYnrkeSU4lu1p+LEOeDpFsNBssa0OGGriWdA4hvaE= +github.com/filecoin-project/go-paramfetch v0.0.2-0.20200218225740-47c639bab663/go.mod h1:fZzmf4tftbwf9S37XRifoJlz7nCjRdIrMGLR07dKLCc= +github.com/filecoin-project/go-sectorbuilder v0.0.2-0.20200326160829-51775363aa18/go.mod h1:xAd/X905Ncgj8kkHsP2pmQUf6MQT2qJTDcOEfkwCjYc= +github.com/filecoin-project/go-statemachine v0.0.0-20200226041606-2074af6d51d9/go.mod h1:FGwQgZAt2Gh5mjlwJUlVB62JeYdo+if0xWxSEfBD9ig= +github.com/filecoin-project/go-statestore v0.1.0/go.mod h1:LFc9hD+fRxPqiHiaqUEZOinUJB4WARkRfNl10O7kTnI= +github.com/filecoin-project/specs-actors v0.0.0-20200210130641-2d1fbd8672cf/go.mod h1:xtDZUB6pe4Pksa/bAJbJ693OilaC5Wbot9jMhLm3cZA= +github.com/filecoin-project/specs-actors v0.0.0-20200226200336-94c9b92b2775 h1:j6EdTc5hIHe31ydt9SjgJNDi/ck50y5wa1OJ+4YZ6zk= +github.com/filecoin-project/specs-actors v0.0.0-20200226200336-94c9b92b2775/go.mod h1:0HAWYrvajFHDgRaKbF0rl+IybVLZL5z4gQ8koCMPhoU= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/go-check/check v0.0.0-20180628173108-788fd7840127 h1:0gkP6mzaMqkmpcJYCFOLkIBwI7xFExG03bbkOkCvUPI= @@ -63,16 +87,21 @@ github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5y github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a1/R87v0= github.com/golang/protobuf v1.3.1 h1:YF8+flBXS5eO826T4nzqPrxfhQThhXl0YzfuUPu4SBg= github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0 h1:crn/baboCvb5fXaQ0IJ1SGTsTVrWpDsCWC8EGETZijY= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= +github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20190812055157-5d271430af9f/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.0 h1:WDFjx/TMzVgy9VdMMQi2K2Emtwi2QcUQsztZ/zLaH/Q= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= @@ -81,6 +110,8 @@ github.com/gxed/hashland/keccakpg v0.0.1 h1:wrk3uMNaMxbXiHibbPO4S0ymqJMm41WiudyF github.com/gxed/hashland/keccakpg v0.0.1/go.mod h1:kRzw3HkwxFU1mpmPP8v1WyQzwdGfmKFJ6tItnhQ67kU= github.com/gxed/hashland/murmur3 v0.0.1 h1:SheiaIt0sda5K+8FLz952/1iWS9zrnKsEJaOJu4ZbSc= github.com/gxed/hashland/murmur3 v0.0.1/go.mod h1:KjXop02n4/ckmZSnY2+HKcLud/tcmvhST0bie/0lS48= +github.com/hannahhoward/cbor-gen-for v0.0.0-20191216214420-3e450425c40c/go.mod h1:WVPCl0HO/0RAL5+vBH2GMxBomlxBF70MAS78+Lu1//k= +github.com/hannahhoward/cbor-gen-for v0.0.0-20191218204337-9ab7b1bcc099/go.mod h1:WVPCl0HO/0RAL5+vBH2GMxBomlxBF70MAS78+Lu1//k= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1 h1:0hERBMJE1eitiLkihrMvRVBYAkpHzc/J3QdDN+dAcgU= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -115,6 +146,7 @@ github.com/ipfs/go-cid v0.0.2 h1:tuuKaZPU1M6HcejsO3AcYWW8sZ8MTvyxfc4uqB4eFE8= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3 h1:UIAh32wymBpStoe83YCzwVQQ5Oy/H0FdxvUS6DJDzms= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= +github.com/ipfs/go-cid v0.0.4-0.20191112011718-79e75dffeb10/go.mod h1:/BYOuUoxkE+0f6tGzlzMvycuN+5l35VOR4Bpg2sCmds= github.com/ipfs/go-cid v0.0.4/go.mod h1:4LLaPOQwmk5z9LBgQnpkivrx8BJjUyGwTXCd5Xfj6+M= github.com/ipfs/go-cid v0.0.5 h1:o0Ix8e/ql7Zb5UVUJEUfjsWCIY8t48++9lR8qi6oiJU= github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67FexhXog= @@ -122,7 +154,9 @@ github.com/ipfs/go-datastore v0.0.1 h1:AW/KZCScnBWlSb5JbnEnLKFWXL224LBEh/9KXXOrU github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.0.5 h1:q3OfiOZV5rlsK1H5V8benjeUApRfMGs4Mrhmr6NriQo= github.com/ipfs/go-datastore v0.0.5/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= +github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= +github.com/ipfs/go-datastore v0.3.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= github.com/ipfs/go-datastore v0.4.0/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= github.com/ipfs/go-datastore v0.4.4 h1:rjvQ9+muFaJ+QZ7dN5B1MSDNQ0JVZKkkES/rMZmA8X8= @@ -134,14 +168,20 @@ github.com/ipfs/go-ds-badger v0.0.5/go.mod h1:g5AuuCGmr7efyzQhLL8MzwqcauPojGPUaH github.com/ipfs/go-ds-badger v0.2.1/go.mod h1:Tx7l3aTph3FMFrRS838dcSJh+jjA7cX9DrGVwx/NOwE= github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= github.com/ipfs/go-ds-leveldb v0.4.1/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= +github.com/ipfs/go-graphsync v0.0.6-0.20200408061628-e1a98fc64c42/go.mod h1:pX9G90DyPVZCFGcjWlpDTl1rTGU+TAG3SA6bg8GXQXc= +github.com/ipfs/go-hamt-ipld v0.0.15-0.20200131012125-dd88a59d3f2e h1:bUtmeXx6JpjxRPlMdlKfPXC5kKhLHuueXKgs1Txb9ZU= +github.com/ipfs/go-hamt-ipld v0.0.15-0.20200131012125-dd88a59d3f2e/go.mod h1:9aQJu/i/TaRDW6jqB5U217dLIDopn50wxLdHXM2CTfE= github.com/ipfs/go-ipfs-blockstore v0.0.1 h1:O9n3PbmTYZoNhkgkEyrXTznbmktIXif62xLX+8dPHzc= github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= +github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= github.com/ipfs/go-ipfs-blockstore v0.1.4 h1:2SGI6U1B44aODevza8Rde3+dY30Pb+lbcObe1LETxOQ= github.com/ipfs/go-ipfs-blockstore v0.1.4/go.mod h1:Jxm3XMVjh6R17WvxFEiyKBLUGr86HgIYJW/D/MwqeYQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= github.com/ipfs/go-ipfs-chunker v0.0.1 h1:cHUUxKFQ99pozdahi+uSC/3Y6HeRpi9oTeUHbE27SEw= github.com/ipfs/go-ipfs-chunker v0.0.1/go.mod h1:tWewYK0we3+rMbOh7pPFGDyypCtvGcBFymgY4rSDLAw= +github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8= +github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= @@ -156,6 +196,8 @@ github.com/ipfs/go-ipfs-exchange-offline v0.0.1/go.mod h1:WhHSFCVYX36H/anEKQboAz github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= github.com/ipfs/go-ipfs-files v0.0.4 h1:WzRCivcybUQch/Qh6v8LBRhKtRsjnwyiuOV09mK7mrE= github.com/ipfs/go-ipfs-files v0.0.4/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= +github.com/ipfs/go-ipfs-files v0.0.8 h1:8o0oFJkJ8UkO/ABl8T6ac6tKF3+NIpj67aAB6ZpusRg= +github.com/ipfs/go-ipfs-files v0.0.8/go.mod h1:wiN/jSG8FKyk7N0WyctKSvq3ljIa2NNTiZB55kpTdOs= github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs= github.com/ipfs/go-ipfs-posinfo v0.0.1/go.mod h1:SwyeVP+jCwiDu0C313l/8jg6ZxM0qqtlt2a0vILTc1A= github.com/ipfs/go-ipfs-pq v0.0.1 h1:zgUotX8dcAB/w/HidJh1zzc1yFq6Vm8J7T2F4itj/RU= @@ -168,19 +210,30 @@ github.com/ipfs/go-ipfs-util v0.0.1 h1:Wz9bL2wB2YBJqggkA4dD7oSmqB4cAnpNbGrlHJulv github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= github.com/ipfs/go-ipld-cbor v0.0.2 h1:amzFztBQQQ69UA5+f7JRfoXF/z2l//MGfEDHVkS20+s= github.com/ipfs/go-ipld-cbor v0.0.2/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= +github.com/ipfs/go-ipld-cbor v0.0.3 h1:ENsxvybwkmke7Z/QJOmeJfoguj6GH3Y0YOaGrfy9Q0I= +github.com/ipfs/go-ipld-cbor v0.0.3/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= +github.com/ipfs/go-ipld-cbor v0.0.4 h1:Aw3KPOKXjvrm6VjwJvFf1F1ekR/BH3jdof3Bk7OTiSA= +github.com/ipfs/go-ipld-cbor v0.0.4/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4= github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms= github.com/ipfs/go-ipld-format v0.0.2 h1:OVAGlyYT6JPZ0pEfGntFPS40lfrDmaDbQwNHEY2G9Zs= github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k= +github.com/ipfs/go-ipld-format v0.2.0 h1:xGlJKkArkmBvowr+GMCX0FEZtkro71K1AwiKnL37mwA= +github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs= github.com/ipfs/go-log v0.0.1 h1:9XTUN/rW64BCG1YhPK9Hoy3q8nr4gOmHHBpgFdfw6Lc= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= +github.com/ipfs/go-log v1.0.0/go.mod h1:JO7RzlMK6rA+CIxFMLOuB6Wf5b81GDiKElL7UPSIKjA= +github.com/ipfs/go-log v1.0.1/go.mod h1:HuWlQttfN6FWNHRhlY5yMk/lW7evQC0HHGOxEwMRR8I= github.com/ipfs/go-log v1.0.2 h1:s19ZwJxH8rPWzypjcDpqPLIyV7BnbLqvpli3iZoqYK0= github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk= +github.com/ipfs/go-log/v2 v2.0.1/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= github.com/ipfs/go-log/v2 v2.0.2 h1:xguurydRdfKMJjKyxNXNU8lYP0VZH1NUwJRwUorjuEw= github.com/ipfs/go-log/v2 v2.0.2/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= github.com/ipfs/go-merkledag v0.2.3 h1:aMdkK9G1hEeNvn3VXfiEMLY0iJnbiQQUHnM0HFJREsE= github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= github.com/ipfs/go-merkledag v0.2.4 h1:ZSHQSe9BENfixUjT+MaLeHEeZGxrZQfgo3KT3SLosF8= github.com/ipfs/go-merkledag v0.2.4/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= +github.com/ipfs/go-merkledag v0.3.1 h1:3UqWINBEr3/N+r6OwgFXAddDP/8zpQX/8J7IGVOCqRQ= +github.com/ipfs/go-merkledag v0.3.1/go.mod h1:fvkZNNZixVW6cKSZ/JfLlON5OlgTXNdRLz0p6QG/I2M= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/ipfs/go-peertaskqueue v0.1.0 h1:bpRbgv76eT4avutNPDFZuCPOQus6qTgurEYxfulgZW4= @@ -190,12 +243,20 @@ github.com/ipfs/go-peertaskqueue v0.2.0 h1:2cSr7exUGKYyDeUyQ7P/nHPs9P7Ht/B+ROrpN github.com/ipfs/go-peertaskqueue v0.2.0/go.mod h1:5/eNrBEbtSKWCG+kQK8K8fGNixoYUnr+P7jivavs9lY= github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb h1:tmWYgjltxwM7PDmFJgWgLuy5qx24csUvRoIiO+F/zQ4= github.com/ipfs/go-unixfs v0.2.2-0.20190827150610-868af2e9e5cb/go.mod h1:IwAAgul1UQIcNZzKPYZWOCijryFBeCV79cNubPzol+k= +github.com/ipfs/go-unixfs v0.2.4 h1:6NwppOXefWIyysZ4LR/qUBPvXd5//8J3jiMdvpbw6Lo= +github.com/ipfs/go-unixfs v0.2.4/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw= github.com/ipfs/go-verifcid v0.0.1 h1:m2HI7zIuR5TFyQ1b79Da5N9dnnCP1vcu2QqawmWlK2E= github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= +github.com/ipld/go-car v0.0.5-0.20200316204026-3e2cf7af0fab/go.mod h1:yR5AsJ38xTwwgwGpbh60ICtdLPp5lGfuH28PAAzaEhM= github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785 h1:fASnkvtR+SmB2y453RxmDD3Uvd4LonVUgFGk9JoDaZs= github.com/ipld/go-ipld-prime v0.0.2-0.20191108012745-28a82f04c785/go.mod h1:bDDSvVz7vaK12FNvMeRYnpRFkSUPNQOiCYQezMD/P3w= +github.com/ipld/go-ipld-prime v0.0.2-0.20200229094926-eb71617f4aeb h1:uvoZ2aTjsTt/6W5hkzj3JkPMY3oHvcla4WA29k4Jk0I= +github.com/ipld/go-ipld-prime v0.0.2-0.20200229094926-eb71617f4aeb/go.mod h1:uVIwe/u0H4VdKv3kaN1ck7uCb6yD9cFLS9/ELyXbsw8= github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5 h1:lSip43rAdyGA+yRQuy6ju0ucZkWpYc1F2CTQtZTVW/4= github.com/ipld/go-ipld-prime-proto v0.0.0-20191113031812-e32bd156a1e5/go.mod h1:gcvzoEDBjwycpXt3LBE061wT9f46szXGHAmj9uoP6fU= +github.com/ipld/go-ipld-prime-proto v0.0.0-20200409003434-8cf97d9cb362 h1:ZPMg5SLiHu/FzBIv7p8obE3VGwiKaRNnM+QxiYsk+tg= +github.com/ipld/go-ipld-prime-proto v0.0.0-20200409003434-8cf97d9cb362/go.mod h1:ScHZhxNfNoOTfM87iqdAC37cDh56S+0/LK0NtyriaBk= +github.com/ipsn/go-secp256k1 v0.0.0-20180726113642-9d62b9f0bc52/go.mod h1:fdg+/X9Gg4AsAIzWpEHwnqd+QY3b7lajxyjE1m4hkq4= github.com/jackpal/gateway v1.0.5 h1:qzXWUJfuMdlLMtt0a3Dgt+xkWQiA5itDEITVJtuSwMc= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1 h1:i0LektDkO1QlrTm/cSuP+PyBCDnYvjPLGl4LdWEMiaA= @@ -303,6 +364,7 @@ github.com/libp2p/go-libp2p-pnet v0.2.0 h1:J6htxttBipJujEjz1y0a5+eYoiPcFHhSYHH6n github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYcO0BW4wssv21LA= github.com/libp2p/go-libp2p-record v0.1.0 h1:wHwBGbFzymoIl69BpgwIu0O6ta3TXGcMPvHUAcodzRc= github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7xI4hAIl8pE6wu5Q= +github.com/libp2p/go-libp2p-record v0.1.1/go.mod h1:VRgKajOyMVgP/F0L5g3kH7SVskp17vFi2xheb5uMJtg= github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= github.com/libp2p/go-libp2p-secio v0.2.0 h1:ywzZBsWEEz2KNTn5RtzauEDq5RFEefPsttXYwAWqHng= github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g= @@ -378,11 +440,15 @@ github.com/mattn/go-colorable v0.1.1 h1:G1f5SKeVxmagw/IyvzvtZE4Gybcc4Tr1tf7I8z0X github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-isatty v0.0.5 h1:tHXDdz1cpzGaovsTB+TVB8q90WEokoVmfMqoVcrLUgw= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.9/go.mod h1:YNRxwqDuOph6SZLI9vUUz6OYw3QyUt7WiY2yME+cCiQ= +github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/minio/blake2b-simd v0.0.0-20160723061019-3f5f724cb5b1 h1:lYpkrQH5ajf0OXOcUbGjvZxxijuBwbbmlSxLiuofa+g= @@ -440,6 +506,7 @@ github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKT github.com/multiformats/go-multihash v0.0.5 h1:1wxmCvTXAifAepIMyF39vZinRw5sbqjPs/UIi93+uik= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= +github.com/multiformats/go-multihash v0.0.9/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.10/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= github.com/multiformats/go-multihash v0.0.13 h1:06x+mk/zj1FoMsgNejLpy6QTvJqlSt/BhLEy87zidlc= github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= @@ -467,6 +534,9 @@ github.com/opentracing/opentracing-go v1.0.2 h1:3jA2P6O1F9UOrWVpwrIo17pu01KWvNWg github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= github.com/opentracing/opentracing-go v1.1.0 h1:pWlfV3Bxv7k65HYwkikxat0+s3pV4bsqf19k25Ur8rU= github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= +github.com/otiai10/copy v1.0.2/go.mod h1:c7RpqBkwMom4bYTSkLSym4VSJz/XtncWRAj/J4PEIMY= +github.com/otiai10/curr v0.0.0-20150429015615-9b4961190c95/go.mod h1:9qAhocn7zKJG+0mI8eUu6xqkFDYS2kb2saOteoSB3cE= +github.com/otiai10/mint v1.3.0/go.mod h1:F5AjcsTsWUqX+Na9fpHb52P8pcRX2CI6A3ctIT91xUo= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -477,15 +547,24 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polydawn/refmt v0.0.0-20190408063855-01bf1e26dd14 h1:2m16U/rLwVaRdz7ANkHtHTodP3zTP3N451MADg64x5k= github.com/polydawn/refmt v0.0.0-20190408063855-01bf1e26dd14/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1 h1:CskT+S6Ay54OwxBGB0R3Rsx4Muto6UnEYTyKJbyRIAI= +github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a h1:hjZfReYVLbqFkAtr2us7vdy04YWz3LVAirzP7reh8+M= +github.com/polydawn/refmt v0.0.0-20190809202753-05966cbd336a/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= +github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= +github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= +github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/goconvey v0.0.0-20190731233626-505e41936337/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smola/gocompat v0.2.0/go.mod h1:1B0MlxbmoZNo3h8guHp8HztB3BSYR5itql9qtVc0ypY= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a h1:/eS3yfGjQKG+9kayBkj0ip1BGhq6zJ3eaVksphxAaek= github.com/spacemonkeygo/openssl v0.0.0-20181017203307-c2dcc5cca94a/go.mod h1:7AyxJNCJ7SBZ1MfVQCWD6Uqo2oubI2Eq2y2eqf+A5r0= @@ -509,9 +588,19 @@ github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJy github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli/v2 v2.0.0/go.mod h1:SE9GqnLQmjVa0iPEY0f1w3ygNIYcIJ0OKPMoW2caLfQ= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830 h1:8kxMKmKzXXL4Ru1nyhvdms/JjWt+3YLpvRb/bAjO/y0= github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a h1:G++j5e0OC488te356JvdhaM8YS6nMsjLAYF7JxCv07w= +github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/whyrusleeping/cbor-gen v0.0.0-20191212224538-d370462a7e8a/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= +github.com/whyrusleeping/cbor-gen v0.0.0-20191216205031-b047b6acb3c0/go.mod h1:xdlJQaiqipF0HW+Mzpg7XRM3fWbGvfgFlcppuvlkIvY= +github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= +github.com/whyrusleeping/cbor-gen v0.0.0-20200206220010-03c9665e2a66/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= +github.com/whyrusleeping/cbor-gen v0.0.0-20200402171437-3d27c146c105 h1:Sh6UG5dW5xW8Ek2CtRGq4ipdEvvx9hOyBJjEGyTYDl0= +github.com/whyrusleeping/cbor-gen v0.0.0-20200402171437-3d27c146c105/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= @@ -536,8 +625,13 @@ go.opencensus.io v0.22.3 h1:8sGtKOrtQqkN1bp2AtX+misvLIlOmsEsNd+9NIcPEm8= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= +go.uber.org/atomic v1.5.0 h1:OI5t8sDa1Or+q8AeE+yKeB/SDYioSHAgcVljj9JIETY= +go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= +go.uber.org/multierr v1.4.0 h1:f3WCSC2KzAcBXGATIxAB1E2XuCpNU255wNKZ505qi3E= +go.uber.org/multierr v1.4.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= +go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -549,6 +643,7 @@ golang.org/x/crypto v0.0.0-20190225124518-7f87c0fbb88b/go.mod h1:6SG95UA2DQfeDnf golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734 h1:p/H982KKEjUnLJkM3tt/LemDnOc1GiZL5FCVlORJ5zo= golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -556,12 +651,17 @@ golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443 h1:IcSOAf4PyMp3U3XbIEj1/x golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200117160349-530e935923ad/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20190930215403-16217165b5de/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/mod v0.0.0-20190513183733-4bf6d317e70e/go.mod h1:mXi4GBBbnImb6dmsKGUJ2LatrhH/nqhxcFungHvyanc= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -576,11 +676,14 @@ golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859 h1:R/3boaszxrf1GEUWTVDzSKVwLmSJpwZ1yqXm8j0v2QI= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80 h1:Ao/3l156eZf2AW5wK8a7/smtodRU+gha3+BeqJ69lRk= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180202135801-37707fdb30a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -599,7 +702,11 @@ golang.org/x/sys v0.0.0-20190524122548-abf6ff778158/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb h1:fgwFCsaw9buMuxNd6+DQfAuSFqbNiQZpcgJQAgJsK6k= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20191120155948-bd437916bb0e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20191210023423-ac6580df4449/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200122134326-e047566fdf82/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae h1:/WDfKMnPU+m5M4xB+6x4kaepxRw6jWvR5iDRdvjHgy8= golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -615,8 +722,15 @@ golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGm golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190621195816-6e04913cbbac/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7 h1:9zdDQZ7Thm29KFXgAX/+yaf3eVbP7djjWp/dXAppNCc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -630,6 +744,8 @@ gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33 gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.28/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/src-d/go-cli.v0 v0.0.0-20181105080154-d492247bbc0d/go.mod h1:z+K8VcOYVYcSwSjGebuDL6176A1XskgbtNl64NSg+n8= @@ -641,4 +757,6 @@ gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gotest.tools v2.2.0+incompatible/go.mod h1:DsYFclhRJ6vuDpmuTbkuFWG+y2sxOXAzmJt81HFBacw= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= +honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt0JzvZhAg= diff --git a/graphsync.go b/graphsync.go index cf4efbd3..0e35f236 100644 --- a/graphsync.go +++ b/graphsync.go @@ -135,6 +135,18 @@ type ResponseData interface { Extension(name ExtensionName) ([]byte, bool) } +// BlockData gives information about a block included in a graphsync response +type BlockData interface { + // Link is the link/cid for the block + Link() ipld.Link + + // BlockSize specifies the size of the block + BlockSize() uint64 + + // BlockSize specifies the amount of data actually transmitted over the network + BlockSizeOnWire() uint64 +} + // IncomingRequestHookActions are actions that a request hook can take to change // behavior for the response type IncomingRequestHookActions interface { @@ -152,6 +164,14 @@ type OutgoingRequestHookActions interface { UseNodeBuilderChooser(traversal.NodeBuilderChooser) } +// OutgoingBlockHookActions are actions that an outgoing block hook can take to +// change the execution of this request +type OutgoingBlockHookActions interface { + SendExtensionData(ExtensionData) + TerminateWithError(error) + PauseResponse() +} + // OnIncomingRequestHook is a hook that runs each time a new request is received. // It receives the peer that sent the request and all data about the request. // It receives an interface for customizing the response to this request @@ -167,6 +187,13 @@ type OnIncomingResponseHook func(p peer.ID, responseData ResponseData) error // It receives an interface for customizing how we handle executing this request type OnOutgoingRequestHook func(p peer.ID, request RequestData, hookActions OutgoingRequestHookActions) +// OnOutgoingBlockHook is a hook that runs immediately after a requestor sends a new block +// on a response +// It receives the peer we're sending a request to, all the data aobut the request, a link for the block sent, +// and the size of the block sent +// It receives an interface for taking further action on the response +type OnOutgoingBlockHook func(p peer.ID, request RequestData, block BlockData, hookActions OutgoingBlockHookActions) + // UnregisterHookFunc is a function call to unregister a hook that was previously registered type UnregisterHookFunc func() @@ -186,4 +213,10 @@ type GraphExchange interface { // RegisterOutgoingRequestHook adds a hook that runs immediately prior to sending a new request RegisterOutgoingRequestHook(hook OnOutgoingRequestHook) UnregisterHookFunc + + // RegisterOutgoingBlockHook adds a hook that runs every time a block is sent from a responder + RegisterOutgoingBlockHook(hook OnOutgoingBlockHook) UnregisterHookFunc + + // UnpauseResponse unpauses a response that was paused in a block hook based on peer ID and request ID + UnpauseResponse(peer.ID, RequestID) error } diff --git a/impl/graphsync.go b/impl/graphsync.go index 2a531e38..992bfc8f 100644 --- a/impl/graphsync.go +++ b/impl/graphsync.go @@ -11,7 +11,10 @@ import ( "github.com/ipfs/go-graphsync/requestmanager" "github.com/ipfs/go-graphsync/requestmanager/asyncloader" "github.com/ipfs/go-graphsync/responsemanager" + "github.com/ipfs/go-graphsync/responsemanager/blockhooks" "github.com/ipfs/go-graphsync/responsemanager/peerresponsemanager" + "github.com/ipfs/go-graphsync/responsemanager/persistenceoptions" + "github.com/ipfs/go-graphsync/responsemanager/requesthooks" "github.com/ipfs/go-graphsync/selectorvalidator" logging "github.com/ipfs/go-log" "github.com/ipfs/go-peertaskqueue" @@ -35,6 +38,9 @@ type GraphSync struct { peerResponseManager *peerresponsemanager.PeerResponseManager peerTaskQueue *peertaskqueue.PeerTaskQueue peerManager *peermanager.PeerMessageManager + incomingRequestHooks *requesthooks.IncomingRequestHooks + outgoingBlockHooks *blockhooks.OutgoingBlockHooks + persistenceOptions *persistenceoptions.PersistenceOptions ctx context.Context cancel context.CancelFunc unregisterDefaultValidator graphsync.UnregisterHookFunc @@ -69,8 +75,11 @@ func New(parent context.Context, network gsnet.GraphSyncNetwork, return peerresponsemanager.NewResponseSender(ctx, p, peerManager) } peerResponseManager := peerresponsemanager.New(ctx, createdResponseQueue) - responseManager := responsemanager.New(ctx, loader, peerResponseManager, peerTaskQueue) - unregisterDefaultValidator := responseManager.RegisterRequestHook(selectorvalidator.SelectorValidator(maxRecursionDepth)) + persistenceOptions := persistenceoptions.New() + incomingRequestHooks := requesthooks.New(persistenceOptions) + outgoingBlockHooks := blockhooks.New() + responseManager := responsemanager.New(ctx, loader, peerResponseManager, peerTaskQueue, incomingRequestHooks, outgoingBlockHooks) + unregisterDefaultValidator := incomingRequestHooks.Register(selectorvalidator.SelectorValidator(maxRecursionDepth)) graphSync := &GraphSync{ network: network, loader: loader, @@ -78,6 +87,9 @@ func New(parent context.Context, network gsnet.GraphSyncNetwork, asyncLoader: asyncLoader, requestManager: requestManager, peerManager: peerManager, + persistenceOptions: persistenceOptions, + incomingRequestHooks: incomingRequestHooks, + outgoingBlockHooks: outgoingBlockHooks, peerTaskQueue: peerTaskQueue, peerResponseManager: peerResponseManager, responseManager: responseManager, @@ -108,7 +120,7 @@ func (gs *GraphSync) Request(ctx context.Context, p peer.ID, root ipld.Link, sel // it is considered to have "validated" the request -- and that validation supersedes // the normal validation of requests Graphsync does (i.e. all selectors can be accepted) func (gs *GraphSync) RegisterIncomingRequestHook(hook graphsync.OnIncomingRequestHook) graphsync.UnregisterHookFunc { - return gs.responseManager.RegisterRequestHook(hook) + return gs.incomingRequestHooks.Register(hook) } // RegisterIncomingResponseHook adds a hook that runs when a response is received @@ -127,7 +139,17 @@ func (gs *GraphSync) RegisterPersistenceOption(name string, loader ipld.Loader, if err != nil { return err } - return gs.responseManager.RegisterPersistenceOption(name, loader) + return gs.persistenceOptions.Register(name, loader) +} + +// RegisterOutgoingBlockHook registers a hook that runs after each block is sent in a response +func (gs *GraphSync) RegisterOutgoingBlockHook(hook graphsync.OnOutgoingBlockHook) graphsync.UnregisterHookFunc { + return gs.outgoingBlockHooks.Register(hook) +} + +// UnpauseResponse unpauses a response that was paused in a block hook based on peer ID and request ID +func (gs *GraphSync) UnpauseResponse(p peer.ID, requestID graphsync.RequestID) error { + return gs.responseManager.UnpauseResponse(p, requestID) } type graphSyncReceiver GraphSync diff --git a/impl/graphsync_test.go b/impl/graphsync_test.go index 52c7878a..d70d5656 100644 --- a/impl/graphsync_test.go +++ b/impl/graphsync_test.go @@ -220,6 +220,58 @@ func TestGraphsyncRoundTrip(t *testing.T) { require.Equal(t, td.extensionResponseData, receivedResponseData, "did not receive correct extension response data") } +func TestPauseResume(t *testing.T) { + // create network + ctx := context.Background() + ctx, cancel := context.WithTimeout(ctx, 2*time.Second) + defer cancel() + td := newGsTestData(ctx, t) + + // initialize graphsync on first node to make requests + requestor := td.GraphSyncHost1() + + // setup receiving peer to just record message coming in + blockChainLength := 100 + blockChain := testutil.SetupBlockChain(ctx, t, td.loader2, td.storer2, 100, blockChainLength) + + // initialize graphsync on second node to response to requests + responder := td.GraphSyncHost2() + + stopPoint := 50 + blocksSent := 0 + requestIDChan := make(chan graphsync.RequestID, 1) + responder.RegisterOutgoingBlockHook(func(p peer.ID, requestData graphsync.RequestData, blockData graphsync.BlockData, hookActions graphsync.OutgoingBlockHookActions) { + _, has := requestData.Extension(td.extensionName) + if has { + select { + case requestIDChan <- requestData.ID(): + default: + } + blocksSent++ + if blocksSent == stopPoint { + hookActions.PauseResponse() + } + } else { + hookActions.TerminateWithError(errors.New("should have sent extension")) + } + }) + + progressChan, errChan := requestor.Request(ctx, td.host2.ID(), blockChain.TipLink, blockChain.Selector(), td.extension) + + blockChain.VerifyResponseRange(ctx, progressChan, 0, stopPoint) + timer := time.NewTimer(100 * time.Millisecond) + testutil.AssertDoesReceiveFirst(t, timer.C, "should pause request", progressChan) + + requestID := <-requestIDChan + err := responder.UnpauseResponse(td.host1.ID(), requestID) + require.NoError(t, err) + + blockChain.VerifyRemainder(ctx, progressChan, stopPoint) + testutil.VerifyEmptyErrors(ctx, t, errChan) + require.Len(t, td.blockStore1, blockChainLength, "did not store all blocks") + +} + func TestGraphsyncRoundTripAlternatePersistenceAndNodes(t *testing.T) { // create network ctx := context.Background() @@ -273,7 +325,7 @@ func TestGraphsyncRoundTripAlternatePersistenceAndNodes(t *testing.T) { progressChan, errChan := requestor.Request(ctx, td.host2.ID(), blockChain.TipLink, blockChain.Selector()) testutil.VerifyEmptyResponse(ctx, t, progressChan) - testutil.VerifySingleTerminalError(ctx, t, errChan) + testutil.VerifyHasErrors(ctx, t, errChan) progressChan, errChan = requestor.Request(ctx, td.host2.ID(), blockChain.TipLink, blockChain.Selector(), extension) diff --git a/ipldutil/ipldutil.go b/ipldutil/ipldutil.go index d431eab6..92704eb1 100644 --- a/ipldutil/ipldutil.go +++ b/ipldutil/ipldutil.go @@ -23,8 +23,8 @@ func ErrDoNotFollow() error { } var ( - defaultChooser traversal.NodeBuilderChooser = dagpb.AddDagPBSupportToChooser(func(ipld.Link, ipld.LinkContext) ipld.NodeBuilder { - return free.NodeBuilder() + defaultChooser traversal.NodeBuilderChooser = dagpb.AddDagPBSupportToChooser(func(ipld.Link, ipld.LinkContext) (ipld.NodeBuilder, error) { + return free.NodeBuilder(), nil }) ) @@ -32,7 +32,10 @@ func Traverse(ctx context.Context, loader ipld.Loader, chooser traversal.NodeBui if chooser == nil { chooser = defaultChooser } - builder := chooser(root, ipld.LinkContext{}) + builder, err := chooser(root, ipld.LinkContext{}) + if err != nil { + return err + } node, err := root.Load(ctx, ipld.LinkContext{}, builder, loader) if err != nil { return err diff --git a/ipldutil/traverser.go b/ipldutil/traverser.go new file mode 100644 index 00000000..d995e28d --- /dev/null +++ b/ipldutil/traverser.go @@ -0,0 +1,206 @@ +package ipldutil + +import ( + "context" + "errors" + "io" + + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/traversal" + "github.com/ipld/go-ipld-prime/traversal/selector" +) + +var defaultVisitor traversal.AdvVisitFn = func(traversal.Progress, ipld.Node, traversal.VisitReason) error { return nil } + +// TraversalBuilder defines parameters for an iterative traversal +type TraversalBuilder struct { + Root ipld.Link + Selector ipld.Node + Visitor traversal.AdvVisitFn + Chooser traversal.NodeBuilderChooser +} + +// Traverser is an interface for performing a selector traversal that operates iteratively -- +// it stops and waits for a manual load every time a block boundary is encountered +type Traverser interface { + // IsComplete returns the completion state (boolean) and if so, the final error result from IPLD + IsComplete() (bool, error) + // Current request returns the current link waiting to be loaded + CurrentRequest() (ipld.Link, ipld.LinkContext) + // Advance advances the traversal successfully by supplying the given reader as the result of the next IPLD load + Advance(reader io.Reader) error + // Error errors the traversal by returning the given error as the result of the next IPLD load + Error(err error) +} + +type state struct { + isDone bool + completionErr error + currentLink ipld.Link + currentContext ipld.LinkContext +} + +type nextResponse struct { + input io.Reader + err error +} + +// Start initiates the traversal (run in a go routine because the regular +// selector traversal expects a call back) +func (tb TraversalBuilder) Start(ctx context.Context) Traverser { + t := &traverser{ + ctx: ctx, + root: tb.Root, + selector: tb.Selector, + visitor: defaultVisitor, + chooser: defaultChooser, + awaitRequest: make(chan struct{}, 1), + stateChan: make(chan state, 1), + responses: make(chan nextResponse), + } + if tb.Visitor != nil { + t.visitor = tb.Visitor + } + if tb.Chooser != nil { + t.chooser = tb.Chooser + } + t.start() + return t +} + +// traverser is a class to perform a selector traversal that stops every time a new block is loaded +// and waits for manual input (in the form of advance or error) +type traverser struct { + ctx context.Context + root ipld.Link + selector ipld.Node + visitor traversal.AdvVisitFn + chooser traversal.NodeBuilderChooser + currentLink ipld.Link + currentContext ipld.LinkContext + isDone bool + completionErr error + awaitRequest chan struct{} + stateChan chan state + responses chan nextResponse +} + +func (t *traverser) checkState() { + select { + case <-t.awaitRequest: + select { + case <-t.ctx.Done(): + t.isDone = true + t.completionErr = errors.New("Context cancelled") + case newState := <-t.stateChan: + t.isDone = newState.isDone + t.completionErr = newState.completionErr + t.currentLink = newState.currentLink + t.currentContext = newState.currentContext + } + default: + } +} + +func (t *traverser) writeDone(err error) { + select { + case <-t.ctx.Done(): + case t.stateChan <- state{true, err, nil, ipld.LinkContext{}}: + } +} + +func (t *traverser) start() { + select { + case <-t.ctx.Done(): + return + case t.awaitRequest <- struct{}{}: + } + go func() { + loader := func(lnk ipld.Link, lnkCtx ipld.LinkContext) (io.Reader, error) { + select { + case <-t.ctx.Done(): + return nil, errors.New("Context cancelled") + case t.stateChan <- state{false, nil, lnk, lnkCtx}: + } + select { + case <-t.ctx.Done(): + return nil, errors.New("Context cancelled") + case response := <-t.responses: + return response.input, response.err + } + } + nb, err := t.chooser(t.root, ipld.LinkContext{}) + if err != nil { + t.writeDone(err) + return + } + nd, err := t.root.Load(t.ctx, ipld.LinkContext{}, nb, loader) + if err != nil { + t.writeDone(err) + return + } + + sel, err := selector.ParseSelector(t.selector) + if err != nil { + t.writeDone(err) + return + } + err = traversal.Progress{ + Cfg: &traversal.Config{ + Ctx: t.ctx, + LinkLoader: loader, + LinkNodeBuilderChooser: t.chooser, + }, + }.WalkAdv(nd, sel, t.visitor) + t.writeDone(err) + }() +} + +// IsComplete returns true if a traversal is complete +func (t *traverser) IsComplete() (bool, error) { + t.checkState() + return t.isDone, t.completionErr +} + +// CurrentRequest returns the current block load waiting to be fulfilled in order +// to advance further +func (t *traverser) CurrentRequest() (ipld.Link, ipld.LinkContext) { + t.checkState() + return t.currentLink, t.currentContext +} + +// Advance advances the traversal with an io.Reader for the next requested block +func (t *traverser) Advance(reader io.Reader) error { + isComplete, _ := t.IsComplete() + if isComplete { + return errors.New("cannot advance when done") + } + select { + case <-t.ctx.Done(): + return errors.New("context cancelled") + case t.awaitRequest <- struct{}{}: + } + select { + case <-t.ctx.Done(): + return errors.New("context cancelled") + case t.responses <- nextResponse{reader, nil}: + } + return nil +} + +// Error aborts the traversal with an error +func (t *traverser) Error(err error) { + isComplete, _ := t.IsComplete() + if isComplete { + return + } + select { + case <-t.ctx.Done(): + return + case t.awaitRequest <- struct{}{}: + } + select { + case <-t.ctx.Done(): + case t.responses <- nextResponse{nil, err}: + } +} diff --git a/ipldutil/traverser_test.go b/ipldutil/traverser_test.go new file mode 100644 index 00000000..e08b82fe --- /dev/null +++ b/ipldutil/traverser_test.go @@ -0,0 +1,89 @@ +package ipldutil + +import ( + "bytes" + "context" + "testing" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-graphsync" + "github.com/ipfs/go-graphsync/testutil" + ipld "github.com/ipld/go-ipld-prime" + ipldfree "github.com/ipld/go-ipld-prime/impl/free" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipld/go-ipld-prime/traversal" + "github.com/ipld/go-ipld-prime/traversal/selector" + "github.com/ipld/go-ipld-prime/traversal/selector/builder" + "github.com/stretchr/testify/require" +) + +func TestTraverser(t *testing.T) { + ctx := context.Background() + + t.Run("traverses correctly, simple struct", func(t *testing.T) { + testdata := testutil.NewTestIPLDTree() + ssb := builder.NewSelectorSpecBuilder(ipldfree.NodeBuilder()) + sel := ssb.ExploreRecursive(selector.RecursionLimitNone(), ssb.ExploreAll(ssb.ExploreRecursiveEdge())).Node() + traverser := TraversalBuilder{ + Root: testdata.RootNodeLnk, + Selector: sel, + }.Start(ctx) + checkTraverseSequence(ctx, t, traverser, []blocks.Block{ + testdata.RootBlock, + testdata.LeafAlphaBlock, + testdata.MiddleMapBlock, + testdata.LeafAlphaBlock, + testdata.MiddleListBlock, + testdata.LeafAlphaBlock, + testdata.LeafAlphaBlock, + testdata.LeafBetaBlock, + testdata.LeafAlphaBlock, + }) + }) + + t.Run("traverses correctly, blockchain", func(t *testing.T) { + store := make(map[ipld.Link][]byte) + loader, storer := testutil.NewTestStore(store) + blockChain := testutil.SetupBlockChain(ctx, t, loader, storer, 100, 10) + inProgressChan := make(chan graphsync.ResponseProgress) + done := make(chan struct{}) + traverser := TraversalBuilder{ + Root: blockChain.TipLink, + Selector: blockChain.Selector(), + Chooser: blockChain.Chooser, + Visitor: func(tp traversal.Progress, node ipld.Node, r traversal.VisitReason) error { + select { + case <-ctx.Done(): + case inProgressChan <- graphsync.ResponseProgress{ + Node: node, + Path: tp.Path, + LastBlock: tp.LastBlock, + }: + } + return nil + }, + }.Start(ctx) + go func() { + blockChain.VerifyWholeChainWithTypes(ctx, inProgressChan) + close(done) + }() + checkTraverseSequence(ctx, t, traverser, blockChain.AllBlocks()) + close(inProgressChan) + testutil.AssertDoesReceive(ctx, t, done, "should have completed verification but did not") + }) +} + +func checkTraverseSequence(ctx context.Context, t *testing.T, traverser Traverser, expectedBlks []blocks.Block) { + for _, blk := range expectedBlks { + isComplete, err := traverser.IsComplete() + require.False(t, isComplete) + require.NoError(t, err) + lnk, _ := traverser.CurrentRequest() + require.Equal(t, lnk.(cidlink.Link).Cid, blk.Cid()) + err = traverser.Advance(bytes.NewBuffer(blk.RawData())) + require.NoError(t, err) + } + isComplete, err := traverser.IsComplete() + require.True(t, isComplete) + require.NoError(t, err) +} diff --git a/requestmanager/requestmanager.go b/requestmanager/requestmanager.go index 4ee7745d..4bb82b83 100644 --- a/requestmanager/requestmanager.go +++ b/requestmanager/requestmanager.go @@ -3,7 +3,6 @@ package requestmanager import ( "context" "fmt" - "math" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-graphsync" @@ -23,8 +22,8 @@ import ( var log = logging.Logger("graphsync") const ( - // maxPriority is the max priority as defined by the bitswap protocol - maxPriority = graphsync.Priority(math.MaxInt32) + // defaultPriority is the default priority for requests sent by graphsync + defaultPriority = graphsync.Priority(0) ) type inProgressRequestStatus struct { @@ -466,7 +465,7 @@ func (rm *RequestManager) setupRequest(requestID graphsync.RequestID, p peer.ID, rm.inProgressRequestStatuses[requestID] = &inProgressRequestStatus{ ctx, cancel, p, networkErrorChan, } - request := gsmsg.NewRequest(requestID, asCidLink.Cid, selectorSpec, maxPriority, extensions...) + request := gsmsg.NewRequest(requestID, asCidLink.Cid, selectorSpec, defaultPriority, extensions...) ha := &hookActions{} for _, hook := range rm.requestHooks { hook.hook(p, request, ha) diff --git a/requestmanager/requestmanager_test.go b/requestmanager/requestmanager_test.go index 5a0137e1..85328a4f 100644 --- a/requestmanager/requestmanager_test.go +++ b/requestmanager/requestmanager_test.go @@ -207,8 +207,8 @@ func TestNormalSimultaneousFetch(t *testing.T) { require.Equal(t, peers[0], requestRecords[1].p) require.False(t, requestRecords[0].gsr.IsCancel()) require.False(t, requestRecords[1].gsr.IsCancel()) - require.Equal(t, maxPriority, requestRecords[0].gsr.Priority()) - require.Equal(t, maxPriority, requestRecords[1].gsr.Priority()) + require.Equal(t, defaultPriority, requestRecords[0].gsr.Priority()) + require.Equal(t, defaultPriority, requestRecords[1].gsr.Priority()) require.Equal(t, blockChain1.Selector(), requestRecords[0].gsr.Selector(), "did not encode selector properly") require.Equal(t, blockChain2.Selector(), requestRecords[1].gsr.Selector(), "did not encode selector properly") diff --git a/responsemanager/blockhooks/blockhooks.go b/responsemanager/blockhooks/blockhooks.go new file mode 100644 index 00000000..713fbe49 --- /dev/null +++ b/responsemanager/blockhooks/blockhooks.go @@ -0,0 +1,93 @@ +package blockhooks + +import ( + "errors" + "sync" + + "github.com/ipfs/go-graphsync" + peer "github.com/libp2p/go-libp2p-core/peer" +) + +// ErrPaused indicates a request should stop processing, but only cause it's paused +var ErrPaused = errors.New("request has been paused") + +type blockHook struct { + key uint64 + hook graphsync.OnOutgoingBlockHook +} + +// OutgoingBlockHooks is a set of outgoing block hooks that can be processed +type OutgoingBlockHooks struct { + blockHooksLk sync.RWMutex + blockHooksNextKey uint64 + blockHooks []blockHook +} + +// New returns a new list of outgoing block hooks +func New() *OutgoingBlockHooks { + return &OutgoingBlockHooks{} +} + +// Register registers an hook to process outgoing blocks in a response +func (obh *OutgoingBlockHooks) Register(hook graphsync.OnOutgoingBlockHook) graphsync.UnregisterHookFunc { + obh.blockHooksLk.Lock() + bh := blockHook{obh.blockHooksNextKey, hook} + obh.blockHooksNextKey++ + obh.blockHooks = append(obh.blockHooks, bh) + obh.blockHooksLk.Unlock() + return func() { + obh.blockHooksLk.Lock() + defer obh.blockHooksLk.Unlock() + for i, matchHook := range obh.blockHooks { + if bh.key == matchHook.key { + obh.blockHooks = append(obh.blockHooks[:i], obh.blockHooks[i+1:]...) + return + } + } + } +} + +// Result is the result of processing block hooks +type Result struct { + Err error + Extensions []graphsync.ExtensionData +} + +// ProcessBlockHooks runs block hooks against a request and block data +func (obh *OutgoingBlockHooks) ProcessBlockHooks(p peer.ID, request graphsync.RequestData, blockData graphsync.BlockData) Result { + obh.blockHooksLk.RLock() + defer obh.blockHooksLk.RUnlock() + bha := &blockHookActions{} + for _, bh := range obh.blockHooks { + bh.hook(p, request, blockData, bha) + if bha.hasError() { + break + } + } + return bha.result() +} + +type blockHookActions struct { + err error + extensions []graphsync.ExtensionData +} + +func (bha *blockHookActions) hasError() bool { + return bha.err != nil +} + +func (bha *blockHookActions) result() Result { + return Result{bha.err, bha.extensions} +} + +func (bha *blockHookActions) SendExtensionData(data graphsync.ExtensionData) { + bha.extensions = append(bha.extensions, data) +} + +func (bha *blockHookActions) TerminateWithError(err error) { + bha.err = err +} + +func (bha *blockHookActions) PauseResponse() { + bha.err = ErrPaused +} diff --git a/responsemanager/blockhooks/blookhooks_test.go b/responsemanager/blockhooks/blookhooks_test.go new file mode 100644 index 00000000..f9dcfddc --- /dev/null +++ b/responsemanager/blockhooks/blookhooks_test.go @@ -0,0 +1,200 @@ +package blockhooks_test + +import ( + "errors" + "math/rand" + "testing" + + "github.com/ipfs/go-graphsync" + gsmsg "github.com/ipfs/go-graphsync/message" + "github.com/ipfs/go-graphsync/responsemanager/blockhooks" + "github.com/ipfs/go-graphsync/testutil" + "github.com/ipld/go-ipld-prime" + ipldfree "github.com/ipld/go-ipld-prime/impl/free" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" + "github.com/ipld/go-ipld-prime/traversal/selector/builder" + peer "github.com/libp2p/go-libp2p-core/peer" + "github.com/stretchr/testify/require" +) + +type fakeBlkData struct { + link ipld.Link + size uint64 +} + +func (fbd fakeBlkData) Link() ipld.Link { + return fbd.link +} + +func (fbd fakeBlkData) BlockSize() uint64 { + return fbd.size +} + +func (fbd fakeBlkData) BlockSizeOnWire() uint64 { + return fbd.size +} + +func TestBlockHookProcessing(t *testing.T) { + extensionData := testutil.RandomBytes(100) + extensionName := graphsync.ExtensionName("AppleSauce/McGee") + extension := graphsync.ExtensionData{ + Name: extensionName, + Data: extensionData, + } + extensionResponseData := testutil.RandomBytes(100) + extensionResponse := graphsync.ExtensionData{ + Name: extensionName, + Data: extensionResponseData, + } + + root := testutil.GenerateCids(1)[0] + requestID := graphsync.RequestID(rand.Int31()) + ssb := builder.NewSelectorSpecBuilder(ipldfree.NodeBuilder()) + request := gsmsg.NewRequest(requestID, root, ssb.Matcher().Node(), graphsync.Priority(0), extension) + p := testutil.GeneratePeers(1)[0] + blockData := &fakeBlkData{ + link: cidlink.Link{Cid: testutil.GenerateCids(1)[0]}, + size: rand.Uint64(), + } + testCases := map[string]struct { + configure func(t *testing.T, blockHooks *blockhooks.OutgoingBlockHooks) + assert func(t *testing.T, result blockhooks.Result) + }{ + "no hooks": { + assert: func(t *testing.T, result blockhooks.Result) { + require.Empty(t, result.Extensions) + require.NoError(t, result.Err) + }, + }, + "send extension data": { + configure: func(t *testing.T, blockHooks *blockhooks.OutgoingBlockHooks) { + blockHooks.Register(func(p peer.ID, requestData graphsync.RequestData, blockData graphsync.BlockData, hookActions graphsync.OutgoingBlockHookActions) { + hookActions.SendExtensionData(extensionResponse) + }) + }, + assert: func(t *testing.T, result blockhooks.Result) { + require.Len(t, result.Extensions, 1) + require.Contains(t, result.Extensions, extensionResponse) + require.NoError(t, result.Err) + }, + }, + "terminate with error": { + configure: func(t *testing.T, blockHooks *blockhooks.OutgoingBlockHooks) { + blockHooks.Register(func(p peer.ID, requestData graphsync.RequestData, blockData graphsync.BlockData, hookActions graphsync.OutgoingBlockHookActions) { + hookActions.TerminateWithError(errors.New("failed")) + }) + }, + assert: func(t *testing.T, result blockhooks.Result) { + require.Empty(t, result.Extensions) + require.EqualError(t, result.Err, "failed") + }, + }, + "pause response": { + configure: func(t *testing.T, blockHooks *blockhooks.OutgoingBlockHooks) { + blockHooks.Register(func(p peer.ID, requestData graphsync.RequestData, blockData graphsync.BlockData, hookActions graphsync.OutgoingBlockHookActions) { + hookActions.PauseResponse() + }) + }, + assert: func(t *testing.T, result blockhooks.Result) { + require.Empty(t, result.Extensions) + require.EqualError(t, result.Err, blockhooks.ErrPaused.Error()) + }, + }, + } + for testCase, data := range testCases { + t.Run(testCase, func(t *testing.T) { + blockHooks := blockhooks.New() + if data.configure != nil { + data.configure(t, blockHooks) + } + result := blockHooks.ProcessBlockHooks(p, request, blockData) + if data.assert != nil { + data.assert(t, result) + } + }) + } +} + +/* + + t.Run("test block hook processing", func(t *testing.T) { + t.Run("can send extension data", func(t *testing.T) { + td := newTestData(t) + defer td.cancel() + responseManager := New(td.ctx, td.loader, td.peerManager, td.queryQueue, td.requestHooks, td.blockHooks) + responseManager.Startup() + td.requestHooks.Register(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { + hookActions.ValidateRequest() + }) + td.blockHooks.Register(func(p peer.ID, requestData graphsync.RequestData, blockData graphsync.BlockData, hookActions graphsync.OutgoingBlockHookActions) { + hookActions.SendExtensionData(td.extensionResponse) + }) + responseManager.ProcessRequests(td.ctx, td.p, td.requests) + var lastRequest completedRequest + testutil.AssertReceive(td.ctx, t, td.completedRequestChan, &lastRequest, "should complete request") + require.True(t, gsmsg.IsTerminalSuccessCode(lastRequest.result), "request should succeed") + for i := 0; i < td.blockChainLength; i++ { + var receivedExtension sentExtension + testutil.AssertReceive(td.ctx, t, td.sentExtensions, &receivedExtension, "should send extension response") + require.Equal(t, td.extensionResponse, receivedExtension.extension, "incorrect extension response sent") + } + }) + + t.Run("can send errors", func(t *testing.T) { + td := newTestData(t) + defer td.cancel() + responseManager := New(td.ctx, td.loader, td.peerManager, td.queryQueue, td.requestHooks, td.blockHooks) + responseManager.Startup() + td.requestHooks.Register(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { + hookActions.ValidateRequest() + }) + td.blockHooks.Register(func(p peer.ID, requestData graphsync.RequestData, blockData graphsync.BlockData, hookActions graphsync.OutgoingBlockHookActions) { + hookActions.TerminateWithError(errors.New("failed")) + }) + responseManager.ProcessRequests(td.ctx, td.p, td.requests) + var lastRequest completedRequest + testutil.AssertReceive(td.ctx, t, td.completedRequestChan, &lastRequest, "should complete request") + require.True(t, gsmsg.IsTerminalFailureCode(lastRequest.result), "request should succeed") + }) + + t.Run("can pause/unpause", func(t *testing.T) { + td := newTestData(t) + defer td.cancel() + responseManager := New(td.ctx, td.loader, td.peerManager, td.queryQueue, td.requestHooks, td.blockHooks) + responseManager.Startup() + td.requestHooks.Register(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { + hookActions.ValidateRequest() + }) + blkIndex := 1 + blockCount := 3 + var hasPaused bool + td.blockHooks.Register(func(p peer.ID, requestData graphsync.RequestData, blockData graphsync.BlockData, hookActions graphsync.OutgoingBlockHookActions) { + if blkIndex >= blockCount && !hasPaused { + hookActions.PauseResponse() + hasPaused = true + } + blkIndex++ + }) + responseManager.ProcessRequests(td.ctx, td.p, td.requests) + timer := time.NewTimer(500 * time.Millisecond) + testutil.AssertDoesReceiveFirst(t, timer.C, "should not complete request while paused", td.completedRequestChan) + var sentResponses []sentResponse + nomoreresponses: + for { + select { + case sentResponse := <-td.sentResponses: + sentResponses = append(sentResponses, sentResponse) + default: + break nomoreresponses + } + } + require.LessOrEqual(t, len(sentResponses), blockCount) + err := responseManager.UnpauseResponse(td.p, td.requestID) + require.NoError(t, err) + var lastRequest completedRequest + testutil.AssertReceive(td.ctx, t, td.completedRequestChan, &lastRequest, "should complete request") + require.True(t, gsmsg.IsTerminalSuccessCode(lastRequest.result), "request should succeed") + }) + + }) +*/ diff --git a/responsemanager/loader/loader.go b/responsemanager/loader/loader.go deleted file mode 100644 index 8c45f285..00000000 --- a/responsemanager/loader/loader.go +++ /dev/null @@ -1,51 +0,0 @@ -package loader - -import ( - "bytes" - "context" - "errors" - "io" - - "github.com/ipfs/go-graphsync" - "github.com/ipfs/go-graphsync/ipldutil" - ipld "github.com/ipld/go-ipld-prime" -) - -// ResponseSender sends responses over the network -type ResponseSender interface { - SendResponse( - requestID graphsync.RequestID, - link ipld.Link, - data []byte, - ) -} - -// WrapLoader wraps a given loader with an interceptor that sends loaded -// blocks out to the network with the given response sender. -func WrapLoader(ctx context.Context, - loader ipld.Loader, - requestID graphsync.RequestID, - responseSender ResponseSender) ipld.Loader { - return func(lnk ipld.Link, lnkCtx ipld.LinkContext) (io.Reader, error) { - result, err := loader(lnk, lnkCtx) - var data []byte - var blockBuffer bytes.Buffer - if err == nil { - _, err = io.Copy(&blockBuffer, result) - if err == nil { - result = &blockBuffer - data = blockBuffer.Bytes() - } - } - responseSender.SendResponse(requestID, lnk, data) - if data == nil { - err = ipldutil.ErrDoNotFollow() - } - select { - case <-ctx.Done(): - return nil, errors.New("context cancelled") - default: - } - return result, err - } -} diff --git a/responsemanager/loader/loader_test.go b/responsemanager/loader/loader_test.go deleted file mode 100644 index 0e58ec7f..00000000 --- a/responsemanager/loader/loader_test.go +++ /dev/null @@ -1,71 +0,0 @@ -package loader - -import ( - "bytes" - "context" - "fmt" - "io" - "io/ioutil" - "math/rand" - "testing" - - "github.com/ipfs/go-graphsync" - "github.com/ipfs/go-graphsync/ipldutil" - "github.com/ipfs/go-graphsync/testutil" - "github.com/stretchr/testify/require" - - ipld "github.com/ipld/go-ipld-prime" -) - -type fakeResponseSender struct { - lastRequestID graphsync.RequestID - lastLink ipld.Link - lastData []byte -} - -func (frs *fakeResponseSender) SendResponse( - requestID graphsync.RequestID, - link ipld.Link, - data []byte, -) { - frs.lastRequestID = requestID - frs.lastLink = link - frs.lastData = data -} - -func TestWrappedLoaderSendsResponses(t *testing.T) { - ctx := context.Background() - frs := &fakeResponseSender{} - link1 := testutil.NewTestLink() - link2 := testutil.NewTestLink() - sourceBytes := testutil.RandomBytes(100) - byteBuffer := bytes.NewReader(sourceBytes) - - loader := func(ipldLink ipld.Link, lnkCtx ipld.LinkContext) (io.Reader, error) { - if ipldLink == link1 { - return byteBuffer, nil - } - return nil, fmt.Errorf("unable to load block") - } - requestID := graphsync.RequestID(rand.Int31()) - wrappedLoader := WrapLoader(ctx, loader, requestID, frs) - - reader, err := wrappedLoader(link1, ipld.LinkContext{}) - require.NoError(t, err, "Should not have error if underlying loader returns valid buffer and no error") - result, err := ioutil.ReadAll(reader) - require.NoError(t, err) - require.Equal(t, sourceBytes, result, "Should return reader that functions identical to source reader") - require.Equal(t, requestID, frs.lastRequestID, "should send block to response sender with correct params") - require.Equal(t, link1, frs.lastLink, "should send block to response sender with correct params") - require.Equal(t, sourceBytes, frs.lastData, "should send block to response sender with correct params") - - reader, err = wrappedLoader(link2, ipld.LinkContext{}) - - require.Nil(t, reader, "should return empty reader") - require.Error(t, err, "should return an error") - - require.Equal(t, ipldutil.ErrDoNotFollow(), err, "Should convert error to a do not follow error") - require.Equal(t, requestID, frs.lastRequestID) - require.Equal(t, link2, frs.lastLink, "Should send metadata") - require.Nil(t, frs.lastData, "Should not send block") -} diff --git a/responsemanager/peerresponsemanager/peerresponsesender.go b/responsemanager/peerresponsemanager/peerresponsesender.go index 6321fd8f..be8d406c 100644 --- a/responsemanager/peerresponsemanager/peerresponsesender.go +++ b/responsemanager/peerresponsemanager/peerresponsesender.go @@ -21,7 +21,7 @@ import ( const ( // max block size is the maximum size for batching blocks in a single payload - maxBlockSize = 512 * 1024 + maxBlockSize uint64 = 512 * 1024 ) var log = logging.Logger("graphsync") @@ -53,7 +53,7 @@ type PeerResponseSender interface { requestID graphsync.RequestID, link ipld.Link, data []byte, - ) + ) graphsync.BlockData SendExtensionData(graphsync.RequestID, graphsync.ExtensionData) FinishRequest(requestID graphsync.RequestID) FinishWithError(requestID graphsync.RequestID, status graphsync.ResponseStatusCode) @@ -91,25 +91,45 @@ func (prm *peerResponseSender) SendExtensionData(requestID graphsync.RequestID, } } +type blockData struct { + link ipld.Link + blockSize uint64 + sendBlock bool +} + +func (bd blockData) Link() ipld.Link { + return bd.link +} + +func (bd blockData) BlockSize() uint64 { + return bd.blockSize +} + +func (bd blockData) BlockSizeOnWire() uint64 { + if !bd.sendBlock { + return 0 + } + return bd.blockSize +} + // SendResponse sends a given link for a given // requestID across the wire, as well as its corresponding // block if the block is present and has not already been sent +// it returns the number of block bytes sent func (prm *peerResponseSender) SendResponse( requestID graphsync.RequestID, link ipld.Link, data []byte, -) { +) graphsync.BlockData { hasBlock := data != nil prm.linkTrackerLk.Lock() sendBlock := hasBlock && prm.linkTracker.BlockRefCount(link) == 0 - blkSize := len(data) - if !sendBlock { - blkSize = 0 - } + blkSize := uint64(len(data)) + bd := blockData{link, blkSize, sendBlock} prm.linkTracker.RecordLinkTraversal(requestID, link, hasBlock) prm.linkTrackerLk.Unlock() - if prm.buildResponse(blkSize, func(responseBuilder *responsebuilder.ResponseBuilder) { + if prm.buildResponse(bd.BlockSizeOnWire(), func(responseBuilder *responsebuilder.ResponseBuilder) { if sendBlock { cidLink := link.(cidlink.Link) block, err := blocks.NewBlockWithCid(data, cidLink.Cid) @@ -122,6 +142,7 @@ func (prm *peerResponseSender) SendResponse( }) { prm.signalWork() } + return bd } // FinishRequest marks the given requestID as having sent all responses @@ -154,7 +175,7 @@ func (prm *peerResponseSender) finish(requestID graphsync.RequestID, status grap prm.signalWork() } } -func (prm *peerResponseSender) buildResponse(blkSize int, buildResponseFn func(*responsebuilder.ResponseBuilder)) bool { +func (prm *peerResponseSender) buildResponse(blkSize uint64, buildResponseFn func(*responsebuilder.ResponseBuilder)) bool { prm.responseBuildersLk.Lock() defer prm.responseBuildersLk.Unlock() if shouldBeginNewResponse(prm.responseBuilders, blkSize) { @@ -165,7 +186,7 @@ func (prm *peerResponseSender) buildResponse(blkSize int, buildResponseFn func(* return !responseBuilder.Empty() } -func shouldBeginNewResponse(responseBuilders []*responsebuilder.ResponseBuilder, blkSize int) bool { +func shouldBeginNewResponse(responseBuilders []*responsebuilder.ResponseBuilder, blkSize uint64) bool { if len(responseBuilders) == 0 { return true } diff --git a/responsemanager/peerresponsemanager/peerresponsesender_test.go b/responsemanager/peerresponsemanager/peerresponsesender_test.go index c63fa77c..a411d51c 100644 --- a/responsemanager/peerresponsemanager/peerresponsesender_test.go +++ b/responsemanager/peerresponsemanager/peerresponsesender_test.go @@ -54,8 +54,10 @@ func TestPeerResponseManagerSendsResponses(t *testing.T) { peerResponseManager := NewResponseSender(ctx, p, fph) peerResponseManager.Startup() - peerResponseManager.SendResponse(requestID1, links[0], blks[0].RawData()) - + bd := peerResponseManager.SendResponse(requestID1, links[0], blks[0].RawData()) + require.Equal(t, links[0], bd.Link()) + require.Equal(t, uint64(len(blks[0].RawData())), bd.BlockSize()) + require.Equal(t, uint64(len(blks[0].RawData())), bd.BlockSizeOnWire()) testutil.AssertDoesReceive(ctx, t, sent, "did not send first message") require.Len(t, fph.lastBlocks, 1) @@ -65,9 +67,18 @@ func TestPeerResponseManagerSendsResponses(t *testing.T) { require.Equal(t, requestID1, fph.lastResponses[0].RequestID()) require.Equal(t, graphsync.PartialResponse, fph.lastResponses[0].Status()) - peerResponseManager.SendResponse(requestID2, links[0], blks[0].RawData()) - peerResponseManager.SendResponse(requestID1, links[1], blks[1].RawData()) - peerResponseManager.SendResponse(requestID1, links[2], nil) + bd = peerResponseManager.SendResponse(requestID2, links[0], blks[0].RawData()) + require.Equal(t, links[0], bd.Link()) + require.Equal(t, uint64(len(blks[0].RawData())), bd.BlockSize()) + require.Equal(t, uint64(0), bd.BlockSizeOnWire()) + bd = peerResponseManager.SendResponse(requestID1, links[1], blks[1].RawData()) + require.Equal(t, links[1], bd.Link()) + require.Equal(t, uint64(len(blks[1].RawData())), bd.BlockSize()) + require.Equal(t, uint64(len(blks[1].RawData())), bd.BlockSizeOnWire()) + bd = peerResponseManager.SendResponse(requestID1, links[2], nil) + require.Equal(t, links[2], bd.Link()) + require.Equal(t, uint64(0), bd.BlockSize()) + require.Equal(t, uint64(0), bd.BlockSizeOnWire()) peerResponseManager.FinishRequest(requestID1) // let peer reponse manager know last message was sent so message sending can continue diff --git a/responsemanager/persistenceoptions/persistenceoptions.go b/responsemanager/persistenceoptions/persistenceoptions.go new file mode 100644 index 00000000..7701dc14 --- /dev/null +++ b/responsemanager/persistenceoptions/persistenceoptions.go @@ -0,0 +1,41 @@ +package persistenceoptions + +import ( + "errors" + "sync" + + "github.com/ipld/go-ipld-prime" +) + +// PersistenceOptions is a registry of loaders for persistence options +type PersistenceOptions struct { + persistenceOptionsLk sync.RWMutex + persistenceOptions map[string]ipld.Loader +} + +// New returns a new registry of persistence options +func New() *PersistenceOptions { + return &PersistenceOptions{ + persistenceOptions: make(map[string]ipld.Loader), + } +} + +// Register registers a new loader for the response manager +func (po *PersistenceOptions) Register(name string, loader ipld.Loader) error { + po.persistenceOptionsLk.Lock() + defer po.persistenceOptionsLk.Unlock() + _, ok := po.persistenceOptions[name] + if ok { + return errors.New("persistence option alreayd registered") + } + po.persistenceOptions[name] = loader + return nil +} + +// GetLoader returns the loader for the named persistence option +func (po *PersistenceOptions) GetLoader(name string) (ipld.Loader, bool) { + po.persistenceOptionsLk.RLock() + defer po.persistenceOptionsLk.RUnlock() + loader, ok := po.persistenceOptions[name] + return loader, ok +} diff --git a/responsemanager/requesthooks/requesthooks.go b/responsemanager/requesthooks/requesthooks.go new file mode 100644 index 00000000..f45ed99a --- /dev/null +++ b/responsemanager/requesthooks/requesthooks.go @@ -0,0 +1,128 @@ +package requesthooks + +import ( + "errors" + "sync" + + "github.com/ipfs/go-graphsync" + "github.com/ipld/go-ipld-prime" + "github.com/ipld/go-ipld-prime/traversal" + peer "github.com/libp2p/go-libp2p-core/peer" +) + +type requestHook struct { + key uint64 + hook graphsync.OnIncomingRequestHook +} + +// PersistenceOptions is an interface for getting loaders by name +type PersistenceOptions interface { + GetLoader(name string) (ipld.Loader, bool) +} + +// IncomingRequestHooks is a set of incoming request hooks that can be processed +type IncomingRequestHooks struct { + persistenceOptions PersistenceOptions + requestHooksLk sync.RWMutex + requestHookNextKey uint64 + requestHooks []requestHook +} + +// New returns a new list of incoming request hooks +func New(persistenceOptions PersistenceOptions) *IncomingRequestHooks { + return &IncomingRequestHooks{ + persistenceOptions: persistenceOptions, + } +} + +// Register registers an extension to process new incoming requests +func (irh *IncomingRequestHooks) Register(hook graphsync.OnIncomingRequestHook) graphsync.UnregisterHookFunc { + irh.requestHooksLk.Lock() + rh := requestHook{irh.requestHookNextKey, hook} + irh.requestHookNextKey++ + irh.requestHooks = append(irh.requestHooks, rh) + irh.requestHooksLk.Unlock() + return func() { + irh.requestHooksLk.Lock() + defer irh.requestHooksLk.Unlock() + for i, matchHook := range irh.requestHooks { + if rh.key == matchHook.key { + irh.requestHooks = append(irh.requestHooks[:i], irh.requestHooks[i+1:]...) + return + } + } + } +} + +// Result is the outcome of running requesthooks +type Result struct { + IsValidated bool + CustomLoader ipld.Loader + CustomChooser traversal.NodeBuilderChooser + Err error + Extensions []graphsync.ExtensionData +} + +// ProcessRequestHooks runs request hooks against an incoming request +func (irh *IncomingRequestHooks) ProcessRequestHooks(p peer.ID, request graphsync.RequestData) Result { + irh.requestHooksLk.RLock() + defer irh.requestHooksLk.RUnlock() + ha := &hookActions{ + persistenceOptions: irh.persistenceOptions, + } + for _, requestHook := range irh.requestHooks { + requestHook.hook(p, request, ha) + if ha.hasError() { + break + } + } + return ha.result() +} + +type hookActions struct { + persistenceOptions PersistenceOptions + isValidated bool + err error + loader ipld.Loader + chooser traversal.NodeBuilderChooser + extensions []graphsync.ExtensionData +} + +func (ha *hookActions) hasError() bool { + return ha.err != nil +} + +func (ha *hookActions) result() Result { + return Result{ + IsValidated: ha.isValidated, + CustomLoader: ha.loader, + CustomChooser: ha.chooser, + Err: ha.err, + Extensions: ha.extensions, + } +} + +func (ha *hookActions) SendExtensionData(ext graphsync.ExtensionData) { + ha.extensions = append(ha.extensions, ext) +} + +func (ha *hookActions) TerminateWithError(err error) { + ha.err = err +} + +func (ha *hookActions) ValidateRequest() { + ha.isValidated = true +} + +func (ha *hookActions) UsePersistenceOption(name string) { + loader, ok := ha.persistenceOptions.GetLoader(name) + if !ok { + ha.TerminateWithError(errors.New("unknown loader option")) + return + } + ha.loader = loader +} + +func (ha *hookActions) UseNodeBuilderChooser(chooser traversal.NodeBuilderChooser) { + ha.chooser = chooser +} diff --git a/responsemanager/requesthooks/requesthooks_test.go b/responsemanager/requesthooks/requesthooks_test.go new file mode 100644 index 00000000..79f1647c --- /dev/null +++ b/responsemanager/requesthooks/requesthooks_test.go @@ -0,0 +1,203 @@ +package requesthooks_test + +import ( + "errors" + "io" + "math/rand" + "testing" + + "github.com/ipfs/go-graphsync" + gsmsg "github.com/ipfs/go-graphsync/message" + "github.com/ipfs/go-graphsync/responsemanager/requesthooks" + "github.com/ipfs/go-graphsync/testutil" + "github.com/ipld/go-ipld-prime" + ipldfree "github.com/ipld/go-ipld-prime/impl/free" + "github.com/ipld/go-ipld-prime/traversal/selector/builder" + peer "github.com/libp2p/go-libp2p-core/peer" + "github.com/stretchr/testify/require" +) + +type fakePersistenceOptions struct { + po map[string]ipld.Loader +} + +func (fpo *fakePersistenceOptions) GetLoader(name string) (ipld.Loader, bool) { + loader, ok := fpo.po[name] + return loader, ok +} + +func TestRequestHookProcessing(t *testing.T) { + fakeChooser := func(ipld.Link, ipld.LinkContext) (ipld.NodeBuilder, error) { + return ipldfree.NodeBuilder(), nil + } + fakeLoader := func(link ipld.Link, lnkCtx ipld.LinkContext) (io.Reader, error) { + return nil, nil + } + fpo := &fakePersistenceOptions{ + po: map[string]ipld.Loader{ + "chainstore": fakeLoader, + }, + } + extensionData := testutil.RandomBytes(100) + extensionName := graphsync.ExtensionName("AppleSauce/McGee") + extension := graphsync.ExtensionData{ + Name: extensionName, + Data: extensionData, + } + extensionResponseData := testutil.RandomBytes(100) + extensionResponse := graphsync.ExtensionData{ + Name: extensionName, + Data: extensionResponseData, + } + + root := testutil.GenerateCids(1)[0] + requestID := graphsync.RequestID(rand.Int31()) + ssb := builder.NewSelectorSpecBuilder(ipldfree.NodeBuilder()) + request := gsmsg.NewRequest(requestID, root, ssb.Matcher().Node(), graphsync.Priority(0), extension) + p := testutil.GeneratePeers(1)[0] + testCases := map[string]struct { + configure func(t *testing.T, requestHooks *requesthooks.IncomingRequestHooks) + assert func(t *testing.T, result requesthooks.Result) + }{ + "no hooks": { + assert: func(t *testing.T, result requesthooks.Result) { + require.False(t, result.IsValidated) + require.Empty(t, result.Extensions) + require.Nil(t, result.CustomChooser) + require.Nil(t, result.CustomLoader) + require.NoError(t, result.Err) + }, + }, + "sending extension data, no validation": { + configure: func(t *testing.T, requestHooks *requesthooks.IncomingRequestHooks) { + requestHooks.Register(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { + hookActions.SendExtensionData(extensionResponse) + }) + }, + assert: func(t *testing.T, result requesthooks.Result) { + require.False(t, result.IsValidated) + require.Len(t, result.Extensions, 1) + require.Contains(t, result.Extensions, extensionResponse) + require.Nil(t, result.CustomChooser) + require.Nil(t, result.CustomLoader) + require.NoError(t, result.Err) + }, + }, + "sending extension data, with validation": { + configure: func(t *testing.T, requestHooks *requesthooks.IncomingRequestHooks) { + requestHooks.Register(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { + hookActions.ValidateRequest() + hookActions.SendExtensionData(extensionResponse) + }) + }, + assert: func(t *testing.T, result requesthooks.Result) { + require.True(t, result.IsValidated) + require.Len(t, result.Extensions, 1) + require.Contains(t, result.Extensions, extensionResponse) + require.Nil(t, result.CustomChooser) + require.Nil(t, result.CustomLoader) + require.NoError(t, result.Err) + }, + }, + "short circuit on error": { + configure: func(t *testing.T, requestHooks *requesthooks.IncomingRequestHooks) { + requestHooks.Register(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { + hookActions.TerminateWithError(errors.New("something went wrong")) + }) + requestHooks.Register(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { + hookActions.ValidateRequest() + hookActions.SendExtensionData(extensionResponse) + }) + }, + assert: func(t *testing.T, result requesthooks.Result) { + require.False(t, result.IsValidated) + require.Empty(t, result.Extensions) + require.Nil(t, result.CustomChooser) + require.Nil(t, result.CustomLoader) + require.EqualError(t, result.Err, "something went wrong") + }, + }, + "hooks unregistered": { + configure: func(t *testing.T, requestHooks *requesthooks.IncomingRequestHooks) { + unregister := requestHooks.Register(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { + hookActions.ValidateRequest() + hookActions.SendExtensionData(extensionResponse) + }) + unregister() + }, + assert: func(t *testing.T, result requesthooks.Result) { + require.False(t, result.IsValidated) + require.Empty(t, result.Extensions) + require.Nil(t, result.CustomChooser) + require.Nil(t, result.CustomLoader) + require.NoError(t, result.Err) + }, + }, + "hooks alter the loader": { + configure: func(t *testing.T, requestHooks *requesthooks.IncomingRequestHooks) { + requestHooks.Register(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { + if _, found := requestData.Extension(extensionName); found { + hookActions.UsePersistenceOption("chainstore") + hookActions.SendExtensionData(extensionResponse) + } + }) + }, + assert: func(t *testing.T, result requesthooks.Result) { + require.False(t, result.IsValidated) + require.Len(t, result.Extensions, 1) + require.Contains(t, result.Extensions, extensionResponse) + require.Nil(t, result.CustomChooser) + require.NotNil(t, result.CustomLoader) + require.NoError(t, result.Err) + }, + }, + "hooks alter to non-existent loader": { + configure: func(t *testing.T, requestHooks *requesthooks.IncomingRequestHooks) { + requestHooks.Register(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { + if _, found := requestData.Extension(extensionName); found { + hookActions.UsePersistenceOption("applesauce") + hookActions.SendExtensionData(extensionResponse) + } + }) + }, + assert: func(t *testing.T, result requesthooks.Result) { + require.False(t, result.IsValidated) + require.Len(t, result.Extensions, 1) + require.Contains(t, result.Extensions, extensionResponse) + require.Nil(t, result.CustomChooser) + require.Nil(t, result.CustomLoader) + require.EqualError(t, result.Err, "unknown loader option") + }, + }, + "hooks alter the node builder chooser": { + configure: func(t *testing.T, requestHooks *requesthooks.IncomingRequestHooks) { + requestHooks.Register(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { + if _, found := requestData.Extension(extensionName); found { + hookActions.UseNodeBuilderChooser(fakeChooser) + hookActions.SendExtensionData(extensionResponse) + } + }) + }, + assert: func(t *testing.T, result requesthooks.Result) { + require.False(t, result.IsValidated) + require.Len(t, result.Extensions, 1) + require.Contains(t, result.Extensions, extensionResponse) + require.NotNil(t, result.CustomChooser) + require.Nil(t, result.CustomLoader) + require.NoError(t, result.Err) + }, + }, + } + for testCase, data := range testCases { + t.Run(testCase, func(t *testing.T) { + requestHooks := requesthooks.New(fpo) + if data.configure != nil { + data.configure(t, requestHooks) + } + result := requestHooks.ProcessRequestHooks(p, request) + if data.assert != nil { + data.assert(t, result) + } + }) + } +} diff --git a/responsemanager/responsebuilder/responsebuilder.go b/responsemanager/responsebuilder/responsebuilder.go index dea8b5fa..86865ea5 100644 --- a/responsemanager/responsebuilder/responsebuilder.go +++ b/responsemanager/responsebuilder/responsebuilder.go @@ -13,7 +13,7 @@ import ( // GraphSync message components once responses are ready to send. type ResponseBuilder struct { outgoingBlocks []blocks.Block - blkSize int + blkSize uint64 completedResponses map[graphsync.RequestID]graphsync.ResponseStatusCode outgoingResponses map[graphsync.RequestID]metadata.Metadata extensions map[graphsync.RequestID][]graphsync.ExtensionData @@ -30,7 +30,7 @@ func New() *ResponseBuilder { // AddBlock adds the given block to the response. func (rb *ResponseBuilder) AddBlock(block blocks.Block) { - rb.blkSize += len(block.RawData()) + rb.blkSize += uint64(len(block.RawData())) rb.outgoingBlocks = append(rb.outgoingBlocks, block) } @@ -40,7 +40,7 @@ func (rb *ResponseBuilder) AddExtensionData(requestID graphsync.RequestID, exten } // BlockSize returns the total size of all blocks in this response -func (rb *ResponseBuilder) BlockSize() int { +func (rb *ResponseBuilder) BlockSize() uint64 { return rb.blkSize } diff --git a/responsemanager/responsebuilder/responsebuilder_test.go b/responsemanager/responsebuilder/responsebuilder_test.go index a063c27f..f537ba5e 100644 --- a/responsemanager/responsebuilder/responsebuilder_test.go +++ b/responsemanager/responsebuilder/responsebuilder_test.go @@ -47,7 +47,7 @@ func TestMessageBuilding(t *testing.T) { rb.AddBlock(block) } - require.Equal(t, 300, rb.BlockSize(), "did not calculate block size correctly") + require.Equal(t, uint64(300), rb.BlockSize(), "did not calculate block size correctly") extensionData1 := testutil.RandomBytes(100) extensionName1 := graphsync.ExtensionName("AppleSauce/McGee") diff --git a/responsemanager/responsemanager.go b/responsemanager/responsemanager.go index 5e69f9c1..d97ea8e3 100644 --- a/responsemanager/responsemanager.go +++ b/responsemanager/responsemanager.go @@ -3,30 +3,38 @@ package responsemanager import ( "context" "errors" - "sync" + "math" "time" + "github.com/ipfs/go-graphsync/responsemanager/blockhooks" + "github.com/ipfs/go-graphsync" "github.com/ipfs/go-graphsync/ipldutil" gsmsg "github.com/ipfs/go-graphsync/message" - "github.com/ipfs/go-graphsync/responsemanager/loader" "github.com/ipfs/go-graphsync/responsemanager/peerresponsemanager" + "github.com/ipfs/go-graphsync/responsemanager/requesthooks" + "github.com/ipfs/go-graphsync/responsemanager/runtraversal" + logging "github.com/ipfs/go-log" "github.com/ipfs/go-peertaskqueue/peertask" ipld "github.com/ipld/go-ipld-prime" cidlink "github.com/ipld/go-ipld-prime/linking/cid" - "github.com/ipld/go-ipld-prime/traversal" "github.com/libp2p/go-libp2p-core/peer" ) +var log = logging.Logger("graphsync") + const ( maxInProcessRequests = 6 thawSpeed = time.Millisecond * 100 ) type inProgressResponseStatus struct { - ctx context.Context - cancelFn func() - request gsmsg.GraphSyncRequest + ctx context.Context + cancelFn func() + request gsmsg.GraphSyncRequest + loader ipld.Loader + traverser ipldutil.Traverser + isPaused bool } type responseKey struct { @@ -35,13 +43,10 @@ type responseKey struct { } type responseTaskData struct { - ctx context.Context - request gsmsg.GraphSyncRequest -} - -type requestHook struct { - key uint64 - hook graphsync.OnIncomingRequestHook + ctx context.Context + request gsmsg.GraphSyncRequest + loader ipld.Loader + traverser ipldutil.Traverser } // QueryQueue is an interface that can receive new selector query tasks @@ -54,6 +59,16 @@ type QueryQueue interface { ThawRound() } +// RequestHooks is an interface for processing request hooks +type RequestHooks interface { + ProcessRequestHooks(p peer.ID, request graphsync.RequestData) requesthooks.Result +} + +// BlockHooks is an interface for processing block hooks +type BlockHooks interface { + ProcessBlockHooks(p peer.ID, request graphsync.RequestData, blockData graphsync.BlockData) blockhooks.Result +} + // PeerManager is an interface that returns sender interfaces for peer responses. type PeerManager interface { SenderForPeer(p peer.ID) peerresponsemanager.PeerResponseSender @@ -66,21 +81,18 @@ type responseManagerMessage interface { // ResponseManager handles incoming requests from the network, initiates selector // traversals, and transmits responses type ResponseManager struct { - ctx context.Context - cancelFn context.CancelFunc - loader ipld.Loader - peerManager PeerManager - queryQueue QueryQueue - - messages chan responseManagerMessage - workSignal chan struct{} - ticker *time.Ticker - inProgressResponses map[responseKey]inProgressResponseStatus - requestHooksLk sync.RWMutex - requestHookNextKey uint64 - requestHooks []requestHook - persistenceOptionsLk sync.RWMutex - persistenceOptions map[string]ipld.Loader + ctx context.Context + cancelFn context.CancelFunc + loader ipld.Loader + peerManager PeerManager + queryQueue QueryQueue + requestHooks RequestHooks + blockHooks BlockHooks + + messages chan responseManagerMessage + workSignal chan struct{} + ticker *time.Ticker + inProgressResponses map[responseKey]*inProgressResponseStatus } // New creates a new response manager from the given context, loader, @@ -88,7 +100,9 @@ type ResponseManager struct { func New(ctx context.Context, loader ipld.Loader, peerManager PeerManager, - queryQueue QueryQueue) *ResponseManager { + queryQueue QueryQueue, + requestHooks RequestHooks, + blockHooks BlockHooks) *ResponseManager { ctx, cancelFn := context.WithCancel(ctx) return &ResponseManager{ ctx: ctx, @@ -96,11 +110,12 @@ func New(ctx context.Context, loader: loader, peerManager: peerManager, queryQueue: queryQueue, + requestHooks: requestHooks, + blockHooks: blockHooks, messages: make(chan responseManagerMessage, 16), workSignal: make(chan struct{}, 1), ticker: time.NewTicker(thawSpeed), - inProgressResponses: make(map[responseKey]inProgressResponseStatus), - persistenceOptions: make(map[string]ipld.Loader), + inProgressResponses: make(map[responseKey]*inProgressResponseStatus), } } @@ -118,34 +133,25 @@ func (rm *ResponseManager) ProcessRequests(ctx context.Context, p peer.ID, reque } } -// RegisterPersistenceOption registers a new loader for the response manager -func (rm *ResponseManager) RegisterPersistenceOption(name string, loader ipld.Loader) error { - rm.persistenceOptionsLk.Lock() - defer rm.persistenceOptionsLk.Unlock() - _, ok := rm.persistenceOptions[name] - if ok { - return errors.New("persistence option alreayd registered") - } - rm.persistenceOptions[name] = loader - return nil +type unpauseRequestMessage struct { + p peer.ID + requestID graphsync.RequestID + response chan error } -// RegisterRequestHook registers an extension to process new incoming requests -func (rm *ResponseManager) RegisterRequestHook(hook graphsync.OnIncomingRequestHook) graphsync.UnregisterHookFunc { - rm.requestHooksLk.Lock() - rh := requestHook{rm.requestHookNextKey, hook} - rm.requestHookNextKey++ - rm.requestHooks = append(rm.requestHooks, rh) - rm.requestHooksLk.Unlock() - return func() { - rm.requestHooksLk.Lock() - defer rm.requestHooksLk.Unlock() - for i, matchHook := range rm.requestHooks { - if rh.key == matchHook.key { - rm.requestHooks = append(rm.requestHooks[:i], rm.requestHooks[i+1:]...) - return - } - } +// UnpauseResponse unpauses a response that was previously paused +func (rm *ResponseManager) UnpauseResponse(p peer.ID, requestID graphsync.RequestID) error { + response := make(chan error, 1) + select { + case <-rm.ctx.Done(): + return errors.New("Context Cancelled") + case rm.messages <- &unpauseRequestMessage{p, requestID, response}: + } + select { + case <-rm.ctx.Done(): + return errors.New("Context Cancelled") + case err := <-response: + return err } } @@ -171,8 +177,15 @@ type responseDataRequest struct { taskDataChan chan *responseTaskData } -type finishResponseRequest struct { +type finishTaskRequest struct { key responseKey + err error +} + +type setResponseDataRequest struct { + key responseKey + loader ipld.Loader + traverser ipldutil.Traverser } func (rm *ResponseManager) processQueriesWorker() { @@ -204,9 +217,9 @@ func (rm *ResponseManager) processQueriesWorker() { case <-rm.ctx.Done(): return } - rm.executeQuery(taskData.ctx, key.p, taskData.request) + err := rm.executeTask(key, taskData) select { - case rm.messages <- &finishResponseRequest{key}: + case rm.messages <- &finishTaskRequest{key, err}: case <-rm.ctx.Done(): } } @@ -216,81 +229,75 @@ func (rm *ResponseManager) processQueriesWorker() { } -func noopVisitor(tp traversal.Progress, n ipld.Node, tr traversal.VisitReason) error { - return nil -} - -type hookActions struct { - persistenceOptions map[string]ipld.Loader - isValidated bool - requestID graphsync.RequestID - peerResponseSender peerresponsemanager.PeerResponseSender - err error - loader ipld.Loader - chooser traversal.NodeBuilderChooser -} - -func (ha *hookActions) SendExtensionData(ext graphsync.ExtensionData) { - ha.peerResponseSender.SendExtensionData(ha.requestID, ext) -} - -func (ha *hookActions) TerminateWithError(err error) { - ha.err = err - ha.peerResponseSender.FinishWithError(ha.requestID, graphsync.RequestFailedUnknown) -} - -func (ha *hookActions) ValidateRequest() { - ha.isValidated = true -} - -func (ha *hookActions) UsePersistenceOption(name string) { - loader, ok := ha.persistenceOptions[name] - if !ok { - ha.TerminateWithError(errors.New("unknown loader option")) - return +func (rm *ResponseManager) executeTask(key responseKey, taskData *responseTaskData) error { + var err error + loader := taskData.loader + traverser := taskData.traverser + if loader == nil || traverser == nil { + loader, traverser, err = rm.prepareQuery(taskData.ctx, key.p, taskData.request) + if err != nil { + return err + } + select { + case <-rm.ctx.Done(): + return nil + case rm.messages <- &setResponseDataRequest{key, loader, traverser}: + } } - ha.loader = loader + return rm.executeQuery(key.p, taskData.request, loader, traverser) } -func (ha *hookActions) UseNodeBuilderChooser(chooser traversal.NodeBuilderChooser) { - ha.chooser = chooser -} - -func (rm *ResponseManager) executeQuery(ctx context.Context, +func (rm *ResponseManager) prepareQuery(ctx context.Context, p peer.ID, - request gsmsg.GraphSyncRequest) { + request gsmsg.GraphSyncRequest) (ipld.Loader, ipldutil.Traverser, error) { + result := rm.requestHooks.ProcessRequestHooks(p, request) peerResponseSender := rm.peerManager.SenderForPeer(p) - selectorSpec := request.Selector() - rm.requestHooksLk.RLock() - rm.persistenceOptionsLk.RLock() - ha := &hookActions{rm.persistenceOptions, false, request.ID(), peerResponseSender, nil, rm.loader, nil} - for _, requestHook := range rm.requestHooks { - requestHook.hook(p, request, ha) - if ha.err != nil { - rm.requestHooksLk.RUnlock() - rm.persistenceOptionsLk.RUnlock() - return - } + for _, extension := range result.Extensions { + peerResponseSender.SendExtensionData(request.ID(), extension) } - rm.persistenceOptionsLk.RUnlock() - rm.requestHooksLk.RUnlock() - if !ha.isValidated { + if result.Err != nil || !result.IsValidated { peerResponseSender.FinishWithError(request.ID(), graphsync.RequestFailedUnknown) - return - } - selector, err := ipldutil.ParseSelector(selectorSpec) - if err != nil { - peerResponseSender.FinishWithError(request.ID(), graphsync.RequestFailedUnknown) - return + return nil, nil, errors.New("request not valid") } rootLink := cidlink.Link{Cid: request.Root()} - wrappedLoader := loader.WrapLoader(ctx, ha.loader, request.ID(), peerResponseSender) - err = ipldutil.Traverse(ctx, wrappedLoader, ha.chooser, rootLink, selector, noopVisitor) + traverser := ipldutil.TraversalBuilder{ + Root: rootLink, + Selector: request.Selector(), + Chooser: result.CustomChooser, + }.Start(ctx) + loader := result.CustomLoader + if loader == nil { + loader = rm.loader + } + return loader, traverser, nil +} + +func (rm *ResponseManager) executeQuery(p peer.ID, + request gsmsg.GraphSyncRequest, + loader ipld.Loader, + traverser ipldutil.Traverser) error { + peerResponseSender := rm.peerManager.SenderForPeer(p) + err := runtraversal.RunTraversal(loader, traverser, func(link ipld.Link, data []byte) error { + blockData := peerResponseSender.SendResponse(request.ID(), link, data) + if blockData.BlockSize() > 0 { + result := rm.blockHooks.ProcessBlockHooks(p, request, blockData) + for _, extension := range result.Extensions { + peerResponseSender.SendExtensionData(request.ID(), extension) + } + if result.Err != nil { + return result.Err + } + } + return nil + }) if err != nil { - peerResponseSender.FinishWithError(request.ID(), graphsync.RequestFailedUnknown) - return + if err != blockhooks.ErrPaused { + peerResponseSender.FinishWithError(request.ID(), graphsync.RequestFailedUnknown) + } + return err } peerResponseSender.FinishRequest(request.ID()) + return nil } // Startup starts processing for the WantManager. @@ -331,7 +338,7 @@ func (prm *processRequestMessage) handle(rm *ResponseManager) { if !request.IsCancel() { ctx, cancelFn := context.WithCancel(rm.ctx) rm.inProgressResponses[key] = - inProgressResponseStatus{ + &inProgressResponseStatus{ ctx: ctx, cancelFn: cancelFn, request: request, @@ -356,7 +363,7 @@ func (rdr *responseDataRequest) handle(rm *ResponseManager) { response, ok := rm.inProgressResponses[rdr.key] var taskData *responseTaskData if ok { - taskData = &responseTaskData{response.ctx, response.request} + taskData = &responseTaskData{response.ctx, response.request, response.loader, response.traverser} } else { taskData = nil } @@ -366,18 +373,60 @@ func (rdr *responseDataRequest) handle(rm *ResponseManager) { } } -func (frr *finishResponseRequest) handle(rm *ResponseManager) { - response, ok := rm.inProgressResponses[frr.key] +func (ftr *finishTaskRequest) handle(rm *ResponseManager) { + response, ok := rm.inProgressResponses[ftr.key] if !ok { return } - delete(rm.inProgressResponses, frr.key) + if ftr.err == blockhooks.ErrPaused { + response.isPaused = true + return + } + if ftr.err != nil { + log.Infof("response failed: %w", ftr.err) + } + delete(rm.inProgressResponses, ftr.key) response.cancelFn() } +func (srdr *setResponseDataRequest) handle(rm *ResponseManager) { + response, ok := rm.inProgressResponses[srdr.key] + if !ok { + return + } + response.loader = srdr.loader + response.traverser = srdr.traverser +} + func (sm *synchronizeMessage) handle(rm *ResponseManager) { select { case <-rm.ctx.Done(): case sm.sync <- struct{}{}: } } + +func (urm *unpauseRequestMessage) unpauseRequest(rm *ResponseManager) error { + key := responseKey{urm.p, urm.requestID} + inProgressResponse, ok := rm.inProgressResponses[key] + if !ok { + return errors.New("could not find request") + } + if !inProgressResponse.isPaused { + return errors.New("request is not paused") + } + inProgressResponse.isPaused = false + rm.queryQueue.PushTasks(urm.p, peertask.Task{Topic: key, Priority: math.MaxInt32, Work: 1}) + select { + case rm.workSignal <- struct{}{}: + default: + } + return nil +} + +func (urm *unpauseRequestMessage) handle(rm *ResponseManager) { + err := urm.unpauseRequest(rm) + select { + case <-rm.ctx.Done(): + case urm.response <- err: + } +} diff --git a/responsemanager/responsemanager_test.go b/responsemanager/responsemanager_test.go index 2e64a21d..3dadf223 100644 --- a/responsemanager/responsemanager_test.go +++ b/responsemanager/responsemanager_test.go @@ -3,7 +3,6 @@ package responsemanager import ( "context" "errors" - "math" "math/rand" "sync" "testing" @@ -11,7 +10,10 @@ import ( "github.com/ipfs/go-graphsync" gsmsg "github.com/ipfs/go-graphsync/message" + "github.com/ipfs/go-graphsync/responsemanager/blockhooks" "github.com/ipfs/go-graphsync/responsemanager/peerresponsemanager" + "github.com/ipfs/go-graphsync/responsemanager/persistenceoptions" + "github.com/ipfs/go-graphsync/responsemanager/requesthooks" "github.com/ipfs/go-graphsync/selectorvalidator" "github.com/ipfs/go-graphsync/testutil" "github.com/ipfs/go-peertaskqueue/peertask" @@ -104,12 +106,30 @@ type fakePeerResponseSender struct { func (fprs *fakePeerResponseSender) Startup() {} func (fprs *fakePeerResponseSender) Shutdown() {} +type fakeBlkData struct { + link ipld.Link + size uint64 +} + +func (fbd fakeBlkData) Link() ipld.Link { + return fbd.link +} + +func (fbd fakeBlkData) BlockSize() uint64 { + return fbd.size +} + +func (fbd fakeBlkData) BlockSizeOnWire() uint64 { + return fbd.size +} + func (fprs *fakePeerResponseSender) SendResponse( requestID graphsync.RequestID, link ipld.Link, data []byte, -) { +) graphsync.BlockData { fprs.sentResponses <- sentResponse{requestID, link, data} + return fakeBlkData{link, uint64(len(data))} } func (fprs *fakePeerResponseSender) SendExtensionData( @@ -128,332 +148,416 @@ func (fprs *fakePeerResponseSender) FinishWithError(requestID graphsync.RequestI } func TestIncomingQuery(t *testing.T) { - ctx := context.Background() - ctx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - - blockStore := make(map[ipld.Link][]byte) - loader, storer := testutil.NewTestStore(blockStore) - blockChain := testutil.SetupBlockChain(ctx, t, loader, storer, 100, 5) - blks := blockChain.AllBlocks() - - requestIDChan := make(chan completedRequest, 1) - sentResponses := make(chan sentResponse, len(blks)) - sentExtensions := make(chan sentExtension, 1) - fprs := &fakePeerResponseSender{lastCompletedRequest: requestIDChan, sentResponses: sentResponses, sentExtensions: sentExtensions} - peerManager := &fakePeerManager{peerResponseSender: fprs} - queryQueue := &fakeQueryQueue{} - responseManager := New(ctx, loader, peerManager, queryQueue) - responseManager.RegisterRequestHook(selectorvalidator.SelectorValidator(100)) + td := newTestData(t) + defer td.cancel() + blks := td.blockChain.AllBlocks() + + responseManager := New(td.ctx, td.loader, td.peerManager, td.queryQueue, td.requestHooks, td.blockHooks) + td.requestHooks.Register(selectorvalidator.SelectorValidator(100)) responseManager.Startup() - requestID := graphsync.RequestID(rand.Int31()) - requests := []gsmsg.GraphSyncRequest{ - gsmsg.NewRequest(requestID, blockChain.TipLink.(cidlink.Link).Cid, blockChain.Selector(), graphsync.Priority(math.MaxInt32)), - } - p := testutil.GeneratePeers(1)[0] - responseManager.ProcessRequests(ctx, p, requests) - testutil.AssertDoesReceive(ctx, t, requestIDChan, "Should have completed request but didn't") + responseManager.ProcessRequests(td.ctx, td.p, td.requests) + testutil.AssertDoesReceive(td.ctx, t, td.completedRequestChan, "Should have completed request but didn't") for i := 0; i < len(blks); i++ { var sentResponse sentResponse - testutil.AssertReceive(ctx, t, sentResponses, &sentResponse, "did not send responses") + testutil.AssertReceive(td.ctx, t, td.sentResponses, &sentResponse, "did not send responses") k := sentResponse.link.(cidlink.Link) blockIndex := testutil.IndexOf(blks, k.Cid) require.NotEqual(t, blockIndex, -1, "sent incorrect link") require.Equal(t, blks[blockIndex].RawData(), sentResponse.data, "sent incorrect data") - require.Equal(t, requestID, sentResponse.requestID, "has incorrect response id") + require.Equal(t, td.requestID, sentResponse.requestID, "has incorrect response id") } } func TestCancellationQueryInProgress(t *testing.T) { - ctx := context.Background() - ctx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - - blockStore := make(map[ipld.Link][]byte) - loader, storer := testutil.NewTestStore(blockStore) - blockChain := testutil.SetupBlockChain(ctx, t, loader, storer, 100, 5) - blks := blockChain.AllBlocks() - - requestIDChan := make(chan completedRequest) - sentResponses := make(chan sentResponse) - sentExtensions := make(chan sentExtension, 1) - fprs := &fakePeerResponseSender{lastCompletedRequest: requestIDChan, sentResponses: sentResponses, sentExtensions: sentExtensions} - peerManager := &fakePeerManager{peerResponseSender: fprs} - queryQueue := &fakeQueryQueue{} - responseManager := New(ctx, loader, peerManager, queryQueue) - responseManager.RegisterRequestHook(selectorvalidator.SelectorValidator(100)) + td := newTestData(t) + defer td.cancel() + blks := td.blockChain.AllBlocks() + responseManager := New(td.ctx, td.loader, td.peerManager, td.queryQueue, td.requestHooks, td.blockHooks) + td.requestHooks.Register(selectorvalidator.SelectorValidator(100)) responseManager.Startup() - - requestID := graphsync.RequestID(rand.Int31()) - requests := []gsmsg.GraphSyncRequest{ - gsmsg.NewRequest(requestID, blockChain.TipLink.(cidlink.Link).Cid, blockChain.Selector(), graphsync.Priority(math.MaxInt32)), - } - p := testutil.GeneratePeers(1)[0] - responseManager.ProcessRequests(ctx, p, requests) + responseManager.ProcessRequests(td.ctx, td.p, td.requests) // read one block var sentResponse sentResponse - testutil.AssertReceive(ctx, t, sentResponses, &sentResponse, "did not send response") + testutil.AssertReceive(td.ctx, t, td.sentResponses, &sentResponse, "did not send response") k := sentResponse.link.(cidlink.Link) blockIndex := testutil.IndexOf(blks, k.Cid) require.NotEqual(t, blockIndex, -1, "sent incorrect link") require.Equal(t, blks[blockIndex].RawData(), sentResponse.data, "sent incorrect data") - require.Equal(t, requestID, sentResponse.requestID, "has incorrect response id") + require.Equal(t, td.requestID, sentResponse.requestID, "has incorrect response id") // send a cancellation - requests = []gsmsg.GraphSyncRequest{ - gsmsg.CancelRequest(requestID), + cancelRequests := []gsmsg.GraphSyncRequest{ + gsmsg.CancelRequest(td.requestID), } - responseManager.ProcessRequests(ctx, p, requests) + responseManager.ProcessRequests(td.ctx, td.p, cancelRequests) responseManager.synchronize() // at this point we should receive at most one more block, then traversal // should complete - testutil.AssertReceiveFirst(t, sentResponses, &sentResponse, "should send one additional response", ctx.Done(), requestIDChan) - k = sentResponse.link.(cidlink.Link) - blockIndex = testutil.IndexOf(blks, k.Cid) - require.NotEqual(t, blockIndex, -1, "did not send correct link") - require.Equal(t, blks[blockIndex].RawData(), sentResponse.data, "sent incorrect data") - require.Equal(t, requestID, sentResponse.requestID, "incorrect response id") - - // We should now be done - testutil.AssertDoesReceiveFirst(t, requestIDChan, "should complete request", ctx.Done(), sentResponses) + additionalBlocks := 0 + for { + select { + case <-td.ctx.Done(): + t.Fatal("should complete request before context closes") + case sentResponse = <-td.sentResponses: + k = sentResponse.link.(cidlink.Link) + blockIndex = testutil.IndexOf(blks, k.Cid) + require.NotEqual(t, blockIndex, -1, "did not send correct link") + require.Equal(t, blks[blockIndex].RawData(), sentResponse.data, "sent incorrect data") + require.Equal(t, td.requestID, sentResponse.requestID, "incorrect response id") + additionalBlocks++ + case <-td.completedRequestChan: + require.LessOrEqual(t, additionalBlocks, 1, "should send at most 1 additional block") + return + } + } } func TestEarlyCancellation(t *testing.T) { - ctx := context.Background() - ctx, cancel := context.WithTimeout(ctx, 1*time.Second) - defer cancel() - - blockStore := make(map[ipld.Link][]byte) - loader, storer := testutil.NewTestStore(blockStore) - blockChain := testutil.SetupBlockChain(ctx, t, loader, storer, 100, 5) - - requestIDChan := make(chan completedRequest) - sentResponses := make(chan sentResponse) - sentExtensions := make(chan sentExtension, 1) - fprs := &fakePeerResponseSender{lastCompletedRequest: requestIDChan, sentResponses: sentResponses, sentExtensions: sentExtensions} - peerManager := &fakePeerManager{peerResponseSender: fprs} - queryQueue := &fakeQueryQueue{} - queryQueue.popWait.Add(1) - responseManager := New(ctx, loader, peerManager, queryQueue) + td := newTestData(t) + defer td.cancel() + td.queryQueue.popWait.Add(1) + responseManager := New(td.ctx, td.loader, td.peerManager, td.queryQueue, td.requestHooks, td.blockHooks) responseManager.Startup() - - requestID := graphsync.RequestID(rand.Int31()) - requests := []gsmsg.GraphSyncRequest{ - gsmsg.NewRequest(requestID, blockChain.TipLink.(cidlink.Link).Cid, blockChain.Selector(), graphsync.Priority(math.MaxInt32)), - } - p := testutil.GeneratePeers(1)[0] - responseManager.ProcessRequests(ctx, p, requests) + responseManager.ProcessRequests(td.ctx, td.p, td.requests) // send a cancellation - requests = []gsmsg.GraphSyncRequest{ - gsmsg.CancelRequest(requestID), + cancelRequests := []gsmsg.GraphSyncRequest{ + gsmsg.CancelRequest(td.requestID), } - responseManager.ProcessRequests(ctx, p, requests) + responseManager.ProcessRequests(td.ctx, td.p, cancelRequests) responseManager.synchronize() // unblock popping from queue - queryQueue.popWait.Done() + td.queryQueue.popWait.Done() + timer := time.NewTimer(time.Second) // verify no responses processed - testutil.AssertDoesReceiveFirst(t, ctx.Done(), "should not process more responses", sentResponses, requestIDChan) + testutil.AssertDoesReceiveFirst(t, timer.C, "should not process more responses", td.sentResponses, td.completedRequestChan) } func TestValidationAndExtensions(t *testing.T) { - ctx := context.Background() - ctx, cancel := context.WithTimeout(ctx, 10*time.Second) - defer cancel() - - blockStore := make(map[ipld.Link][]byte) - loader, storer := testutil.NewTestStore(blockStore) - blockChain := testutil.SetupBlockChain(ctx, t, loader, storer, 100, 5) - - completedRequestChan := make(chan completedRequest, 1) - sentResponses := make(chan sentResponse, 100) - sentExtensions := make(chan sentExtension, 1) - fprs := &fakePeerResponseSender{lastCompletedRequest: completedRequestChan, sentResponses: sentResponses, sentExtensions: sentExtensions} - peerManager := &fakePeerManager{peerResponseSender: fprs} - queryQueue := &fakeQueryQueue{} - - extensionData := testutil.RandomBytes(100) - extensionName := graphsync.ExtensionName("AppleSauce/McGee") - extension := graphsync.ExtensionData{ - Name: extensionName, - Data: extensionData, - } - extensionResponseData := testutil.RandomBytes(100) - extensionResponse := graphsync.ExtensionData{ - Name: extensionName, - Data: extensionResponseData, - } - - requestID := graphsync.RequestID(rand.Int31()) - requests := []gsmsg.GraphSyncRequest{ - gsmsg.NewRequest(requestID, blockChain.TipLink.(cidlink.Link).Cid, blockChain.Selector(), graphsync.Priority(math.MaxInt32), extension), - } - p := testutil.GeneratePeers(1)[0] - t.Run("on its own, should fail validation", func(t *testing.T) { - responseManager := New(ctx, loader, peerManager, queryQueue) + td := newTestData(t) + defer td.cancel() + responseManager := New(td.ctx, td.loader, td.peerManager, td.queryQueue, td.requestHooks, td.blockHooks) responseManager.Startup() - responseManager.ProcessRequests(ctx, p, requests) + responseManager.ProcessRequests(td.ctx, td.p, td.requests) var lastRequest completedRequest - testutil.AssertReceive(ctx, t, completedRequestChan, &lastRequest, "should complete request") + testutil.AssertReceive(td.ctx, t, td.completedRequestChan, &lastRequest, "should complete request") require.True(t, gsmsg.IsTerminalFailureCode(lastRequest.result), "should terminate with failure") }) t.Run("if non validating hook succeeds, does not pass validation", func(t *testing.T) { - responseManager := New(ctx, loader, peerManager, queryQueue) + td := newTestData(t) + defer td.cancel() + responseManager := New(td.ctx, td.loader, td.peerManager, td.queryQueue, td.requestHooks, td.blockHooks) responseManager.Startup() - responseManager.RegisterRequestHook(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { - hookActions.SendExtensionData(extensionResponse) + td.requestHooks.Register(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { + hookActions.SendExtensionData(td.extensionResponse) }) - responseManager.ProcessRequests(ctx, p, requests) + responseManager.ProcessRequests(td.ctx, td.p, td.requests) var lastRequest completedRequest - testutil.AssertReceive(ctx, t, completedRequestChan, &lastRequest, "should complete request") + testutil.AssertReceive(td.ctx, t, td.completedRequestChan, &lastRequest, "should complete request") require.True(t, gsmsg.IsTerminalFailureCode(lastRequest.result), "should terminate with failure") var receivedExtension sentExtension - testutil.AssertReceive(ctx, t, sentExtensions, &receivedExtension, "should send extension response") - require.Equal(t, extensionResponse, receivedExtension.extension, "incorrect extension response sent") + testutil.AssertReceive(td.ctx, t, td.sentExtensions, &receivedExtension, "should send extension response") + require.Equal(t, td.extensionResponse, receivedExtension.extension, "incorrect extension response sent") }) t.Run("if validating hook succeeds, should pass validation", func(t *testing.T) { - responseManager := New(ctx, loader, peerManager, queryQueue) + td := newTestData(t) + defer td.cancel() + responseManager := New(td.ctx, td.loader, td.peerManager, td.queryQueue, td.requestHooks, td.blockHooks) responseManager.Startup() - responseManager.RegisterRequestHook(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { + td.requestHooks.Register(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { hookActions.ValidateRequest() - hookActions.SendExtensionData(extensionResponse) + hookActions.SendExtensionData(td.extensionResponse) }) - responseManager.ProcessRequests(ctx, p, requests) + responseManager.ProcessRequests(td.ctx, td.p, td.requests) var lastRequest completedRequest - testutil.AssertReceive(ctx, t, completedRequestChan, &lastRequest, "should complete request") + testutil.AssertReceive(td.ctx, t, td.completedRequestChan, &lastRequest, "should complete request") require.True(t, gsmsg.IsTerminalSuccessCode(lastRequest.result), "request should succeed") var receivedExtension sentExtension - testutil.AssertReceive(ctx, t, sentExtensions, &receivedExtension, "should send extension response") - require.Equal(t, extensionResponse, receivedExtension.extension, "incorrect extension response sent") + testutil.AssertReceive(td.ctx, t, td.sentExtensions, &receivedExtension, "should send extension response") + require.Equal(t, td.extensionResponse, receivedExtension.extension, "incorrect extension response sent") }) t.Run("if any hook fails, should fail", func(t *testing.T) { - responseManager := New(ctx, loader, peerManager, queryQueue) + td := newTestData(t) + defer td.cancel() + responseManager := New(td.ctx, td.loader, td.peerManager, td.queryQueue, td.requestHooks, td.blockHooks) responseManager.Startup() - responseManager.RegisterRequestHook(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { + td.requestHooks.Register(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { hookActions.ValidateRequest() }) - responseManager.RegisterRequestHook(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { - hookActions.SendExtensionData(extensionResponse) + td.requestHooks.Register(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { + hookActions.SendExtensionData(td.extensionResponse) hookActions.TerminateWithError(errors.New("everything went to crap")) }) - responseManager.ProcessRequests(ctx, p, requests) + responseManager.ProcessRequests(td.ctx, td.p, td.requests) var lastRequest completedRequest - testutil.AssertReceive(ctx, t, completedRequestChan, &lastRequest, "should complete request") + testutil.AssertReceive(td.ctx, t, td.completedRequestChan, &lastRequest, "should complete request") require.True(t, gsmsg.IsTerminalFailureCode(lastRequest.result), "should terminate with failure") var receivedExtension sentExtension - testutil.AssertReceive(ctx, t, sentExtensions, &receivedExtension, "should send extension response") - require.Equal(t, extensionResponse, receivedExtension.extension, "incorrect extension response sent") + testutil.AssertReceive(td.ctx, t, td.sentExtensions, &receivedExtension, "should send extension response") + require.Equal(t, td.extensionResponse, receivedExtension.extension, "incorrect extension response sent") }) t.Run("hooks can be unregistered", func(t *testing.T) { - responseManager := New(ctx, loader, peerManager, queryQueue) + td := newTestData(t) + defer td.cancel() + responseManager := New(td.ctx, td.loader, td.peerManager, td.queryQueue, td.requestHooks, td.blockHooks) responseManager.Startup() - unregister := responseManager.RegisterRequestHook(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { + unregister := td.requestHooks.Register(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { hookActions.ValidateRequest() - hookActions.SendExtensionData(extensionResponse) + hookActions.SendExtensionData(td.extensionResponse) }) // hook validates request - responseManager.ProcessRequests(ctx, p, requests) + responseManager.ProcessRequests(td.ctx, td.p, td.requests) var lastRequest completedRequest - testutil.AssertReceive(ctx, t, completedRequestChan, &lastRequest, "should complete request") + testutil.AssertReceive(td.ctx, t, td.completedRequestChan, &lastRequest, "should complete request") require.True(t, gsmsg.IsTerminalSuccessCode(lastRequest.result), "request should succeed") var receivedExtension sentExtension - testutil.AssertReceive(ctx, t, sentExtensions, &receivedExtension, "should send extension response") - require.Equal(t, extensionResponse, receivedExtension.extension, "incorrect extension response sent") + testutil.AssertReceive(td.ctx, t, td.sentExtensions, &receivedExtension, "should send extension response") + require.Equal(t, td.extensionResponse, receivedExtension.extension, "incorrect extension response sent") // unregister unregister() // no same request should fail - responseManager.ProcessRequests(ctx, p, requests) - testutil.AssertReceive(ctx, t, completedRequestChan, &lastRequest, "should complete request") + responseManager.ProcessRequests(td.ctx, td.p, td.requests) + testutil.AssertReceive(td.ctx, t, td.completedRequestChan, &lastRequest, "should complete request") require.True(t, gsmsg.IsTerminalFailureCode(lastRequest.result), "should terminate with failure") }) t.Run("hooks can alter the loader", func(t *testing.T) { + td := newTestData(t) + defer td.cancel() obs := make(map[ipld.Link][]byte) oloader, _ := testutil.NewTestStore(obs) - responseManager := New(ctx, oloader, peerManager, queryQueue) + responseManager := New(td.ctx, oloader, td.peerManager, td.queryQueue, td.requestHooks, td.blockHooks) responseManager.Startup() // add validating hook -- so the request SHOULD succeed - responseManager.RegisterRequestHook(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { + td.requestHooks.Register(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { hookActions.ValidateRequest() }) // request fails with base loader reading from block store that's missing data var lastRequest completedRequest - responseManager.ProcessRequests(ctx, p, requests) - testutil.AssertReceive(ctx, t, completedRequestChan, &lastRequest, "should complete request") + responseManager.ProcessRequests(td.ctx, td.p, td.requests) + testutil.AssertReceive(td.ctx, t, td.completedRequestChan, &lastRequest, "should complete request") require.True(t, gsmsg.IsTerminalFailureCode(lastRequest.result), "should terminate with failure") - err := responseManager.RegisterPersistenceOption("chainstore", loader) + err := td.peristenceOptions.Register("chainstore", td.loader) require.NoError(t, err) // register hook to use different loader - _ = responseManager.RegisterRequestHook(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { - if _, found := requestData.Extension(extensionName); found { + _ = td.requestHooks.Register(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { + if _, found := requestData.Extension(td.extensionName); found { hookActions.UsePersistenceOption("chainstore") - hookActions.SendExtensionData(extensionResponse) + hookActions.SendExtensionData(td.extensionResponse) } }) - // hook uses different loader that should make request succeed - responseManager.ProcessRequests(ctx, p, requests) - testutil.AssertReceive(ctx, t, completedRequestChan, &lastRequest, "should complete request") + responseManager.ProcessRequests(td.ctx, td.p, td.requests) + testutil.AssertReceive(td.ctx, t, td.completedRequestChan, &lastRequest, "should complete request") require.True(t, gsmsg.IsTerminalSuccessCode(lastRequest.result), "request should succeed") var receivedExtension sentExtension - testutil.AssertReceive(ctx, t, sentExtensions, &receivedExtension, "should send extension response") - require.Equal(t, extensionResponse, receivedExtension.extension, "incorrect extension response sent") + testutil.AssertReceive(td.ctx, t, td.sentExtensions, &receivedExtension, "should send extension response") + require.Equal(t, td.extensionResponse, receivedExtension.extension, "incorrect extension response sent") }) t.Run("hooks can alter the node builder chooser", func(t *testing.T) { - responseManager := New(ctx, loader, peerManager, queryQueue) + td := newTestData(t) + defer td.cancel() + responseManager := New(td.ctx, td.loader, td.peerManager, td.queryQueue, td.requestHooks, td.blockHooks) responseManager.Startup() customChooserCallCount := 0 - customChooser := func(ipld.Link, ipld.LinkContext) ipld.NodeBuilder { + customChooser := func(ipld.Link, ipld.LinkContext) (ipld.NodeBuilder, error) { customChooserCallCount++ - return ipldfree.NodeBuilder() + return ipldfree.NodeBuilder(), nil } // add validating hook -- so the request SHOULD succeed - responseManager.RegisterRequestHook(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { + td.requestHooks.Register(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { hookActions.ValidateRequest() }) // with default chooser, customer chooser not called var lastRequest completedRequest - responseManager.ProcessRequests(ctx, p, requests) - testutil.AssertReceive(ctx, t, completedRequestChan, &lastRequest, "should complete request") + responseManager.ProcessRequests(td.ctx, td.p, td.requests) + testutil.AssertReceive(td.ctx, t, td.completedRequestChan, &lastRequest, "should complete request") require.True(t, gsmsg.IsTerminalSuccessCode(lastRequest.result), "request should succeed") require.Equal(t, 0, customChooserCallCount) // register hook to use custom chooser - _ = responseManager.RegisterRequestHook(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { - if _, found := requestData.Extension(extensionName); found { + _ = td.requestHooks.Register(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { + if _, found := requestData.Extension(td.extensionName); found { hookActions.UseNodeBuilderChooser(customChooser) - hookActions.SendExtensionData(extensionResponse) + hookActions.SendExtensionData(td.extensionResponse) } }) // verify now that request succeeds and uses custom chooser - responseManager.ProcessRequests(ctx, p, requests) - testutil.AssertReceive(ctx, t, completedRequestChan, &lastRequest, "should complete request") + responseManager.ProcessRequests(td.ctx, td.p, td.requests) + testutil.AssertReceive(td.ctx, t, td.completedRequestChan, &lastRequest, "should complete request") require.True(t, gsmsg.IsTerminalSuccessCode(lastRequest.result), "request should succeed") var receivedExtension sentExtension - testutil.AssertReceive(ctx, t, sentExtensions, &receivedExtension, "should send extension response") - require.Equal(t, extensionResponse, receivedExtension.extension, "incorrect extension response sent") + testutil.AssertReceive(td.ctx, t, td.sentExtensions, &receivedExtension, "should send extension response") + require.Equal(t, td.extensionResponse, receivedExtension.extension, "incorrect extension response sent") require.Equal(t, 5, customChooserCallCount) }) + + t.Run("test block hook processing", func(t *testing.T) { + t.Run("can send extension data", func(t *testing.T) { + td := newTestData(t) + defer td.cancel() + responseManager := New(td.ctx, td.loader, td.peerManager, td.queryQueue, td.requestHooks, td.blockHooks) + responseManager.Startup() + td.requestHooks.Register(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { + hookActions.ValidateRequest() + }) + td.blockHooks.Register(func(p peer.ID, requestData graphsync.RequestData, blockData graphsync.BlockData, hookActions graphsync.OutgoingBlockHookActions) { + hookActions.SendExtensionData(td.extensionResponse) + }) + responseManager.ProcessRequests(td.ctx, td.p, td.requests) + var lastRequest completedRequest + testutil.AssertReceive(td.ctx, t, td.completedRequestChan, &lastRequest, "should complete request") + require.True(t, gsmsg.IsTerminalSuccessCode(lastRequest.result), "request should succeed") + for i := 0; i < td.blockChainLength; i++ { + var receivedExtension sentExtension + testutil.AssertReceive(td.ctx, t, td.sentExtensions, &receivedExtension, "should send extension response") + require.Equal(t, td.extensionResponse, receivedExtension.extension, "incorrect extension response sent") + } + }) + + t.Run("can send errors", func(t *testing.T) { + td := newTestData(t) + defer td.cancel() + responseManager := New(td.ctx, td.loader, td.peerManager, td.queryQueue, td.requestHooks, td.blockHooks) + responseManager.Startup() + td.requestHooks.Register(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { + hookActions.ValidateRequest() + }) + td.blockHooks.Register(func(p peer.ID, requestData graphsync.RequestData, blockData graphsync.BlockData, hookActions graphsync.OutgoingBlockHookActions) { + hookActions.TerminateWithError(errors.New("failed")) + }) + responseManager.ProcessRequests(td.ctx, td.p, td.requests) + var lastRequest completedRequest + testutil.AssertReceive(td.ctx, t, td.completedRequestChan, &lastRequest, "should complete request") + require.True(t, gsmsg.IsTerminalFailureCode(lastRequest.result), "request should succeed") + }) + + t.Run("can pause/unpause", func(t *testing.T) { + td := newTestData(t) + defer td.cancel() + responseManager := New(td.ctx, td.loader, td.peerManager, td.queryQueue, td.requestHooks, td.blockHooks) + responseManager.Startup() + td.requestHooks.Register(func(p peer.ID, requestData graphsync.RequestData, hookActions graphsync.IncomingRequestHookActions) { + hookActions.ValidateRequest() + }) + blkIndex := 1 + blockCount := 3 + var hasPaused bool + td.blockHooks.Register(func(p peer.ID, requestData graphsync.RequestData, blockData graphsync.BlockData, hookActions graphsync.OutgoingBlockHookActions) { + if blkIndex >= blockCount && !hasPaused { + hookActions.PauseResponse() + hasPaused = true + } + blkIndex++ + }) + responseManager.ProcessRequests(td.ctx, td.p, td.requests) + timer := time.NewTimer(500 * time.Millisecond) + testutil.AssertDoesReceiveFirst(t, timer.C, "should not complete request while paused", td.completedRequestChan) + var sentResponses []sentResponse + nomoreresponses: + for { + select { + case sentResponse := <-td.sentResponses: + sentResponses = append(sentResponses, sentResponse) + default: + break nomoreresponses + } + } + require.LessOrEqual(t, len(sentResponses), blockCount) + err := responseManager.UnpauseResponse(td.p, td.requestID) + require.NoError(t, err) + var lastRequest completedRequest + testutil.AssertReceive(td.ctx, t, td.completedRequestChan, &lastRequest, "should complete request") + require.True(t, gsmsg.IsTerminalSuccessCode(lastRequest.result), "request should succeed") + }) + + }) +} + +type testData struct { + ctx context.Context + cancel func() + blockStore map[ipld.Link][]byte + loader ipld.Loader + storer ipld.Storer + blockChainLength int + blockChain *testutil.TestBlockChain + completedRequestChan chan completedRequest + sentResponses chan sentResponse + sentExtensions chan sentExtension + peerManager *fakePeerManager + queryQueue *fakeQueryQueue + extensionData []byte + extensionName graphsync.ExtensionName + extension graphsync.ExtensionData + extensionResponseData []byte + extensionResponse graphsync.ExtensionData + requestID graphsync.RequestID + requests []gsmsg.GraphSyncRequest + p peer.ID + peristenceOptions *persistenceoptions.PersistenceOptions + requestHooks *requesthooks.IncomingRequestHooks + blockHooks *blockhooks.OutgoingBlockHooks +} + +func newTestData(t *testing.T) testData { + ctx := context.Background() + td := testData{} + td.ctx, td.cancel = context.WithTimeout(ctx, 10*time.Second) + + td.blockStore = make(map[ipld.Link][]byte) + td.loader, td.storer = testutil.NewTestStore(td.blockStore) + td.blockChainLength = 5 + td.blockChain = testutil.SetupBlockChain(ctx, t, td.loader, td.storer, 100, td.blockChainLength) + + td.completedRequestChan = make(chan completedRequest, 1) + td.sentResponses = make(chan sentResponse, td.blockChainLength*2) + td.sentExtensions = make(chan sentExtension, td.blockChainLength*2) + fprs := &fakePeerResponseSender{lastCompletedRequest: td.completedRequestChan, sentResponses: td.sentResponses, sentExtensions: td.sentExtensions} + td.peerManager = &fakePeerManager{peerResponseSender: fprs} + td.queryQueue = &fakeQueryQueue{} + + td.extensionData = testutil.RandomBytes(100) + td.extensionName = graphsync.ExtensionName("AppleSauce/McGee") + td.extension = graphsync.ExtensionData{ + Name: td.extensionName, + Data: td.extensionData, + } + td.extensionResponseData = testutil.RandomBytes(100) + td.extensionResponse = graphsync.ExtensionData{ + Name: td.extensionName, + Data: td.extensionResponseData, + } + + td.requestID = graphsync.RequestID(rand.Int31()) + td.requests = []gsmsg.GraphSyncRequest{ + gsmsg.NewRequest(td.requestID, td.blockChain.TipLink.(cidlink.Link).Cid, td.blockChain.Selector(), graphsync.Priority(0), td.extension), + } + td.p = testutil.GeneratePeers(1)[0] + td.peristenceOptions = persistenceoptions.New() + td.requestHooks = requesthooks.New(td.peristenceOptions) + td.blockHooks = blockhooks.New() + return td } diff --git a/responsemanager/runtraversal/runtraversal.go b/responsemanager/runtraversal/runtraversal.go new file mode 100644 index 00000000..7651a21f --- /dev/null +++ b/responsemanager/runtraversal/runtraversal.go @@ -0,0 +1,51 @@ +package runtraversal + +import ( + "bytes" + "io" + + "github.com/ipfs/go-graphsync/ipldutil" + ipld "github.com/ipld/go-ipld-prime" +) + +// ResponseSender sends responses over the network +type ResponseSender func( + link ipld.Link, + data []byte, +) error + +// RunTraversal wraps a given loader with an interceptor that sends loaded +// blocks out to the network with the given response sender. +func RunTraversal( + loader ipld.Loader, + traverser ipldutil.Traverser, + sendResponse ResponseSender) error { + for { + isComplete, err := traverser.IsComplete() + if isComplete { + return err + } + lnk, lnkCtx := traverser.CurrentRequest() + result, err := loader(lnk, lnkCtx) + var data []byte + if err != nil { + traverser.Error(err) + } else { + var blockBuffer bytes.Buffer + _, err = io.Copy(&blockBuffer, result) + if err != nil { + traverser.Error(err) + } else { + data = blockBuffer.Bytes() + err = traverser.Advance(&blockBuffer) + if err != nil { + return err + } + } + } + err = sendResponse(lnk, data) + if err != nil { + return err + } + } +} diff --git a/responsemanager/runtraversal/runtraversal_test.go b/responsemanager/runtraversal/runtraversal_test.go new file mode 100644 index 00000000..d0522f88 --- /dev/null +++ b/responsemanager/runtraversal/runtraversal_test.go @@ -0,0 +1,216 @@ +package runtraversal + +import ( + "bytes" + "errors" + "io" + "testing" + + "github.com/ipfs/go-graphsync/testutil" + "github.com/stretchr/testify/require" + + ipld "github.com/ipld/go-ipld-prime" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" +) + +type fakeResponseKey struct { + Link ipld.Link + Data string +} + +type fakeResponseSender struct { + stubbedResponses map[fakeResponseKey]error + expectedResponses map[fakeResponseKey]struct{} + receivedResponses map[fakeResponseKey]struct{} +} + +func newFakeResponseSender() *fakeResponseSender { + return &fakeResponseSender{ + stubbedResponses: make(map[fakeResponseKey]error), + expectedResponses: make(map[fakeResponseKey]struct{}), + receivedResponses: make(map[fakeResponseKey]struct{}), + } +} + +func (frs *fakeResponseSender) SendResponse( + link ipld.Link, + data []byte, +) error { + frs.receivedResponses[fakeResponseKey{link, string(data)}] = struct{}{} + return frs.stubbedResponses[fakeResponseKey{link, string(data)}] +} + +func (frs *fakeResponseSender) expectResponse(link ipld.Link, data []byte, returnVal error) { + frs.expectedResponses[fakeResponseKey{link, string(data)}] = struct{}{} + frs.stubbedResponses[fakeResponseKey{link, string(data)}] = returnVal +} + +func (frs *fakeResponseSender) verifyExpectations(t *testing.T) { + require.Equal(t, frs.expectedResponses, frs.receivedResponses) +} + +type loadedLink struct { + link ipld.Link + linkCtx ipld.LinkContext +} + +type traverseOutcome struct { + isError bool + err error + data []byte +} +type fakeTraverser struct { + finalError error + currentLink int + loadedLinks []loadedLink + receivedOutcomes []traverseOutcome + expectedOutcomes []traverseOutcome +} + +// IsComplete returns the completion state (boolean) and if so, the final error result from IPLD +func (ft *fakeTraverser) IsComplete() (bool, error) { + if ft.currentLink >= len(ft.loadedLinks) { + return true, ft.finalError + } + return false, nil +} + +// Current request returns the current link waiting to be loaded +func (ft *fakeTraverser) CurrentRequest() (ipld.Link, ipld.LinkContext) { + ll := ft.loadedLinks[ft.currentLink] + ft.currentLink++ + return ll.link, ll.linkCtx +} + +// Advance advances the traversal successfully by supplying the given reader as the result of the next IPLD load +func (ft *fakeTraverser) Advance(reader io.Reader) error { + buf := new(bytes.Buffer) + _, _ = io.Copy(buf, reader) + ft.receivedOutcomes = append(ft.receivedOutcomes, traverseOutcome{false, nil, buf.Bytes()}) + return nil +} + +// Error errors the traversal by returning the given error as the result of the next IPLD load +func (ft *fakeTraverser) Error(err error) { + ft.receivedOutcomes = append(ft.receivedOutcomes, traverseOutcome{true, err, nil}) +} + +func (ft *fakeTraverser) verifyExpectations(t *testing.T) { + require.Equal(t, ft.expectedOutcomes, ft.receivedOutcomes) +} + +type fakeLoader struct { + expectedLoads []loadedLink + receivedLoads []loadedLink + loadReturns []traverseOutcome + currentLoader int +} + +func (fl *fakeLoader) Load(link ipld.Link, linkCtx ipld.LinkContext) (io.Reader, error) { + fl.receivedLoads = append(fl.receivedLoads, loadedLink{link, linkCtx}) + outcome := fl.loadReturns[fl.currentLoader] + fl.currentLoader++ + if outcome.isError { + return nil, outcome.err + } + return bytes.NewBuffer(outcome.data), nil +} + +func (fl *fakeLoader) verifyExpectations(t *testing.T) { + require.Equal(t, fl.expectedLoads, fl.receivedLoads) +} + +func TestRunTraversal(t *testing.T) { + blks := testutil.GenerateBlocksOfSize(5, 100) + links := make([]loadedLink, 0, 5) + for _, blk := range blks { + links = append(links, loadedLink{link: cidlink.Link{Cid: blk.Cid()}, linkCtx: ipld.LinkContext{}}) + } + testCases := map[string]struct { + linksToLoad []loadedLink + linkLoadsExpected int + loadOutcomes []traverseOutcome + loadOutcomesExpected int + errorsOnSend []error + finalError error + expectedError error + }{ + "normal operation": { + linksToLoad: links, + linkLoadsExpected: 5, + loadOutcomes: []traverseOutcome{ + {false, nil, blks[0].RawData()}, + {false, nil, blks[1].RawData()}, + {false, nil, blks[2].RawData()}, + {false, nil, blks[3].RawData()}, + {false, nil, blks[4].RawData()}, + }, + errorsOnSend: []error{ + nil, nil, nil, nil, nil, + }, + }, + "error on complete": { + linksToLoad: links, + linkLoadsExpected: 5, + loadOutcomes: []traverseOutcome{ + {false, nil, blks[0].RawData()}, + {false, nil, blks[1].RawData()}, + {false, nil, blks[2].RawData()}, + {false, nil, blks[3].RawData()}, + {false, nil, blks[4].RawData()}, + }, + errorsOnSend: []error{ + nil, nil, nil, nil, nil, + }, + finalError: errors.New("traverse failed"), + expectedError: errors.New("traverse failed"), + }, + "error on load": { + linksToLoad: links[:3], + linkLoadsExpected: 3, + loadOutcomes: []traverseOutcome{ + {false, nil, blks[0].RawData()}, + {false, nil, blks[1].RawData()}, + {true, errors.New("block not found"), nil}, + }, + errorsOnSend: []error{ + nil, nil, nil, + }, + }, + "error on send": { + linksToLoad: links, + linkLoadsExpected: 3, + loadOutcomes: []traverseOutcome{ + {false, nil, blks[0].RawData()}, + {false, nil, blks[1].RawData()}, + {false, nil, blks[2].RawData()}, + }, + errorsOnSend: []error{ + nil, nil, errors.New("something went wrong"), + }, + expectedError: errors.New("something went wrong"), + }, + } + for testCase, data := range testCases { + t.Run(testCase, func(t *testing.T) { + fl := &fakeLoader{ + expectedLoads: data.linksToLoad[:data.linkLoadsExpected], + loadReturns: data.loadOutcomes, + } + ft := &fakeTraverser{ + finalError: data.finalError, + loadedLinks: data.linksToLoad, + expectedOutcomes: data.loadOutcomes, + } + frs := newFakeResponseSender() + for i, err := range data.errorsOnSend { + frs.expectResponse(data.linksToLoad[i].link, data.loadOutcomes[i].data, err) + } + err := RunTraversal(fl.Load, ft, frs.SendResponse) + require.Equal(t, data.expectedError, err) + fl.verifyExpectations(t) + frs.verifyExpectations(t) + ft.verifyExpectations(t) + }) + } +} diff --git a/testutil/chaintypes/testchain_gen.go b/testutil/chaintypes/testchain_gen.go index c0f08d79..e074cfe4 100644 --- a/testutil/chaintypes/testchain_gen.go +++ b/testutil/chaintypes/testchain_gen.go @@ -2,7 +2,6 @@ package chaintypes import ( ipld "github.com/ipld/go-ipld-prime" - "github.com/ipld/go-ipld-prime/impl/typed" "github.com/ipld/go-ipld-prime/schema" ) @@ -13,19 +12,19 @@ type Bytes struct{ x []byte } // TODO generateKindBytes.EmitNativeAccessors // TODO generateKindBytes.EmitNativeBuilder type MaybeBytes struct { - Maybe typed.Maybe + Maybe schema.Maybe Value Bytes } func (m MaybeBytes) Must() Bytes { - if m.Maybe != typed.Maybe_Value { + if m.Maybe != schema.Maybe_Value { panic("unbox of a maybe rejected") } return m.Value } var _ ipld.Node = Bytes{} -var _ typed.Node = Bytes{} +var _ schema.TypedNode = Bytes{} func (Bytes) Type() schema.Type { return nil /*TODO:typelit*/ @@ -81,9 +80,7 @@ func (Bytes) AsLink() (ipld.Link, error) { func (Bytes) NodeBuilder() ipld.NodeBuilder { return _Bytes__NodeBuilder{} } - type _Bytes__NodeBuilder struct{} - func Bytes__NodeBuilder() ipld.NodeBuilder { return _Bytes__NodeBuilder{} } @@ -123,25 +120,24 @@ func (_Bytes__NodeBuilder) CreateLink(ipld.Link) (ipld.Node, error) { func (Bytes) Representation() ipld.Node { panic("TODO representation") } - type Link struct{ x ipld.Link } // TODO generateKindLink.EmitNativeAccessors // TODO generateKindLink.EmitNativeBuilder type MaybeLink struct { - Maybe typed.Maybe + Maybe schema.Maybe Value Link } func (m MaybeLink) Must() Link { - if m.Maybe != typed.Maybe_Value { + if m.Maybe != schema.Maybe_Value { panic("unbox of a maybe rejected") } return m.Value } var _ ipld.Node = Link{} -var _ typed.Node = Link{} +var _ schema.TypedNode = Link{} func (Link) Type() schema.Type { return nil /*TODO:typelit*/ @@ -197,7 +193,6 @@ func (x Link) AsLink() (ipld.Link, error) { func (Link) NodeBuilder() ipld.NodeBuilder { return _Link__NodeBuilder{} } - type _Link__NodeBuilder struct{} func Link__NodeBuilder() ipld.NodeBuilder { @@ -239,13 +234,11 @@ func (nb _Link__NodeBuilder) CreateLink(v ipld.Link) (ipld.Node, error) { func (Link) Representation() ipld.Node { panic("TODO representation") } - type String struct{ x string } func (x String) String() string { return x.x } - type String__Content struct { Value string } @@ -267,19 +260,19 @@ func (b String__Content) MustBuild() String { } type MaybeString struct { - Maybe typed.Maybe + Maybe schema.Maybe Value String } func (m MaybeString) Must() String { - if m.Maybe != typed.Maybe_Value { + if m.Maybe != schema.Maybe_Value { panic("unbox of a maybe rejected") } return m.Value } var _ ipld.Node = String{} -var _ typed.Node = String{} +var _ schema.TypedNode = String{} func (String) Type() schema.Type { return nil /*TODO:typelit*/ @@ -335,9 +328,7 @@ func (String) AsLink() (ipld.Link, error) { func (String) NodeBuilder() ipld.NodeBuilder { return _String__NodeBuilder{} } - type _String__NodeBuilder struct{} - func String__NodeBuilder() ipld.NodeBuilder { return _String__NodeBuilder{} } @@ -377,27 +368,25 @@ func (_String__NodeBuilder) CreateLink(ipld.Link) (ipld.Node, error) { func (String) Representation() ipld.Node { panic("TODO representation") } - -type Parents struct { +type Parents struct{ x []Link } - // TODO generateKindList.EmitNativeAccessors // TODO generateKindList.EmitNativeBuilder type MaybeParents struct { - Maybe typed.Maybe + Maybe schema.Maybe Value Parents } func (m MaybeParents) Must() Parents { - if m.Maybe != typed.Maybe_Value { + if m.Maybe != schema.Maybe_Value { panic("unbox of a maybe rejected") } return m.Value } var _ ipld.Node = Parents{} -var _ typed.Node = Parents{} +var _ schema.TypedNode = Parents{} func (Parents) Type() schema.Type { return nil /*TODO:typelit*/ @@ -440,7 +429,7 @@ type _Parents__Itr struct { idx int } -func (itr *_Parents__Itr) Next() (idx int, value ipld.Node, _ error) { +func (itr *_Parents__Itr) Next() (idx int, value ipld.Node, _ error) { if itr.idx >= len(itr.node.x) { return 0, nil, ipld.ErrIteratorOverread{} } @@ -484,7 +473,6 @@ func (Parents) AsLink() (ipld.Link, error) { func (Parents) NodeBuilder() ipld.NodeBuilder { return _Parents__NodeBuilder{} } - type _Parents__NodeBuilder struct{} func Parents__NodeBuilder() ipld.NodeBuilder { @@ -497,10 +485,10 @@ func (_Parents__NodeBuilder) AmendMap() (ipld.MapBuilder, error) { return nil, ipld.ErrWrongKind{TypeName: "Parents.Builder", MethodName: "AmendMap", AppropriateKind: ipld.ReprKindSet_JustMap, ActualKind: ipld.ReprKind_List} } func (nb _Parents__NodeBuilder) CreateList() (ipld.ListBuilder, error) { - return &_Parents__ListBuilder{v: &Parents{}}, nil + return &_Parents__ListBuilder{v:&Parents{}}, nil } -type _Parents__ListBuilder struct { +type _Parents__ListBuilder struct{ v *Parents } @@ -541,13 +529,13 @@ func (lb *_Parents__ListBuilder) validate(v ipld.Node) error { if v.IsNull() { panic("type mismatch on struct field assignment: cannot assign null to non-nullable field") // FIXME need an error type for this } - tv, ok := v.(typed.Node) + tv, ok := v.(schema.TypedNode) if !ok { - panic("need typed.Node for insertion into struct") // FIXME need an error type for this + panic("need schema.TypedNode for insertion into struct") // FIXME need an error type for this } _, ok = v.(Link) if !ok { - panic("value for type Parents is type Link; cannot assign " + tv.Type().Name()) // FIXME need an error type for this + panic("value for type Parents is type Link; cannot assign "+tv.Type().Name()) // FIXME need an error type for this } return nil } @@ -566,7 +554,7 @@ func (lb *_Parents__ListBuilder) AppendAll(vs []ipld.Node) error { } off := len(lb.v.x) new := off + len(vs) - lb.growList(new - 1) + lb.growList(new-1) for _, v := range vs { lb.unsafeSet(off, v) off++ @@ -631,27 +619,25 @@ func (_Parents__NodeBuilder) CreateLink(ipld.Link) (ipld.Node, error) { func (n Parents) Representation() ipld.Node { panic("TODO representation") } - -type Messages struct { +type Messages struct{ x []Bytes } - // TODO generateKindList.EmitNativeAccessors // TODO generateKindList.EmitNativeBuilder type MaybeMessages struct { - Maybe typed.Maybe + Maybe schema.Maybe Value Messages } func (m MaybeMessages) Must() Messages { - if m.Maybe != typed.Maybe_Value { + if m.Maybe != schema.Maybe_Value { panic("unbox of a maybe rejected") } return m.Value } var _ ipld.Node = Messages{} -var _ typed.Node = Messages{} +var _ schema.TypedNode = Messages{} func (Messages) Type() schema.Type { return nil /*TODO:typelit*/ @@ -694,7 +680,7 @@ type _Messages__Itr struct { idx int } -func (itr *_Messages__Itr) Next() (idx int, value ipld.Node, _ error) { +func (itr *_Messages__Itr) Next() (idx int, value ipld.Node, _ error) { if itr.idx >= len(itr.node.x) { return 0, nil, ipld.ErrIteratorOverread{} } @@ -738,7 +724,6 @@ func (Messages) AsLink() (ipld.Link, error) { func (Messages) NodeBuilder() ipld.NodeBuilder { return _Messages__NodeBuilder{} } - type _Messages__NodeBuilder struct{} func Messages__NodeBuilder() ipld.NodeBuilder { @@ -751,10 +736,10 @@ func (_Messages__NodeBuilder) AmendMap() (ipld.MapBuilder, error) { return nil, ipld.ErrWrongKind{TypeName: "Messages.Builder", MethodName: "AmendMap", AppropriateKind: ipld.ReprKindSet_JustMap, ActualKind: ipld.ReprKind_List} } func (nb _Messages__NodeBuilder) CreateList() (ipld.ListBuilder, error) { - return &_Messages__ListBuilder{v: &Messages{}}, nil + return &_Messages__ListBuilder{v:&Messages{}}, nil } -type _Messages__ListBuilder struct { +type _Messages__ListBuilder struct{ v *Messages } @@ -795,13 +780,13 @@ func (lb *_Messages__ListBuilder) validate(v ipld.Node) error { if v.IsNull() { panic("type mismatch on struct field assignment: cannot assign null to non-nullable field") // FIXME need an error type for this } - tv, ok := v.(typed.Node) + tv, ok := v.(schema.TypedNode) if !ok { - panic("need typed.Node for insertion into struct") // FIXME need an error type for this + panic("need schema.TypedNode for insertion into struct") // FIXME need an error type for this } _, ok = v.(Bytes) if !ok { - panic("value for type Messages is type Bytes; cannot assign " + tv.Type().Name()) // FIXME need an error type for this + panic("value for type Messages is type Bytes; cannot assign "+tv.Type().Name()) // FIXME need an error type for this } return nil } @@ -820,7 +805,7 @@ func (lb *_Messages__ListBuilder) AppendAll(vs []ipld.Node) error { } off := len(lb.v.x) new := off + len(vs) - lb.growList(new - 1) + lb.growList(new-1) for _, v := range vs { lb.unsafeSet(off, v) off++ @@ -885,30 +870,25 @@ func (_Messages__NodeBuilder) CreateLink(ipld.Link) (ipld.Node, error) { func (n Messages) Representation() ipld.Node { panic("TODO representation") } - -type Block struct { - Parents Parents - Messages Messages +type Block struct{ + d Block__Content } -func (x Parents) FieldParents() Parents { - // TODO going to tear through here with changes to Maybe system in a moment anyway - return Parents{} +func (x Block) FieldParents()Parents { + return x.d.Parents } -func (x Messages) FieldMessages() Messages { - // TODO going to tear through here with changes to Maybe system in a moment anyway - return Messages{} +func (x Block) FieldMessages()Messages { + return x.d.Messages } + type Block__Content struct { - // TODO - // TODO + Parents Parents + Messages Messages } func (b Block__Content) Build() (Block, error) { - x := Block{ - // TODO - } + x := Block{b} // FUTURE : want to support customizable validation. // but 'if v, ok := x.(schema.Validatable); ok {' doesn't fly: need a way to work on concrete types. return x, nil @@ -922,19 +902,19 @@ func (b Block__Content) MustBuild() Block { } type MaybeBlock struct { - Maybe typed.Maybe + Maybe schema.Maybe Value Block } func (m MaybeBlock) Must() Block { - if m.Maybe != typed.Maybe_Value { + if m.Maybe != schema.Maybe_Value { panic("unbox of a maybe rejected") } return m.Value } var _ ipld.Node = Block{} -var _ typed.Node = Block{} +var _ schema.TypedNode = Block{} func (Block) Type() schema.Type { return nil /*TODO:typelit*/ @@ -945,11 +925,11 @@ func (Block) ReprKind() ipld.ReprKind { func (x Block) LookupString(key string) (ipld.Node, error) { switch key { case "Parents": - return x.Parents, nil + return x.d.Parents, nil case "Messages": - return x.Messages, nil + return x.d.Messages, nil default: - return nil, typed.ErrNoSuchField{Type: nil /*TODO*/, FieldName: key} + return nil, schema.ErrNoSuchField{Type: nil /*TODO*/, FieldName: key} } } func (x Block) Lookup(key ipld.Node) (ipld.Node, error) { @@ -981,10 +961,10 @@ func (itr *_Block__Itr) Next() (k ipld.Node, v ipld.Node, _ error) { switch itr.idx { case 0: k = String{"Parents"} - v = itr.node.Parents + v = itr.node.d.Parents case 1: k = String{"Messages"} - v = itr.node.Messages + v = itr.node.d.Messages default: panic("unreachable") } @@ -1028,19 +1008,19 @@ func (Block) AsLink() (ipld.Link, error) { func (Block) NodeBuilder() ipld.NodeBuilder { return _Block__NodeBuilder{} } - type _Block__NodeBuilder struct{} func Block__NodeBuilder() ipld.NodeBuilder { return _Block__NodeBuilder{} } func (nb _Block__NodeBuilder) CreateMap() (ipld.MapBuilder, error) { - return &_Block__MapBuilder{v: &Block{}}, nil + mb := &_Block__MapBuilder{v:&Block{}} + return mb, nil } -type _Block__MapBuilder struct { - v *Block - Parents__isset bool +type _Block__MapBuilder struct{ + v *Block + Parents__isset bool Messages__isset bool } @@ -1054,32 +1034,32 @@ func (mb *_Block__MapBuilder) Insert(k, v ipld.Node) error { if v.IsNull() { panic("type mismatch on struct field assignment: cannot assign null to non-nullable field") // FIXME need an error type for this } - tv, ok := v.(typed.Node) + tv, ok := v.(schema.TypedNode) if !ok { - panic("need typed.Node for insertion into struct") // FIXME need an error type for this + panic("need schema.TypedNode for insertion into struct") // FIXME need an error type for this } x, ok := v.(Parents) if !ok { - panic("field 'Parents' in type Block is type Parents; cannot assign " + tv.Type().Name()) // FIXME need an error type for this + panic("field 'Parents' in type Block is type Parents; cannot assign "+tv.Type().Name()) // FIXME need an error type for this } - mb.v.Parents = x + mb.v.d.Parents = x mb.Parents__isset = true case "Messages": if v.IsNull() { panic("type mismatch on struct field assignment: cannot assign null to non-nullable field") // FIXME need an error type for this } - tv, ok := v.(typed.Node) + tv, ok := v.(schema.TypedNode) if !ok { - panic("need typed.Node for insertion into struct") // FIXME need an error type for this + panic("need schema.TypedNode for insertion into struct") // FIXME need an error type for this } x, ok := v.(Messages) if !ok { - panic("field 'Messages' in type Block is type Messages; cannot assign " + tv.Type().Name()) // FIXME need an error type for this + panic("field 'Messages' in type Block is type Messages; cannot assign "+tv.Type().Name()) // FIXME need an error type for this } - mb.v.Messages = x + mb.v.d.Messages = x mb.Messages__isset = true default: - return typed.ErrNoSuchField{Type: nil /*TODO:typelit*/, FieldName: ks} + return schema.ErrNoSuchField{Type: nil /*TODO:typelit*/, FieldName: ks} } return nil } @@ -1107,7 +1087,7 @@ func (mb *_Block__MapBuilder) BuilderForValue(ks string) ipld.NodeBuilder { case "Messages": return Messages__NodeBuilder() default: - panic(typed.ErrNoSuchField{Type: nil /*TODO:typelit*/, FieldName: ks}) + panic(schema.ErrNoSuchField{Type: nil /*TODO:typelit*/, FieldName: ks}) } return nil } @@ -1145,10 +1125,9 @@ func (_Block__NodeBuilder) CreateLink(ipld.Link) (ipld.Node, error) { func (n Block) Representation() ipld.Node { return _Block__Repr{&n} } - var _ ipld.Node = _Block__Repr{} -type _Block__Repr struct { +type _Block__Repr struct{ n *Block } @@ -1158,11 +1137,11 @@ func (_Block__Repr) ReprKind() ipld.ReprKind { func (rn _Block__Repr) LookupString(key string) (ipld.Node, error) { switch key { case "Parents": - return rn.n.Parents, nil + return rn.n.d.Parents, nil case "Messages": - return rn.n.Messages, nil + return rn.n.d.Messages, nil default: - return nil, typed.ErrNoSuchField{Type: nil /*TODO*/, FieldName: key} + return nil, schema.ErrNoSuchField{Type: nil /*TODO*/, FieldName: key} } } func (rn _Block__Repr) Lookup(key ipld.Node) (ipld.Node, error) { @@ -1195,10 +1174,10 @@ func (itr *_Block__ReprItr) Next() (k ipld.Node, v ipld.Node, _ error) { switch itr.idx { case 0: k = String{"Parents"} - v = itr.node.Parents + v = itr.node.d.Parents case 1: k = String{"Messages"} - v = itr.node.Messages + v = itr.node.d.Messages default: panic("unreachable") } @@ -1244,19 +1223,19 @@ func (_Block__Repr) AsLink() (ipld.Link, error) { func (_Block__Repr) NodeBuilder() ipld.NodeBuilder { return _Block__ReprBuilder{} } - type _Block__ReprBuilder struct{} func Block__ReprBuilder() ipld.NodeBuilder { return _Block__ReprBuilder{} } func (nb _Block__ReprBuilder) CreateMap() (ipld.MapBuilder, error) { - return &_Block__ReprMapBuilder{v: &Block{}}, nil + mb := &_Block__ReprMapBuilder{v:&Block{}} + return mb, nil } -type _Block__ReprMapBuilder struct { - v *Block - Parents__isset bool +type _Block__ReprMapBuilder struct{ + v *Block + Parents__isset bool Messages__isset bool } @@ -1273,15 +1252,15 @@ func (mb *_Block__ReprMapBuilder) Insert(k, v ipld.Node) error { if v.IsNull() { panic("type mismatch on struct field assignment: cannot assign null to non-nullable field") // FIXME need an error type for this } - tv, ok := v.(typed.Node) + tv, ok := v.(schema.TypedNode) if !ok { - panic("need typed.Node for insertion into struct") // FIXME need an error type for this + panic("need schema.TypedNode for insertion into struct") // FIXME need an error type for this } x, ok := v.(Parents) if !ok { - panic("field 'Parents' (key: 'Parents') in type Block is type Parents; cannot assign " + tv.Type().Name()) // FIXME need an error type for this + panic("field 'Parents' (key: 'Parents') in type Block is type Parents; cannot assign "+tv.Type().Name()) // FIXME need an error type for this } - mb.v.Parents = x + mb.v.d.Parents = x mb.Parents__isset = true case "Messages": if mb.Messages__isset { @@ -1290,18 +1269,18 @@ func (mb *_Block__ReprMapBuilder) Insert(k, v ipld.Node) error { if v.IsNull() { panic("type mismatch on struct field assignment: cannot assign null to non-nullable field") // FIXME need an error type for this } - tv, ok := v.(typed.Node) + tv, ok := v.(schema.TypedNode) if !ok { - panic("need typed.Node for insertion into struct") // FIXME need an error type for this + panic("need schema.TypedNode for insertion into struct") // FIXME need an error type for this } x, ok := v.(Messages) if !ok { - panic("field 'Messages' (key: 'Messages') in type Block is type Messages; cannot assign " + tv.Type().Name()) // FIXME need an error type for this + panic("field 'Messages' (key: 'Messages') in type Block is type Messages; cannot assign "+tv.Type().Name()) // FIXME need an error type for this } - mb.v.Messages = x + mb.v.d.Messages = x mb.Messages__isset = true default: - return typed.ErrNoSuchField{Type: nil /*TODO:typelit*/, FieldName: ks} + return schema.ErrNoSuchField{Type: nil /*TODO:typelit*/, FieldName: ks} } return nil } @@ -1329,7 +1308,7 @@ func (mb *_Block__ReprMapBuilder) BuilderForValue(ks string) ipld.NodeBuilder { case "Messages": return Messages__NodeBuilder() default: - panic(typed.ErrNoSuchField{Type: nil /*TODO:typelit*/, FieldName: ks}) + panic(schema.ErrNoSuchField{Type: nil /*TODO:typelit*/, FieldName: ks}) } return nil } diff --git a/testutil/test_ipld_tree.go b/testutil/test_ipld_tree.go new file mode 100644 index 00000000..b0668a36 --- /dev/null +++ b/testutil/test_ipld_tree.go @@ -0,0 +1,113 @@ +package testutil + +import ( + "bytes" + "context" + "io" + + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "github.com/ipld/go-ipld-prime" + + // to register multicodec + _ "github.com/ipld/go-ipld-prime/encoding/dagjson" + "github.com/ipld/go-ipld-prime/fluent" + ipldfree "github.com/ipld/go-ipld-prime/impl/free" + cidlink "github.com/ipld/go-ipld-prime/linking/cid" +) + +// TestIPLDTree is a set of IPLD Data that forms a tree spread across some blocks +// with a serialized in memory representation +type TestIPLDTree struct { + Storage map[ipld.Link][]byte + LeafAlpha ipld.Node + LeafAlphaLnk ipld.Link + LeafAlphaBlock blocks.Block + LeafBeta ipld.Node + LeafBetaLnk ipld.Link + LeafBetaBlock blocks.Block + MiddleMapNode ipld.Node + MiddleMapNodeLnk ipld.Link + MiddleMapBlock blocks.Block + MiddleListNode ipld.Node + MiddleListNodeLnk ipld.Link + MiddleListBlock blocks.Block + RootNode ipld.Node + RootNodeLnk ipld.Link + RootBlock blocks.Block +} + +// NewTestIPLDTree returns a fake tree of nodes, spread across 5 blocks +func NewTestIPLDTree() TestIPLDTree { + var storage = make(map[ipld.Link][]byte) + var fnb = fluent.WrapNodeBuilder(ipldfree.NodeBuilder()) // just for the other fixture building + encode := func(n ipld.Node) (ipld.Node, ipld.Link) { + lb := cidlink.LinkBuilder{Prefix: cid.Prefix{ + Version: 1, + Codec: 0x0129, + MhType: 0x17, + MhLength: 4, + }} + lnk, err := lb.Build(context.Background(), ipld.LinkContext{}, n, + func(ipld.LinkContext) (io.Writer, ipld.StoreCommitter, error) { + buf := bytes.Buffer{} + return &buf, func(lnk ipld.Link) error { + storage[lnk] = buf.Bytes() + return nil + }, nil + }, + ) + if err != nil { + panic(err) + } + return n, lnk + } + + var ( + leafAlpha, leafAlphaLnk = encode(fnb.CreateString("alpha")) + leafAlphaBlock, _ = blocks.NewBlockWithCid(storage[leafAlphaLnk], leafAlphaLnk.(cidlink.Link).Cid) + leafBeta, leafBetaLnk = encode(fnb.CreateString("beta")) + leafBetaBlock, _ = blocks.NewBlockWithCid(storage[leafBetaLnk], leafBetaLnk.(cidlink.Link).Cid) + middleMapNode, middleMapNodeLnk = encode(fnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) { + mb.Insert(knb.CreateString("foo"), vnb.CreateBool(true)) + mb.Insert(knb.CreateString("bar"), vnb.CreateBool(false)) + mb.Insert(knb.CreateString("nested"), vnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) { + mb.Insert(knb.CreateString("alink"), vnb.CreateLink(leafAlphaLnk)) + mb.Insert(knb.CreateString("nonlink"), vnb.CreateString("zoo")) + })) + })) + middleMapBlock, _ = blocks.NewBlockWithCid(storage[middleMapNodeLnk], middleMapNodeLnk.(cidlink.Link).Cid) + middleListNode, middleListNodeLnk = encode(fnb.CreateList(func(lb fluent.ListBuilder, vnb fluent.NodeBuilder) { + lb.Append(vnb.CreateLink(leafAlphaLnk)) + lb.Append(vnb.CreateLink(leafAlphaLnk)) + lb.Append(vnb.CreateLink(leafBetaLnk)) + lb.Append(vnb.CreateLink(leafAlphaLnk)) + })) + middleListBlock, _ = blocks.NewBlockWithCid(storage[middleListNodeLnk], middleListNodeLnk.(cidlink.Link).Cid) + rootNode, rootNodeLnk = encode(fnb.CreateMap(func(mb fluent.MapBuilder, knb fluent.NodeBuilder, vnb fluent.NodeBuilder) { + mb.Insert(knb.CreateString("plain"), vnb.CreateString("olde string")) + mb.Insert(knb.CreateString("linkedString"), vnb.CreateLink(leafAlphaLnk)) + mb.Insert(knb.CreateString("linkedMap"), vnb.CreateLink(middleMapNodeLnk)) + mb.Insert(knb.CreateString("linkedList"), vnb.CreateLink(middleListNodeLnk)) + })) + rootBlock, _ = blocks.NewBlockWithCid(storage[rootNodeLnk], rootNodeLnk.(cidlink.Link).Cid) + ) + return TestIPLDTree{ + Storage: storage, + LeafAlpha: leafAlpha, + LeafAlphaLnk: leafAlphaLnk, + LeafAlphaBlock: leafAlphaBlock, + LeafBeta: leafBeta, + LeafBetaLnk: leafBetaLnk, + LeafBetaBlock: leafBetaBlock, + MiddleMapNode: middleMapNode, + MiddleMapNodeLnk: middleMapNodeLnk, + MiddleMapBlock: middleMapBlock, + MiddleListNode: middleListNode, + MiddleListNodeLnk: middleListNodeLnk, + MiddleListBlock: middleListBlock, + RootNode: rootNode, + RootNodeLnk: rootNodeLnk, + RootBlock: rootBlock, + } +} diff --git a/testutil/testchain.go b/testutil/testchain.go index d73ccb31..cfffd2a0 100644 --- a/testutil/testchain.go +++ b/testutil/testchain.go @@ -57,7 +57,6 @@ func createBlock(parents []ipld.Link, size uint64) (ipld.Node, error) { if err != nil { return nil, err } - parentsNdTyped := parentsNd.(chaintypes.Parents) mnb := chaintypes.Messages__NodeBuilder() mnblnb, err := mnb.CreateList() if err != nil { @@ -76,11 +75,29 @@ func createBlock(parents []ipld.Link, size uint64) (ipld.Node, error) { if err != nil { return nil, err } - messagesNdTyped := mesagesNd.(chaintypes.Messages) - return chaintypes.Block{ - Parents: parentsNdTyped, - Messages: messagesNdTyped, - }, nil + blknb := chaintypes.Block__NodeBuilder() + blknbmnb, err := blknb.CreateMap() + if err != nil { + return nil, err + } + snb := chaintypes.String__NodeBuilder() + key, err := snb.CreateString("Parents") + if err != nil { + return nil, err + } + err = blknbmnb.Insert(key, parentsNd) + if err != nil { + return nil, err + } + key, err = snb.CreateString("Messages") + if err != nil { + return nil, err + } + err = blknbmnb.Insert(key, mesagesNd) + if err != nil { + return nil, err + } + return blknbmnb.Build() } // SetupBlockChain creates a new test block chain with the given height @@ -246,6 +263,6 @@ func (tbc *TestBlockChain) RemainderBlocks(from int) []blocks.Block { } // Chooser is a NodeBuilderChooser function that always returns the block chain -func (tbc *TestBlockChain) Chooser(ipld.Link, ipld.LinkContext) ipld.NodeBuilder { - return chaintypes.Block__NodeBuilder() +func (tbc *TestBlockChain) Chooser(ipld.Link, ipld.LinkContext) (ipld.NodeBuilder, error) { + return chaintypes.Block__NodeBuilder(), nil } diff --git a/testutil/testutil.go b/testutil/testutil.go index d961f3d2..6c480dcb 100644 --- a/testutil/testutil.go +++ b/testutil/testutil.go @@ -162,22 +162,33 @@ func ReadNResponses(ctx context.Context, t *testing.T, responseChan <-chan graph // VerifySingleTerminalError verifies that exactly one error was sent over a channel // and then the channel was closed. func VerifySingleTerminalError(ctx context.Context, t *testing.T, errChan <-chan error) { - select { - case err := <-errChan: - require.NotNil(t, err, "should have sent a erminal error but did not") - case <-ctx.Done(): - t.Fatal("no errors sent") - } + var err error + AssertReceive(ctx, t, errChan, &err, "should receive an error") select { case _, ok := <-errChan: - if ok { - t.Fatal("shouldn't have sent second error but did") - } + require.False(t, ok, "shouldn't have sent second error but did") case <-ctx.Done(): t.Fatal("errors not closed") } } +// VerifyHasErrors verifies that at least one error was sent over a channel +func VerifyHasErrors(ctx context.Context, t *testing.T, errChan <-chan error) { + errCount := 0 + for { + select { + case _, ok := <-errChan: + if !ok { + require.NotZero(t, errCount, "should have errors") + return + } + errCount++ + case <-ctx.Done(): + t.Fatal("errors not closed") + } + } +} + // VerifyEmptyErrors verifies that no errors were sent over a channel before // it was closed func VerifyEmptyErrors(ctx context.Context, t *testing.T, errChan <-chan error) {