Upgrade to the latest libp2p, ipfs-repo, ipfs-unixfs-*, ipfs-mfs, ipld-* (#2390)

* chore: bump dependencies for ipfs-repo async update
* tests: update ipfsd-ctl

This new version of ipfsd-ctl is using async/await instead of callbacks.
Hence some rather mechanical changes are needed for tests.

* feat: use async/await based ipfs-repo
* chore: convert internals to promises
* chore: update bundle size

Author: vmx

.aegir.js

@@ -10,7 +10,7 @@ const preloadNode = MockPreloadNode.createNode()
 const echoServer = EchoServer.createServer()
 
 module.exports = {
-  bundlesize: { maxSize: '696kB' },
+  bundlesize: { maxSize: '685kB' },
   webpack: {
     resolve: {
       mainFields: ['browser', 'main'],

.travis.yml

@@ -28,6 +28,13 @@ env:
   # https://travis-ci.community/t/timeout-after-build-finished-and-succeeded/1336
   - YARN_GPG=no
 
+addons:
+  apt:
+    packages:
+      # Fixes error while loading shared libraries: libgconf-2.so.4: cannot open shared object file: No such file or directory
+      # https://github.com/electron/electron/issues/1518
+      - libgconf-2-4
+
 script: npx nyc -s npx aegir test -t node --timeout 10000 --bail
 after_success:
   - npx nyc report --reporter=text-lcov > coverage.lcov && npx codecov

examples/traverse-ipld-graphs/create-node.js

@@ -4,7 +4,8 @@ const IPFS = require('../../src/core')
 // In your project, replace by the following line and install IPFS as a dep
 // const IPFS = require('ipfs')
 
-function createNode (options = {}) {
+function createNode (options) {
+  options = options || {}
   options.path = options.path || '/tmp/ipfs' + Math.random()
   return IPFS.create({ repo: options.path })
 }

examples/traverse-ipld-graphs/eth.js

@@ -6,7 +6,6 @@ const multihashing = require('multihashing-async')
 const Block = require('ipfs-block')
 const CID = require('cids')
 const fs = require('fs').promises
-const { promisify } = require('util')
 
 async function main () {
   const ipfs = await createNode()
@@ -20,7 +19,7 @@ async function main () {
 
   for (const ethBlockPath of ethBlocks) {
     const data = await fs.readFile(ethBlockPath)
-    const multihash = await promisify(multihashing)(data, 'keccak-256')
+    const multihash = await multihashing(data, 'keccak-256')
 
     const cid = new CID(1, 'eth-block', multihash)
     // console.log(cid.toBaseEncodedString())

examples/traverse-ipld-graphs/get-path-accross-formats.js

@@ -1,15 +1,17 @@
 'use strict'
 
 const createNode = require('./create-node')
-const dagPB = require('ipld-dag-pb')
+const {
+  DAGNode
+} = require('ipld-dag-pb')
 
 async function main () {
   const ipfs = await createNode()
 
   console.log('\nStart of the example:')
 
   const someData = Buffer.from('capoeira')
-  const pbNode = dagPB.DAGNode.create(someData)
+  const pbNode = new DAGNode(someData)
 
   const pbNodeCid = await ipfs.dag.put(pbNode, {
     format: 'dag-pb',

examples/traverse-ipld-graphs/git.js

@@ -6,7 +6,6 @@ const multihashing = require('multihashing-async')
 const Block = require('ipfs-block')
 const CID = require('cids')
 const fs = require('fs').promises
-const { promisify } = require('util')
 
 async function main () {
   const ipfs = await createNode()
@@ -28,7 +27,7 @@ async function main () {
 
   await Promise.all(gitObjects.map(async gitObjectsPath => {
     const data = await fs.readFile(gitObjectsPath)
-    const multihash = await promisify(multihashing)(data, 'sha1')
+    const multihash = await multihashing(data, 'sha1')
 
     const cid = new CID(1, 'git-raw', multihash)
     console.log(cid.toString())

examples/traverse-ipld-graphs/tree.js

@@ -1,15 +1,17 @@
 'use strict'
 
 const createNode = require('./create-node')
-const dagPB = require('ipld-dag-pb')
+const {
+  DAGNode
+} = require('ipld-dag-pb')
 
 async function main () {
   const ipfs = await createNode()
 
   console.log('\nStart of the example:')
 
   const someData = Buffer.from('capoeira')
-  const pbNode = dagPB.DAGNode.create(someData)
+  const pbNode = new DAGNode(someData)
 
   const pbNodeCid = await ipfs.dag.put(pbNode, {
     format: 'dag-pb',

package.json

@@ -63,7 +63,7 @@
     "@hapi/ammo": "^3.1.1",
     "@hapi/boom": "^7.4.3",
     "@hapi/hapi": "^18.3.2",
-    "@hapi/joi": "^15.0.1",
+    "@hapi/joi": "^15.0.0",
     "array-shuffle": "^1.0.1",
     "async": "^2.6.1",
     "async-iterator-all": "^1.0.0",
@@ -76,11 +76,13 @@
     "bs58": "^4.0.1",
     "buffer-peek-stream": "^1.0.1",
     "byteman": "^1.3.5",
+    "callbackify": "^1.1.0",
     "cid-tool": "~0.3.0",
     "cids": "~0.7.1",
     "class-is": "^1.1.0",
-    "datastore-core": "~0.6.0",
-    "datastore-pubsub": "~0.1.1",
+    "dag-cbor-links": "^1.3.0",
+    "datastore-core": "~0.7.0",
+    "datastore-pubsub": "^0.2.1",
     "debug": "^4.1.0",
     "dlv": "^1.1.3",
     "err-code": "^2.0.0",
@@ -93,28 +95,28 @@
     "hapi-pino": "^6.1.0",
     "hashlru": "^2.3.0",
     "human-to-milliseconds": "^2.0.0",
-    "interface-datastore": "~0.6.0",
-    "ipfs-bitswap": "~0.25.1",
+    "interface-datastore": "~0.7.0",
+    "ipfs-bitswap": "^0.26.0",
     "ipfs-block": "~0.8.1",
-    "ipfs-block-service": "~0.15.2",
-    "ipfs-http-client": "^37.0.2",
+    "ipfs-block-service": "~0.16.0",
+    "ipfs-http-client": "^38.0.0",
     "ipfs-http-response": "~0.3.1",
-    "ipfs-mfs": "^0.12.2",
+    "ipfs-mfs": "^0.13.0",
     "ipfs-multipart": "^0.2.0",
-    "ipfs-repo": "~0.26.6",
+    "ipfs-repo": "^0.28.0",
     "ipfs-unixfs": "~0.1.16",
-    "ipfs-unixfs-exporter": "~0.37.7",
-    "ipfs-unixfs-importer": "~0.39.11",
+    "ipfs-unixfs-exporter": "^0.38.0",
+    "ipfs-unixfs-importer": "^0.40.0",
     "ipfs-utils": "~0.4.0",
-    "ipld": "~0.24.1",
+    "ipld": "~0.25.0",
     "ipld-bitcoin": "~0.3.0",
     "ipld-dag-cbor": "~0.15.0",
-    "ipld-dag-pb": "~0.17.4",
+    "ipld-dag-pb": "~0.18.0",
     "ipld-ethereum": "^4.0.0",
     "ipld-git": "~0.5.0",
     "ipld-raw": "^4.0.0",
     "ipld-zcash": "~0.3.0",
-    "ipns": "~0.5.2",
+    "ipns": "^0.6.1",
     "is-domain-name": "^1.0.1",
     "is-ipfs": "~0.6.1",
     "is-pull-stream": "~0.0.0",
@@ -124,40 +126,43 @@
     "it-to-stream": "^0.1.1",
     "just-safe-set": "^2.1.0",
     "kind-of": "^6.0.2",
-    "ky": "~0.14.0",
+    "ky": "^0.14.0",
     "ky-universal": "~0.3.0",
-    "libp2p": "~0.26.1",
+    "libp2p": "^0.26.2",
     "libp2p-bootstrap": "~0.9.3",
-    "libp2p-crypto": "~0.16.0",
-    "libp2p-delegated-content-routing": "^0.2.4",
-    "libp2p-delegated-peer-routing": "^0.2.4",
+    "libp2p-crypto": "^0.16.2",
+    "libp2p-delegated-content-routing": "^0.3.1",
+    "libp2p-delegated-peer-routing": "^0.3.1",
     "libp2p-floodsub": "^0.18.0",
     "libp2p-gossipsub": "~0.0.5",
-    "libp2p-kad-dht": "~0.15.3",
-    "libp2p-keychain": "~0.4.2",
+    "libp2p-kad-dht": "~0.16.0",
+    "libp2p-keychain": "^0.5.1",
     "libp2p-mdns": "~0.12.0",
-    "libp2p-record": "~0.6.3",
+    "libp2p-record": "~0.7.0",
     "libp2p-secio": "~0.11.0",
-    "libp2p-tcp": "~0.13.1",
+    "libp2p-tcp": "^0.13.0",
     "libp2p-webrtc-star": "~0.16.0",
     "libp2p-websocket-star-multi": "~0.4.3",
     "libp2p-websockets": "~0.12.3",
     "lodash": "^4.17.15",
+    "lodash.flatten": "^4.4.0",
     "mafmt": "^6.0.10",
     "merge-options": "^1.0.1",
     "mime-types": "^2.1.21",
     "mkdirp": "~0.5.1",
     "mortice": "^2.0.0",
-    "multiaddr": "^6.1.0",
+    "multiaddr": "^6.1.1",
     "multiaddr-to-uri": "^5.0.0",
     "multibase": "~0.6.0",
     "multicodec": "~0.5.5",
     "multihashes": "~0.4.14",
-    "multihashing-async": "~0.6.0",
+    "multihashing-async": "^0.8.0",
+    "node-fetch": "^2.3.0",
+    "p-iteration": "^1.1.8",
     "p-queue": "^6.1.0",
-    "peer-book": "~0.9.0",
-    "peer-id": "~0.12.3",
-    "peer-info": "~0.15.0",
+    "peer-book": "^0.9.1",
+    "peer-id": "^0.12.2",
+    "peer-info": "~0.15.1",
     "progress": "^2.0.1",
     "promise-nodeify": "^3.0.1",
     "promisify-es6": "^1.0.3",
@@ -199,7 +204,7 @@
     "execa": "^2.0.4",
     "form-data": "^2.5.1",
     "hat": "0.0.3",
-    "interface-ipfs-core": "~0.114.0",
+    "interface-ipfs-core": "^0.115.0",
     "ipfs-interop": "~0.1.0",
     "ipfsd-ctl": "^0.47.2",
     "libp2p-websocket-star": "~0.10.2",

src/cli/daemon.js

@@ -9,7 +9,6 @@ const TCP = require('libp2p-tcp')
 const MulticastDNS = require('libp2p-mdns')
 const WS = require('libp2p-websockets')
 const Bootstrap = require('libp2p-bootstrap')
-const promisify = require('promisify-es6')
 
 class Daemon {
   constructor (options) {
@@ -75,7 +74,7 @@ class Daemon {
 
     // for the CLI to know the where abouts of the API
     if (this._httpApi._apiServers.length) {
-      await promisify(ipfs._repo.apiAddr.set)(this._httpApi._apiServers[0].info.ma)
+      await ipfs._repo.apiAddr.set(this._httpApi._apiServers[0].info.ma)
     }
 
     this._log('started')

src/cli/utils.js

@@ -10,6 +10,7 @@ log.error = debug('cli:error')
 const Progress = require('progress')
 const byteman = require('byteman')
 const promisify = require('promisify-es6')
+const callbackify = require('callbackify')
 
 exports.isDaemonOn = isDaemonOn
 function isDaemonOn () {
@@ -52,11 +53,9 @@ exports.getIPFS = (argv, callback) => {
     pass: argv.pass
   })
 
-  const cleanup = promisify((cb) => {
+  const cleanup = callbackify(async () => {
     if (node && node._repo && !node._repo.closed) {
-      node._repo.close((err) => cb(err))
-    } else {
-      cb()
+      await node._repo.close()
     }
   })