refactor: async iterables

Author: Alan Shaw

Diff for: README.md

@@ -53,6 +53,8 @@
     - [Additional Options](#additional-options)
     - [Instance Utils](#instance-utils)
     - [Static Types and Utils](#static-types-and-utils)
+      - [Glob source](#glob-source)
+      - [URL source](#url-source)
 - [Development](#development)
   - [Testing](#testing)
 - [Contribute](#contribute)
@@ -210,34 +212,20 @@ const ipfs = ipfsClient({ timeout: '2m' })
 
 - [Regular Files API](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md)
   - [`ipfs.add(data, [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#add)
-  - [`ipfs.addPullStream([options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#addpullstream)
-  - [`ipfs.addReadableStream([options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#addreadablestream)
-  - [`ipfs.addFromStream(stream)`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#addfromstream)
-  - [`ipfs.addFromFs(path, [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#addfromfs)
-  - [`ipfs.addFromURL(url, [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#addfromurl)
   - [`ipfs.cat(ipfsPath, [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#cat)
-  - [`ipfs.catPullStream(ipfsPath, [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#catpullstream)
-  - [`ipfs.catReadableStream(ipfsPath, [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#catreadablestream)
   - [`ipfs.get(ipfsPath, [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#get)
-  - [`ipfs.getPullStream(ipfsPath, [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#getpullstream)
-  - [`ipfs.getReadableStream(ipfsPath, [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#getreadablestream)
   - [`ipfs.ls(ipfsPath)`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#ls)
-  - [`ipfs.lsPullStream(ipfsPath)`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#lspullstream)
-  - [`ipfs.lsReadableStream(ipfsPath)`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#lsreadablestream)
 - [MFS (mutable file system) specific](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#mutable-file-system)
-
-  _Explore the Mutable File System through interactive coding challenges in our [ProtoSchool tutorial](https://proto.school/#/mutable-file-system/)._
   - [`ipfs.files.cp([from, to])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#filescp)
   - [`ipfs.files.flush([path])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#filesflush)
   - [`ipfs.files.ls([path], [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#filesls)
   - [`ipfs.files.mkdir(path, [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#filesmkdir)
   - [`ipfs.files.mv([from, to])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#filesmv)
   - [`ipfs.files.read(path, [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#filesread)
-  - [`ipfs.files.readPullStream(path, [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#filesreadpullstream)
-  - [`ipfs.files.readReadableStream(path, [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#filesreadreadablestream)
   - [`ipfs.files.rm(path, [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#filesrm)
   - [`ipfs.files.stat(path, [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#filesstat)
   - [`ipfs.files.write(path, content, [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/FILES.md#fileswrite)
+  _Explore the Mutable File System through interactive coding challenges in our [ProtoSchool tutorial](https://proto.school/#/mutable-file-system/)._
 
 - [block](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/BLOCK.md)
   - [`ipfs.block.get(cid, [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/BLOCK.md#blockget)
@@ -246,20 +234,15 @@ const ipfs = ipfsClient({ timeout: '2m' })
 
 - [refs](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/REFS.md)
   - [`ipfs.refs(ipfsPath, [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/REFS.md#refs)
-  - [`ipfs.refsReadableStream(ipfsPath, [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/REFS.md#refsreadablestream)
-  - [`ipfs.refsPullStream(ipfsPath, [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/REFS.md#refspullstream)
   - [`ipfs.refs.local()`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/REFS.md#refslocal)
-  - [`ipfs.refs.localReadableStream()`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/REFS.md#refslocalreadablestream)
-  - [`ipfs.refs.localPullStream()`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/REFS.md#refslocalpullstream)
 
 #### Graph
 
 - [dag](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DAG.md)
-
-  _Explore the DAG API through interactive coding challenges in our [ProtoSchool tutorial](https://proto.school/#/basics)._
   - [`ipfs.dag.get(cid, [path], [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DAG.md#dagget)
   - [`ipfs.dag.put(dagNode, [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DAG.md#dagput)
   - [`ipfs.dag.tree(cid, [path], [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DAG.md#dagtree)
+  _Explore the DAG API through interactive coding challenges in our [ProtoSchool tutorial](https://proto.school/#/basics)._
 
 - [object](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/OBJECT.md)
   - [`ipfs.object.data(multihash, [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/OBJECT.md#objectdata)
@@ -278,9 +261,6 @@ const ipfs = ipfsClient({ timeout: '2m' })
   - [`ipfs.pin.ls([hash], [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/PIN.md#pinls)
   - [`ipfs.pin.rm(hash, [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/PIN.md#pinrm)
 
-- refs
-  - `ipfs.refs.local()`
-
 #### Network
 
 - [bootstrap](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/BOOTSTRAP.md)
@@ -326,8 +306,6 @@ const ipfs = ipfsClient({ timeout: '2m' })
   - [`ipfs.dns(domain)`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/MISCELLANEOUS.md#dns)
   - [`ipfs.id()`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/MISCELLANEOUS.md#id)
   - [`ipfs.ping(id, [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/MISCELLANEOUS.md#ping)
-  - [`ipfs.pingPullStream(id, [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/MISCELLANEOUS.md#pingpullstream)
-  - [`ipfs.pingReadableStream(id, [options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/MISCELLANEOUS.md#pingreadablestream)
   - [`ipfs.stop()`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/MISCELLANEOUS.md#stop). Alias to `ipfs.shutdown`.
   - [`ipfs.version()`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/MISCELLANEOUS.md#version)
 
@@ -341,8 +319,6 @@ const ipfs = ipfsClient({ timeout: '2m' })
 - [stats](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/STATS.md)
   - [`ipfs.stats.bitswap()`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/STATS.md#statsbitswap)
   - [`ipfs.stats.bw([options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/STATS.md#statsbw)
-  - [`ipfs.stats.bwPullStream([options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/STATS.md#statsbwpullstream)
-  - [`ipfs.stats.bwReadableStream([options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/STATS.md#statsbwreadablestream)
   - [`ipfs.stats.repo([options])`](https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/STATS.md#statsrepo)
 
 - log
@@ -382,15 +358,14 @@ Call this on your client instance to return an object containing the `host`, `po
 
 Aside from the default export, `ipfs-http-client` exports various types and utilities that are included in the bundle:
 
-- [`isIPFS`](https://www.npmjs.com/package/is-ipfs)
 - [`Buffer`](https://www.npmjs.com/package/buffer)
-- [`PeerId`](https://www.npmjs.com/package/peer-id)
-- [`PeerInfo`](https://www.npmjs.com/package/peer-info)
 - [`multiaddr`](https://www.npmjs.com/package/multiaddr)
 - [`multibase`](https://www.npmjs.com/package/multibase)
 - [`multicodec`](https://www.npmjs.com/package/multicodec)
-- [`multihash`](https://www.npmjs.com/package/multihash)
+- [`multihash`](https://www.npmjs.com/package/multihashes)
 - [`CID`](https://www.npmjs.com/package/cids)
+- [`globSource`](https://github.com/ipfs/js-ipfs-utils/blob/master/src/files/glob-source.js) (not available in the browser)
+- [`urlSource`](https://github.com/ipfs/js-ipfs-utils/blob/master/src/files/url-source.js)
 
 These can be accessed like this, for example:
 
@@ -400,6 +375,74 @@ const { CID } = require('ipfs-http-client')
 import { CID } from 'ipfs-http-client'
 ```
 
+##### Glob source
+
+A utility to allow files on the file system to be easily added to IPFS.
+
+###### `globSource(path, [options])`
+
+- `path`: A path to a single file or directory to glob from
+- `options`: Optional options
+- `options.recursive`: If `path` is a directory, use option `{ recursive: true }` to add the directory and all its sub-directories.
+- `options.ignore`: To exclude file globs from the directory, use option `{ ignore: ['ignore/this/folder/**', 'and/this/file'] }`.
+- `options.hidden`: Hidden/dot files (files or folders starting with a `.`, for example, `.git/`) are not included by default. To add them, use the option `{ hidden: true }`.
+
+Returns an async iterable that yields `{ path, content }` objects suitable for passing to `ipfs.add`.
+
+###### Example
+
+```js
+const IpfsHttpClient = require('ipfs-http-client')
+const { globSource } = IpfsHttpClient
+const ipfs = IpfsHttpClient()
+
+for await (const file of ipfs.add(globSource('./docs', { recursive: true }))) {
+  console.log(file)
+}
+/*
+{
+  path: 'docs/assets/anchor.js',
+  cid: CID('QmVHxRocoWgUChLEvfEyDuuD6qJ4PhdDL2dTLcpUy3dSC2'),
+  size: 15347
+}
+{
+  path: 'docs/assets/bass-addons.css',
+  cid: CID('QmPiLWKd6yseMWDTgHegb8T7wVS7zWGYgyvfj7dGNt2viQ'),
+  size: 232
+}
+...
+*/
+```
+
+##### URL source
+
+A utility to allow content from the internet to be easily added to IPFS.
+
+###### `urlSource(url)`
+
+- `url`: A string URL or [`URL`](https://developer.mozilla.org/en-US/docs/Web/API/URL) instance to send HTTP GET request to
+
+Returns an async iterable that yields `{ path, content }` objects suitable for passing to `ipfs.add`.
+
+###### Example
+
+```js
+const IpfsHttpClient = require('ipfs-http-client')
+const { urlSource } = IpfsHttpClient
+const ipfs = IpfsHttpClient()
+
+for await (const file of ipfs.add(urlSource('https://ipfs.io/images/ipfs-logo.svg'))) {
+  console.log(file)
+}
+/*
+{
+  path: 'ipfs-logo.svg',
+  cid: CID('QmTqZhR6f7jzdhLgPArDPnsbZpvvgxzCZycXK7ywkLxSyU'),
+  size: 3243
+}
+*/
+```
+
 ## Development
 
 ### Testing

Diff for: package.json

@@ -15,13 +15,9 @@
   ],
   "main": "src/index.js",
   "browser": {
-    "glob": false,
-    "fs": false,
-    "stream": "readable-stream",
-    "ky-universal": "ky/umd",
     "./src/add/form-data.js": "./src/add/form-data.browser.js",
-    "./src/add-from-fs/index.js": "./src/add-from-fs/index.browser.js",
-    "./src/lib/buffer-to-form-data.js": "./src/lib/buffer-to-form-data.browser.js"
+    "./src/lib/buffer-to-form-data.js": "./src/lib/buffer-to-form-data.browser.js",
+    "ipfs-utils/src/files/glob-source": false
   },
   "repository": "github:ipfs/js-ipfs-http-client",
   "scripts": {
@@ -42,55 +38,42 @@
   },
   "dependencies": {
     "abort-controller": "^3.0.0",
-    "async-iterator-to-pull-stream": "^1.3.0",
     "bignumber.js": "^9.0.0",
-    "bl": "^4.0.0",
     "bs58": "^4.0.1",
     "buffer": "^5.4.2",
-    "callbackify": "^1.1.0",
     "cids": "~0.7.1",
     "debug": "^4.1.0",
-    "err-code": "^2.0.0",
-    "explain-error": "^1.0.4",
     "form-data": "^3.0.0",
     "ipfs-block": "~0.8.1",
-    "ipfs-utils": "^0.4.2",
-    "ipld-dag-cbor": "~0.15.0",
-    "ipld-dag-pb": "^0.18.1",
-    "ipld-raw": "^4.0.0",
-    "is-ipfs": "~0.6.1",
-    "it-all": "^1.0.1",
-    "it-glob": "0.0.7",
+    "ipfs-utils": "^0.7.1",
+    "ipld-dag-cbor": "^0.15.1",
+    "ipld-dag-pb": "^0.18.2",
+    "ipld-raw": "^4.0.1",
     "it-tar": "^1.1.1",
     "it-to-stream": "^0.1.1",
     "iterable-ndjson": "^1.1.0",
     "ky": "^0.15.0",
     "ky-universal": "^0.3.0",
     "merge-options": "^2.0.0",
-    "multiaddr": "^6.0.6",
-    "multiaddr-to-uri": "^5.0.0",
+    "multiaddr": "^7.2.1",
+    "multiaddr-to-uri": "^5.1.0",
     "multibase": "~0.6.0",
     "multicodec": "^1.0.0",
     "multihashes": "~0.4.14",
     "parse-duration": "^0.1.1",
-    "peer-id": "~0.12.3",
-    "peer-info": "~0.15.1",
-    "promise-nodeify": "^3.0.1"
+    "stream-to-it": "^0.2.0"
   },
   "devDependencies": {
     "aegir": "^20.4.1",
     "async": "^3.1.0",
     "browser-process-platform": "~0.1.1",
-    "cross-env": "^6.0.0",
-    "detect-node": "^2.0.4",
     "go-ipfs-dep": "^0.4.22",
-    "interface-ipfs-core": "^0.127.0",
-    "ipfsd-ctl": "^1.0.0",
-    "ndjson": "^1.5.0",
-    "nock": "^11.4.0",
-    "pull-stream": "^3.6.14",
-    "pump": "^3.0.0",
-    "stream-equal": "^1.1.1"
+    "interface-ipfs-core": "github:ipfs/interface-js-ipfs-core#refactor/async-iterables",
+    "ipfsd-ctl": "^1.0.2",
+    "it-all": "^1.0.1",
+    "it-concat": "^1.0.0",
+    "it-pipe": "^1.1.0",
+    "nock": "^11.7.2"
   },
   "engines": {
     "node": ">=10.3.0",

Diff for: src/add-from-fs/index.browser.js

@@ -1,8 +1,9 @@
 'use strict'
 
 const ndjson = require('iterable-ndjson')
+const CID = require('cids')
 const configure = require('../lib/configure')
-const toIterable = require('../lib/stream-to-iterable')
+const toIterable = require('stream-to-it/source')
 const { toFormData } = require('./form-data')
 const toCamel = require('../lib/object-to-camel')
 
@@ -55,7 +56,7 @@ module.exports = configure(({ ky }) => {
 function toCoreInterface ({ name, hash, size, mode, mtime }) {
   const output = {
     path: name,
-    hash,
+    cid: new CID(hash),
     size: parseInt(size)
   }
 

Diff for: src/add-from-fs/index.js

@@ -1,9 +1,7 @@
 'use strict'
 
-const callbackify = require('callbackify')
-
-module.exports = (config) => ({
-  wantlist: callbackify.variadic(require('./wantlist')(config)),
-  stat: callbackify.variadic(require('./stat')(config)),
-  unwant: callbackify.variadic(require('./unwant')(config))
+module.exports = config => ({
+  wantlist: require('./wantlist')(config),
+  stat: require('./stat')(config),
+  unwant: require('./unwant')(config)
 })

Diff for: src/add-from-url.js

@@ -2,6 +2,7 @@
 
 const configure = require('../lib/configure')
 const Big = require('bignumber.js')
+const CID = require('cids')
 
 module.exports = configure(({ ky }) => {
   return async (options) => {
@@ -21,8 +22,8 @@ module.exports = configure(({ ky }) => {
 function toCoreInterface (res) {
   return {
     provideBufLen: res.ProvideBufLen,
-    wantlist: res.Wantlist || [],
-    peers: res.Peers || [],
+    wantlist: (res.Wantlist || []).map(k => new CID(k['/'])),
+    peers: (res.Peers || []).map(p => new CID(p)),
     blocksReceived: new Big(res.BlocksReceived),
     dataReceived: new Big(res.DataReceived),
     blocksSent: new Big(res.BlocksSent),

Diff for: src/add/index.js

@@ -24,6 +24,6 @@ module.exports = configure(({ ky }) => {
       searchParams
     }).json()
 
-    return res
+    return (res.Keys || []).map(k => new CID(k['/']))
   }
 })

Diff for: src/bitswap/index.js

@@ -1,23 +1,8 @@
 'use strict'
 
-const nodeify = require('promise-nodeify')
-const callbackify = require('callbackify')
-const { collectify } = require('../lib/converters')
-
-module.exports = config => {
-  const rm = require('./rm-async-iterator')(config)
-
-  return {
-    get: callbackify.variadic(require('./get')(config)),
-    stat: callbackify.variadic(require('./stat')(config)),
-    put: callbackify.variadic(require('./put')(config)),
-    rm: (input, options, callback) => {
-      if (typeof options === 'function') {
-        callback = options
-        options = {}
-      }
-      return nodeify(collectify(rm)(input, options), callback)
-    },
-    _rmAsyncIterator: rm
-  }
-}
+module.exports = config => ({
+  get: require('./get')(config),
+  stat: require('./stat')(config),
+  put: require('./put')(config),
+  rm: require('./rm')(config)
+})