feat: add MFS commands write, cp, mkdir

README.md

@@ -166,17 +166,17 @@ This module is in heavy development, not all API methods are available (or docum
 * dht.put
 * dht.query
 * dns
-* files.cp
+* [files.cp](./API.md#filescp)
 * files.flush
 * files.ls
 * files.lsPullStream
-* files.mkdir
+* [files.mkdir](./API.md#filesmkdir)
 * files.mv
 * files.read
 * files.readPullStream
 * files.rm
 * files.stat
-* files.write
+* [files.write](./API.md#fileswrite)
 * [id](./API.md#id) TODO: add docs
 * key.export
 * key.gen

package.json

@@ -6,7 +6,8 @@
   "main": "src/index.js",
   "browser": {
     "./src/lib/configure.js": "./src/lib/configure.browser.js",
-    "./src/add/form-data.js": "./src/add/form-data.browser.js"
+    "./src/add/form-data.js": "./src/add/form-data.browser.js",
+    "./src/files/write/form-data.js": "./src/files/write/form-data.browser.js"
   },
   "browserslist": ">1.5% or node >=10 and not ios_saf <13 and not ie 11 and not dead",
   "files": [
@@ -42,7 +43,7 @@
     "bignumber.js": "^9.0.0",
     "chai": "^4.2.0",
     "dirty-chai": "^2.0.1",
-    "go-ipfs-dep": "~0.4.21",
+    "go-ipfs-dep": "^0.4.21",
     "interface-ipfs-core": "~0.105.1",
     "ipfs-block": "^0.8.1",
     "ipfsd-ctl": "^0.43.0",

src/add/form-data.browser.js

@@ -3,7 +3,7 @@
 
 const normaliseInput = require('./normalise-input')
 
-exports.toFormData = async function toFormData (input) {
+exports.toFormData = async (input) => {
   const files = normaliseInput(input)
   const formData = new FormData()
   let i = 0

src/add/form-data.js

@@ -1,11 +1,10 @@
 'use strict'
 
-// const toStream = require('async-iterator-to-stream')
 const FormData = require('form-data')
-const { Readable } = require('stream')
 const normaliseInput = require('./normalise-input')
+const toStream = require('../lib/iterable-to-readable-stream')
 
-exports.toFormData = async function toFormData (input) {
+exports.toFormData = async (input) => {
   // In Node.js, FormData can be passed a stream so no need to buffer
   const files = normaliseInput(input)
   const formData = new FormData()
@@ -40,26 +39,3 @@ exports.toFormData = async function toFormData (input) {
 
   return formData
 }
-
-function toStream (iterable) {
-  let reading = false
-  return new Readable({
-    async read (size) {
-      if (reading) return
-      reading = true
-
-      try {
-        while (true) {
-          const { value, done } = await iterable.next(size)
-          if (done) return this.push(null)
-          if (!this.push(value)) break
-        }
-      } catch (err) {
-        this.emit('error', err)
-        if (iterable.return) iterable.return()
-      } finally {
-        reading = false
-      }
-    }
-  })
-}

src/add/index.js

@@ -27,7 +27,8 @@ module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => {
       'shard-split-threshold': options.shardSplitThreshold,
       silent: options.silent,
       trickle: options.trickle,
-      'wrap-with-directory': options.wrapWithDirectory
+      'wrap-with-directory': options.wrapWithDirectory,
+      ...(options.qs || {})
     })
 
     const url = `${apiUrl}${apiPath}/add${qs}`

src/add/normalise-input.js

@@ -1,9 +1,8 @@
 'use strict'
 /* eslint-env browser */
 
-const toIterator = require('pull-stream-to-async-iterator')
 const { Buffer } = require('buffer')
-const blobToAsyncIterable = require('../lib/blob-to-async-iterable')
+const toAsyncIterable = require('../lib/file-data-to-async-iterable')
 
 /*
 Transform one of:
@@ -126,38 +125,3 @@ module.exports = function normalizeInput (input) {
 function normalizeTuple ({ path, content }) {
   return { path: path || '', content: content ? toAsyncIterable(content) : null }
 }
-
-function toAsyncIterable (input) {
-  // Buffer|ArrayBuffer|TypedArray|array of bytes
-  if (input[Symbol.iterator]) {
-    const buf = Buffer.from(input)
-    return Object.assign(
-      (async function * () { yield buf })(), // eslint-disable-line require-await
-      { length: buf.length }
-    )
-  }
-
-  // Blob|File
-  if (typeof Blob !== 'undefined' && input instanceof Blob) {
-    return Object.assign(
-      blobToAsyncIterable(input),
-      { length: input.size }
-    )
-  }
-
-  // AsyncIterable<Buffer>
-  if (input[Symbol.asyncIterator]) {
-    return (async function * () {
-      for await (const chunk of input) {
-        yield Buffer.from(chunk)
-      }
-    })()
-  }
-
-  // PullStream
-  if (typeof input === 'function') {
-    return toIterator(input)
-  }
-
-  throw new Error('Unexpected input: ' + typeof input)
-}

src/bitswap/stat.js

@@ -2,13 +2,15 @@
 
 const configure = require('../lib/configure')
 const { ok } = require('../lib/fetch')
+const { objectToQuery } = require('../lib/querystring')
 const toCamel = require('../lib/to-camel')
 
 module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => {
   return async (options) => {
     options = options || {}
 
-    const url = `${apiUrl}${apiPath}/bitswap/stat`
+    const qs = objectToQuery(options.qs)
+    const url = `${apiUrl}${apiPath}/bitswap/stat${qs}`
     const res = await ok(fetch(url, {
       signal: options.signal,
       headers: options.headers || headers

src/bitswap/wantlist.js

@@ -13,7 +13,12 @@ module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => {
 
     options = options || {}
 
-    const url = `${apiUrl}${apiPath}/bitswap/wantlist${objectToQuery({ peer: peerId })}`
+    const qs = objectToQuery({
+      peer: peerId,
+      ...(options.qs || {})
+    })
+
+    const url = `${apiUrl}${apiPath}/bitswap/wantlist${qs}`
     const res = await ok(fetch(url, {
       signal: options.signal,
       headers: options.headers || headers

src/block/get.js

@@ -3,12 +3,18 @@
 const { Buffer } = require('buffer')
 const configure = require('../lib/configure')
 const { ok } = require('../lib/fetch')
+const { objectToQuery } = require('../lib/querystring')
 
 module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => {
   return async (cid, options) => {
     options = options || {}
 
-    const url = `${apiUrl}${apiPath}/block/get?arg=${encodeURIComponent(cid)}`
+    const qs = objectToQuery({
+      arg: cid.toString(),
+      ...(options.qs || {})
+    })
+
+    const url = `${apiUrl}${apiPath}/block/get${qs}`
     const res = await ok(fetch(url, {
       signal: options.signal,
       headers: options.headers || headers

src/block/put.js

@@ -14,7 +14,8 @@ module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => {
       format: options.format,
       mhtype: options.mhtype,
       mhlen: options.mhlen,
-      pin: options.pin
+      pin: options.pin,
+      ...(options.qs || {})
     })
 
     const url = `${apiUrl}${apiPath}/block/put${qs}`

src/block/stat.js

@@ -1,14 +1,20 @@
 'use strict'
 
 const configure = require('../lib/configure')
+const { objectToQuery } = require('../lib/querystring')
 const { ok } = require('../lib/fetch')
 const toCamel = require('../lib/to-camel')
 
 module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => {
   return async (cid, options) => {
     options = options || {}
 
-    const url = `${apiUrl}${apiPath}/block/stat?arg=${encodeURIComponent(cid)}`
+    const qs = objectToQuery({
+      arg: cid.toString(),
+      ...(options.qs || {})
+    })
+
+    const url = `${apiUrl}${apiPath}/block/stat${qs}`
     const res = await ok(fetch(url, {
       signal: options.signal,
       headers: options.headers || headers

src/cat.js

@@ -3,12 +3,18 @@
 const { Buffer } = require('buffer')
 const configure = require('./lib/configure')
 const { ok, toIterable } = require('./lib/fetch')
+const { objectToQuery } = require('./lib/querystring')
 
 module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => {
   return (cid, options) => (async function * () {
     options = options || {}
 
-    const url = `${apiUrl}${apiPath}/cat?arg=${encodeURIComponent(cid)}`
+    const qs = objectToQuery({
+      arg: cid.toString(),
+      ...(options.qs || {})
+    })
+
+    const url = `${apiUrl}${apiPath}/cat${qs}`
     const res = await ok(fetch(url, {
       signal: options.signal,
       headers: options.headers || headers

src/files/cp.js

@@ -0,0 +1,32 @@
+'use strict'
+
+const configure = require('../lib/configure')
+const { ok } = require('../lib/fetch')
+const { objectToQuery } = require('../lib/querystring')
+
+module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => {
+  return async (...args) => {
+    let options = {}
+
+    if (typeof args[args.length - 1] === 'object') {
+      options = args.pop()
+    }
+
+    const qs = objectToQuery({
+      arg: args,
+      flush: options.flush,
+      format: options.format,
+      hash: options.hashAlg,
+      parents: options.parents,
+      ...(options.qs || {})
+    })
+
+    const url = `${apiUrl}${apiPath}/files/cp${qs}`
+    const res = await ok(fetch(url, {
+      signal: options.signal,
+      headers: options.headers || headers
+    }))
+
+    return res.text()
+  }
+})

src/files/mkdir.js

@@ -0,0 +1,29 @@
+'use strict'
+
+const configure = require('../lib/configure')
+const { ok } = require('../lib/fetch')
+const { objectToQuery } = require('../lib/querystring')
+
+module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => {
+  return async (path, options) => {
+    options = options || {}
+
+    const qs = objectToQuery({
+      arg: path,
+      'cid-version': options.cidVersion,
+      flush: options.flush,
+      format: options.format,
+      hash: options.hashAlg,
+      parents: options.parents,
+      ...(options.qs || {})
+    })
+
+    const url = `${apiUrl}${apiPath}/files/mkdir${qs}`
+    const res = await ok(fetch(url, {
+      signal: options.signal,
+      headers: options.headers || headers
+    }))
+
+    return res.text()
+  }
+})

src/files/write/form-data.browser.js

@@ -0,0 +1,21 @@
+'use strict'
+/* eslint-env browser */
+
+const toAsyncIterable = require('../../lib/file-data-to-async-iterable')
+
+exports.toFormData = async (path, input) => {
+  input = toAsyncIterable(input)
+  const formData = new FormData()
+
+  // In the browser there's _currently_ no streaming upload, buffer up our
+  // async iterator chunks and append a big Blob :(
+  // One day, this will be browser streams
+  const bufs = []
+  for await (const chunk of input) {
+    bufs.push(Buffer.isBuffer(chunk) ? chunk.buffer : chunk)
+  }
+
+  formData.append('file', new Blob(bufs, { type: 'application/octet-stream' }))
+
+  return formData
+}

src/files/write/form-data.js

@@ -0,0 +1,24 @@
+'use strict'
+
+const FormData = require('form-data')
+const toAsyncIterable = require('../../lib/file-data-to-async-iterable')
+const toStream = require('../../lib/iterable-to-readable-stream')
+
+exports.toFormData = (path, input) => {
+  input = toAsyncIterable(input)
+  const formData = new FormData()
+
+  formData.append(
+    'file',
+    // FIXME: add a `path` property to the stream so `form-data` doesn't set
+    // a Content-Length header that is only the sum of the size of the
+    // header/footer when knownLength option (below) is null.
+    Object.assign(toStream(input), { path }),
+    {
+      contentType: 'application/octet-stream',
+      knownLength: input.length // Send Content-Length header if known
+    }
+  )
+
+  return formData
+}

src/files/write/index.js

@@ -0,0 +1,36 @@
+'use strict'
+
+const { objectToQuery } = require('../../lib/querystring')
+const configure = require('../../lib/configure')
+const { ok } = require('../../lib/fetch')
+const { toFormData } = require('./form-data')
+
+module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => {
+  return async (path, input, options) => {
+    options = options || {}
+
+    const qs = objectToQuery({
+      arg: path,
+      'stream-channels': true,
+      'cid-version': options.cidVersion,
+      count: options.count,
+      create: options.create,
+      hash: options.hashAlg,
+      offset: options.offset,
+      parents: options.parents,
+      'raw-leaves': options.rawLeaves,
+      truncate: options.truncate,
+      ...(options.qs || {})
+    })
+
+    const url = `${apiUrl}${apiPath}/files/write${qs}`
+    const res = await ok(fetch(url, {
+      method: 'POST',
+      signal: options.signal,
+      headers: options.headers || headers,
+      body: await toFormData(path, input)
+    }))
+
+    return res.text()
+  }
+})