diff --git a/API.md b/API.md index 4735113..04e5452 100644 --- a/API.md +++ b/API.md @@ -29,17 +29,17 @@ * dht.put * dht.query * dns -* files.cp +* [files.cp](#filescp) * files.flush * files.ls * files.lsPullStream -* files.mkdir +* [files.mkdir](#filesmkdir) * files.mv * files.read * files.readPullStream * files.rm * files.stat -* files.write +* [files.write](#fileswrite) * [id](#id) TODO: add docs * key.export * key.gen @@ -176,6 +176,9 @@ Add/import files and directories to IPFS and retrieve their CID(s). * `options.hashAlg` (optional) - Hashing algorithm to use when creating the CID(s). [Available values]( https://github.com/multiformats/js-multihash/blob/master/src/constants.js#L5-L343). * Type: `String` * Default: `sha2-256` +* `options.headers` (optional) - Custom HTTP headers to send + * Type: `Object` + * Default: `null` * `options.onlyHash` (optional) - Do not add the file(s) to IPFS, only calculate the CID(s). * Type: `Boolean` * Default: `false` @@ -185,6 +188,9 @@ Add/import files and directories to IPFS and retrieve their CID(s). * `options.progress` (optional) - A function that receives progress updates as data is added to IPFS. It is called with the byte length of chunks as a file is added to IPFS. * Type: `Function(bytes)` * Default: `null` +* `options.qs` (optional) - Additional query string parameters + * Type: `Object` + * Default: `null` * `options.quiet` (optional) - Return a minimal output. * Type: `Boolean` * Default: `false` @@ -401,6 +407,12 @@ Fetch a raw block from the IPFS block store or the network via bitswap if not lo * `options` (optional) * Type: `Object` * Default: `null` +* `options.headers` (optional) - Custom HTTP headers to send + * Type: `Object` + * Default: `null` +* `options.qs` (optional) - Additional query string parameters + * Type: `Object` + * Default: `null` * `options.signal` (optional) - A signal that can be used to abort the request * Type: [`AbortSignal`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal) * Default: `null` @@ -434,6 +446,9 @@ Put a block into the IPFS block store. * `options.format` (optional) - Name of the IPLD format this block is encoded with * Type: `String` * Default: `dag-pb` +* `options.headers` (optional) - Custom HTTP headers to send + * Type: `Object` + * Default: `null` * `options.mhtype` (optional) - Name of the multihash hashing algorithm to use * Type: `String` * Default: `sha2-256` @@ -443,6 +458,9 @@ Put a block into the IPFS block store. * `options.pin` (optional) - Pin this block so it is not garbage collected * Type: `Boolean` * Default: `false` +* `options.qs` (optional) - Additional query string parameters + * Type: `Object` + * Default: {} * `options.signal` (optional) - A signal that can be used to abort the request * Type: [`AbortSignal`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal) * Default: `null` @@ -484,6 +502,12 @@ Get status for a block. * `options` (optional) * Type: `Object` * Default: `null` +* `options.headers` (optional) - Custom HTTP headers to send + * Type: `Object` + * Default: `null` +* `options.qs` (optional) - Additional query string parameters + * Type: `Object` + * Default: `null` * `options.signal` (optional) - A signal that can be used to abort the request * Type: [`AbortSignal`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal) * Default: `null` @@ -524,12 +548,18 @@ Read files from IPFS. * `options` (optional) * Type: `Object` * Default: `null` +* `options.headers` (optional) - Custom HTTP headers to send + * Type: `Object` + * Default: `null` * `options.offset` (optional) - Byte offset to start reading from * Type: `Number` * Default: `0` * `options.length` (optional) - Number of bytes to read * Type: `Number` * Default: `null` (read to the end of the file) +* `options.qs` (optional) - Additional query string parameters + * Type: `Object` + * Default: `null` * `options.signal` (optional) - A signal that can be used to abort the request * Type: [`AbortSignal`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal) * Default: `null` @@ -562,6 +592,186 @@ hello world! */ ``` +## files.cp + +Copy files into [MFS](https://docs.ipfs.io/guides/concepts/mfs/). + +### `files.cp(...from, to, [options]): Promise` + +#### Parameters + +* `from` - Path(s) of the source to copy. It might be an existing MFS path to a file or a directory (e.g. `/my-dir/my-file.txt`) or an IPFS path (e.g. `/ipfs/QmWGeRAEgtsHW3ec7U4qW2CyVy7eA2mFRVbk1nb24jFyks`) + * Type: `String` +* `to` - Path of the destination to copy to + * Type: `String` +* `options` (optional) + * Type: `Object` + * Default: `null` +* `options.flush` (optional) - Immediately flush MFS changes to disk + * Type: `Boolean` + * Default: `true` +* `options.format` (optional) - Type of nodes to write any newly created directories as + * Type: `String` + * Default: `dag-pb` +* `options.hashAlg` (optional) - Hashing algorithm to use when creating the CID(s). [Available values]( https://github.com/multiformats/js-multihash/blob/master/src/constants.js#L5-L343). + * Type: `String` + * Default: `sha2-256` +* `options.headers` (optional) - Custom HTTP headers to send + * Type: `Object` + * Default: `null` +* `options.parents` (optional) - Create parent directories if they do not exist + * Type: `Boolean` + * Default: `false` +* `options.qs` (optional) - Additional query string parameters + * Type: `Object` + * Default: `null` +* `options.signal` (optional) - A signal that can be used to abort the request + * Type: [`AbortSignal`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal) + * Default: `null` + +If `from` has multiple values then `to` must be a directory. + +If `from` has a single value and `to` exists and is a directory, `from` will be copied into `to`. + +If `from` has a single value and `to` exists and is a file, `from` must be a file and the contents of `to` will be replaced with the contents of `from` otherwise an error will be thrown. + +If `from` is an IPFS path, and an MFS path exists with the same name, the IPFS path will be chosen. + +#### Returns + +A promise that resolves when the operation is complete. + +* Type: `Promise` + +#### Examples + +```js +// To copy a file +await ipfs.files.cp('/src-file', '/dst-file') + +// To copy a directory +await ipfs.files.cp('/src-dir', '/dst-dir') + +// To copy multiple files to a directory +await ipfs.files.cp('/src-file1', '/src-file2', '/dst-dir') +``` + +## files.mkdir + +Make a directory in [MFS](https://docs.ipfs.io/guides/concepts/mfs/). + +### `files.mkdir(path, [options]): Promise` + +#### Parameters + +* `path` - path to the directory to make + * Type: `String` +* `options` (optional) + * Type: `Object` + * Default: `null` +* `options.cidVersion` (optional) - The CID version to use when storing the data (storage keys are based on the CID, including its version). + * Type: `Number` (0 or 1) + * Default: `0` +* `options.flush` (optional) - Immediately flush MFS changes to disk + * Type: `Boolean` + * Default: `true` +* `options.format` (optional) - Type of nodes to write any newly created directories as + * Type: `String` + * Default: `dag-pb` +* `options.hashAlg` (optional) - Hashing algorithm to use when creating the CID(s). [Available values]( https://github.com/multiformats/js-multihash/blob/master/src/constants.js#L5-L343). + * Type: `String` + * Default: `sha2-256` +* `options.headers` (optional) - Custom HTTP headers to send + * Type: `Object` + * Default: `null` +* `options.parents` (optional) - Create parent directories if they do not exist + * Type: `Boolean` + * Default: `false` +* `options.qs` (optional) - Additional query string parameters + * Type: `Object` + * Default: `null` +* `options.signal` (optional) - A signal that can be used to abort the request + * Type: [`AbortSignal`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal) + * Default: `null` + +#### Returns + +A promise that resolves when the operation is complete. + +* Type: `Promise` + +#### Examples + +```js +await ipfs.files.mkdir('/my/beautiful/directory') +``` + +## files.write + +Write to a file in [MFS](https://docs.ipfs.io/guides/concepts/mfs/). + +### `files.write(path, content, [options]): Promise` + +#### Parameters + +* `path` - Path to the file that should be written to + * Type: `String` +* `content` - File content + * Type (one of): + * `Buffer`, or "buffer like": `ArrayBuffer`/`TypedArray` + * `Blob`/`File` (browser only) + * `Iterable` (e.g. array of bytes) + * `AsyncIterable` (e.g. a Node.js Stream) + * `PullStream` +* `options` (optional) + * Type: `Object` + * Default: `null` +* `options.cidVersion` (optional) - The CID version to use when storing the data (storage keys are based on the CID, including its version). + * Type: `Number` (0 or 1) + * Default: `0` +* `options.count` (optional) - Number of bytes to write + * Type: `Number` + * Default: All bytes provided by `content` +* `options.create` (optional) - Create file if it does not exist + * Type: `Boolean` + * Default: `false` +* `options.hashAlg` (optional) - Hashing algorithm to use when creating the CID(s). [Available values]( https://github.com/multiformats/js-multihash/blob/master/src/constants.js#L5-L343). + * Type: `String` + * Default: `sha2-256` +* `options.headers` (optional) - Custom HTTP headers to send + * Type: `Object` + * Default: `null` +* `options.offset` (optional) - Byte offset to begin writing at + * Type: `Number` + * Default: `0` +* `options.parents` (optional) - Create parent directories if they do not exist + * Type: `Boolean` + * Default: `false` +* `options.qs` (optional) - Additional query string parameters + * Type: `Object` + * Default: `null` +* `options.rawLeaves` (optional) - DAG leaf nodes contain raw file data and are not wrapped in a protobuf + * Type: `Boolean` + * Default: `false` +* `options.signal` (optional) - A signal that can be used to abort the request + * Type: [`AbortSignal`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal) + * Default: `null` +* `options.truncate` (optional) - Truncate the file to size zero before writing + * Type: `Boolean` + * Default: `false` + +#### Returns + +A promise that resolves when the operation is complete. + +* Type: `Promise` + +#### Examples + +```js +await ipfs.files.write('/hello-world', Buffer.from('Hello, world!')) +``` + ## pubsub.ls List subscribed topics by name. @@ -573,6 +783,12 @@ List subscribed topics by name. * `options` (optional) * Type: `Object` * Default: `null` +* `options.headers` (optional) - Custom HTTP headers to send + * Type: `Object` + * Default: `null` +* `options.qs` (optional) - Additional query string parameters + * Type: `Object` + * Default: `null` * `options.signal` (optional) - A signal that can be used to abort the request * Type: [`AbortSignal`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal) * Default: `null` @@ -607,6 +823,12 @@ List peers we are currently pubsubbing with, optionally filtered by topic name. * `options` (optional) * Type: `Object` * Default: `null` +* `options.headers` (optional) - Custom HTTP headers to send + * Type: `Object` + * Default: `null` +* `options.qs` (optional) - Additional query string parameters + * Type: `Object` + * Default: `null` * `options.signal` (optional) - A signal that can be used to abort the request * Type: [`AbortSignal`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal) * Default: `null` @@ -642,6 +864,12 @@ Publish a message to a given pubsub topic. * `options` (optional) * Type: `Object` * Default: `null` +* `options.headers` (optional) - Custom HTTP headers to send + * Type: `Object` + * Default: `null` +* `options.qs` (optional) - Additional query string parameters + * Type: `Object` + * Default: `null` * `options.signal` (optional) - A signal that can be used to abort the request * Type: [`AbortSignal`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal) * Default: `null` @@ -686,9 +914,15 @@ Subscribe to messages on a given topic. * `options.discover` (optional) - Try to discover other peers subscribed to the same topic * Type: `Boolean` * Deafult: `false` +* `options.headers` (optional) - Custom HTTP headers to send + * Type: `Object` + * Default: `null` * `options.onError` (optional) - An error handler called when the request errors or parsing of a given message fails. It is passed two parameters, the error that occurred and a boolean indicating if it was a fatal error or not (fatal errors terminate the subscription). * Type: `Function(err, fatal)` * Default: `null` +* `options.qs` (optional) - Additional query string parameters + * Type: `Object` + * Default: `null` * `options.signal` (optional) - A signal that can be used to abort the request * Type: [`AbortSignal`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal) * Default: `null` @@ -721,7 +955,7 @@ data: hi Stop receiving messages for a given topic. -### `pubsub.unsubscribe(topic, [handler], [options]): Promise` +### `pubsub.unsubscribe(topic, [handler]): Promise` #### Parameters @@ -730,12 +964,6 @@ Stop receiving messages for a given topic. * `handler` (optional) - The handler function currently registered for this topic. If not provided, **all** handlers for the passed topic will be unsubscribed. Note this only works using the Promise API. * Type: `Function` * Default: `null` -* `options` (optional) - * Type: `Object` - * Default: `null` -* `options.signal` (optional) - A signal that can be used to abort the request - * Type: [`AbortSignal`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal) - * Default: `null` #### Returns @@ -769,6 +997,12 @@ Open a connection to a given address. * `options` (optional) * Type: `Object` * Default: `null` +* `options.headers` (optional) - Custom HTTP headers to send + * Type: `Object` + * Default: `null` +* `options.qs` (optional) - Additional query string parameters + * Type: `Object` + * Default: `null` * `options.signal` (optional) - A signal that can be used to abort the request * Type: [`AbortSignal`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal) * Default: `null` @@ -805,12 +1039,18 @@ List peers with open connections. * `options.direction` (optional) - Return direction information for each peer (inbound/outbound). * Type: `Boolean` * Default: `false` -* `options.signal` (optional) - A signal that can be used to abort the request - * Type: [`AbortSignal`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal) +* `options.headers` (optional) - Custom HTTP headers to send + * Type: `Object` * Default: `null` * `options.latency` (optional) - Return latency information for each peer. * Type: `Boolean` * Default: `false` +* `options.qs` (optional) - Additional query string parameters + * Type: `Object` + * Default: `null` +* `options.signal` (optional) - A signal that can be used to abort the request + * Type: [`AbortSignal`](https://developer.mozilla.org/en-US/docs/Web/API/AbortSignal) + * Default: `null` * `options.streams` (optional) - Return streams information for each peer. * Type: `Boolean` * Default: `false` diff --git a/README.md b/README.md index 90d2cb6..19e9350 100644 --- a/README.md +++ b/README.md @@ -166,17 +166,17 @@ This module is in heavy development, not all API methods are available (or docum * dht.put * dht.query * dns -* files.cp +* [files.cp](./API.md#filescp) * files.flush * files.ls * files.lsPullStream -* files.mkdir +* [files.mkdir](./API.md#filesmkdir) * files.mv * files.read * files.readPullStream * files.rm * files.stat -* files.write +* [files.write](./API.md#fileswrite) * [id](./API.md#id) TODO: add docs * key.export * key.gen diff --git a/package.json b/package.json index 2bb2ca2..8af4b04 100644 --- a/package.json +++ b/package.json @@ -6,7 +6,8 @@ "main": "src/index.js", "browser": { "./src/lib/configure.js": "./src/lib/configure.browser.js", - "./src/add/form-data.js": "./src/add/form-data.browser.js" + "./src/add/form-data.js": "./src/add/form-data.browser.js", + "./src/files/write/form-data.js": "./src/files/write/form-data.browser.js" }, "browserslist": ">1.5% or node >=10 and not ios_saf <13 and not ie 11 and not dead", "files": [ @@ -42,7 +43,7 @@ "bignumber.js": "^9.0.0", "chai": "^4.2.0", "dirty-chai": "^2.0.1", - "go-ipfs-dep": "~0.4.21", + "go-ipfs-dep": "^0.4.21", "interface-ipfs-core": "~0.105.1", "ipfs-block": "^0.8.1", "ipfsd-ctl": "^0.43.0", diff --git a/src/add/form-data.browser.js b/src/add/form-data.browser.js index 5ea6054..46f0254 100644 --- a/src/add/form-data.browser.js +++ b/src/add/form-data.browser.js @@ -3,7 +3,7 @@ const normaliseInput = require('./normalise-input') -exports.toFormData = async function toFormData (input) { +exports.toFormData = async (input) => { const files = normaliseInput(input) const formData = new FormData() let i = 0 diff --git a/src/add/form-data.js b/src/add/form-data.js index 90dc0e1..ce82fbe 100644 --- a/src/add/form-data.js +++ b/src/add/form-data.js @@ -1,11 +1,10 @@ 'use strict' -// const toStream = require('async-iterator-to-stream') const FormData = require('form-data') -const { Readable } = require('stream') const normaliseInput = require('./normalise-input') +const toStream = require('../lib/iterable-to-readable-stream') -exports.toFormData = async function toFormData (input) { +exports.toFormData = async (input) => { // In Node.js, FormData can be passed a stream so no need to buffer const files = normaliseInput(input) const formData = new FormData() @@ -40,26 +39,3 @@ exports.toFormData = async function toFormData (input) { return formData } - -function toStream (iterable) { - let reading = false - return new Readable({ - async read (size) { - if (reading) return - reading = true - - try { - while (true) { - const { value, done } = await iterable.next(size) - if (done) return this.push(null) - if (!this.push(value)) break - } - } catch (err) { - this.emit('error', err) - if (iterable.return) iterable.return() - } finally { - reading = false - } - } - }) -} diff --git a/src/add/index.js b/src/add/index.js index 76c3f6e..2bf59ec 100644 --- a/src/add/index.js +++ b/src/add/index.js @@ -27,7 +27,8 @@ module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => { 'shard-split-threshold': options.shardSplitThreshold, silent: options.silent, trickle: options.trickle, - 'wrap-with-directory': options.wrapWithDirectory + 'wrap-with-directory': options.wrapWithDirectory, + ...(options.qs || {}) }) const url = `${apiUrl}${apiPath}/add${qs}` diff --git a/src/add/normalise-input.js b/src/add/normalise-input.js index 3750fcc..b11e07d 100644 --- a/src/add/normalise-input.js +++ b/src/add/normalise-input.js @@ -1,9 +1,8 @@ 'use strict' /* eslint-env browser */ -const toIterator = require('pull-stream-to-async-iterator') const { Buffer } = require('buffer') -const blobToAsyncIterable = require('../lib/blob-to-async-iterable') +const toAsyncIterable = require('../lib/file-data-to-async-iterable') /* Transform one of: @@ -126,38 +125,3 @@ module.exports = function normalizeInput (input) { function normalizeTuple ({ path, content }) { return { path: path || '', content: content ? toAsyncIterable(content) : null } } - -function toAsyncIterable (input) { - // Buffer|ArrayBuffer|TypedArray|array of bytes - if (input[Symbol.iterator]) { - const buf = Buffer.from(input) - return Object.assign( - (async function * () { yield buf })(), // eslint-disable-line require-await - { length: buf.length } - ) - } - - // Blob|File - if (typeof Blob !== 'undefined' && input instanceof Blob) { - return Object.assign( - blobToAsyncIterable(input), - { length: input.size } - ) - } - - // AsyncIterable - if (input[Symbol.asyncIterator]) { - return (async function * () { - for await (const chunk of input) { - yield Buffer.from(chunk) - } - })() - } - - // PullStream - if (typeof input === 'function') { - return toIterator(input) - } - - throw new Error('Unexpected input: ' + typeof input) -} diff --git a/src/bitswap/stat.js b/src/bitswap/stat.js index 1da189f..1b10c35 100644 --- a/src/bitswap/stat.js +++ b/src/bitswap/stat.js @@ -2,13 +2,15 @@ const configure = require('../lib/configure') const { ok } = require('../lib/fetch') +const { objectToQuery } = require('../lib/querystring') const toCamel = require('../lib/to-camel') module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => { return async (options) => { options = options || {} - const url = `${apiUrl}${apiPath}/bitswap/stat` + const qs = objectToQuery(options.qs) + const url = `${apiUrl}${apiPath}/bitswap/stat${qs}` const res = await ok(fetch(url, { signal: options.signal, headers: options.headers || headers diff --git a/src/bitswap/wantlist.js b/src/bitswap/wantlist.js index 21ee8ac..8d31b01 100644 --- a/src/bitswap/wantlist.js +++ b/src/bitswap/wantlist.js @@ -13,7 +13,12 @@ module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => { options = options || {} - const url = `${apiUrl}${apiPath}/bitswap/wantlist${objectToQuery({ peer: peerId })}` + const qs = objectToQuery({ + peer: peerId, + ...(options.qs || {}) + }) + + const url = `${apiUrl}${apiPath}/bitswap/wantlist${qs}` const res = await ok(fetch(url, { signal: options.signal, headers: options.headers || headers diff --git a/src/block/get.js b/src/block/get.js index de4ceeb..83e0204 100644 --- a/src/block/get.js +++ b/src/block/get.js @@ -3,12 +3,18 @@ const { Buffer } = require('buffer') const configure = require('../lib/configure') const { ok } = require('../lib/fetch') +const { objectToQuery } = require('../lib/querystring') module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => { return async (cid, options) => { options = options || {} - const url = `${apiUrl}${apiPath}/block/get?arg=${encodeURIComponent(cid)}` + const qs = objectToQuery({ + arg: cid.toString(), + ...(options.qs || {}) + }) + + const url = `${apiUrl}${apiPath}/block/get${qs}` const res = await ok(fetch(url, { signal: options.signal, headers: options.headers || headers diff --git a/src/block/put.js b/src/block/put.js index 2698396..66014c8 100644 --- a/src/block/put.js +++ b/src/block/put.js @@ -14,7 +14,8 @@ module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => { format: options.format, mhtype: options.mhtype, mhlen: options.mhlen, - pin: options.pin + pin: options.pin, + ...(options.qs || {}) }) const url = `${apiUrl}${apiPath}/block/put${qs}` diff --git a/src/block/stat.js b/src/block/stat.js index a9d0a24..b47162e 100644 --- a/src/block/stat.js +++ b/src/block/stat.js @@ -1,6 +1,7 @@ 'use strict' const configure = require('../lib/configure') +const { objectToQuery } = require('../lib/querystring') const { ok } = require('../lib/fetch') const toCamel = require('../lib/to-camel') @@ -8,7 +9,12 @@ module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => { return async (cid, options) => { options = options || {} - const url = `${apiUrl}${apiPath}/block/stat?arg=${encodeURIComponent(cid)}` + const qs = objectToQuery({ + arg: cid.toString(), + ...(options.qs || {}) + }) + + const url = `${apiUrl}${apiPath}/block/stat${qs}` const res = await ok(fetch(url, { signal: options.signal, headers: options.headers || headers diff --git a/src/cat.js b/src/cat.js index 4c17f45..90519d4 100644 --- a/src/cat.js +++ b/src/cat.js @@ -3,12 +3,18 @@ const { Buffer } = require('buffer') const configure = require('./lib/configure') const { ok, toIterable } = require('./lib/fetch') +const { objectToQuery } = require('./lib/querystring') module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => { return (cid, options) => (async function * () { options = options || {} - const url = `${apiUrl}${apiPath}/cat?arg=${encodeURIComponent(cid)}` + const qs = objectToQuery({ + arg: cid.toString(), + ...(options.qs || {}) + }) + + const url = `${apiUrl}${apiPath}/cat${qs}` const res = await ok(fetch(url, { signal: options.signal, headers: options.headers || headers diff --git a/src/files/cp.js b/src/files/cp.js new file mode 100644 index 0000000..08baedc --- /dev/null +++ b/src/files/cp.js @@ -0,0 +1,32 @@ +'use strict' + +const configure = require('../lib/configure') +const { ok } = require('../lib/fetch') +const { objectToQuery } = require('../lib/querystring') + +module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => { + return async (...args) => { + let options = {} + + if (typeof args[args.length - 1] === 'object') { + options = args.pop() + } + + const qs = objectToQuery({ + arg: args, + flush: options.flush, + format: options.format, + hash: options.hashAlg, + parents: options.parents, + ...(options.qs || {}) + }) + + const url = `${apiUrl}${apiPath}/files/cp${qs}` + const res = await ok(fetch(url, { + signal: options.signal, + headers: options.headers || headers + })) + + return res.text() + } +}) diff --git a/src/files/mkdir.js b/src/files/mkdir.js new file mode 100644 index 0000000..fb01ad1 --- /dev/null +++ b/src/files/mkdir.js @@ -0,0 +1,29 @@ +'use strict' + +const configure = require('../lib/configure') +const { ok } = require('../lib/fetch') +const { objectToQuery } = require('../lib/querystring') + +module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => { + return async (path, options) => { + options = options || {} + + const qs = objectToQuery({ + arg: path, + 'cid-version': options.cidVersion, + flush: options.flush, + format: options.format, + hash: options.hashAlg, + parents: options.parents, + ...(options.qs || {}) + }) + + const url = `${apiUrl}${apiPath}/files/mkdir${qs}` + const res = await ok(fetch(url, { + signal: options.signal, + headers: options.headers || headers + })) + + return res.text() + } +}) diff --git a/src/files/write/form-data.browser.js b/src/files/write/form-data.browser.js new file mode 100644 index 0000000..f2ec032 --- /dev/null +++ b/src/files/write/form-data.browser.js @@ -0,0 +1,21 @@ +'use strict' +/* eslint-env browser */ + +const toAsyncIterable = require('../../lib/file-data-to-async-iterable') + +exports.toFormData = async (path, input) => { + input = toAsyncIterable(input) + const formData = new FormData() + + // In the browser there's _currently_ no streaming upload, buffer up our + // async iterator chunks and append a big Blob :( + // One day, this will be browser streams + const bufs = [] + for await (const chunk of input) { + bufs.push(Buffer.isBuffer(chunk) ? chunk.buffer : chunk) + } + + formData.append('file', new Blob(bufs, { type: 'application/octet-stream' })) + + return formData +} diff --git a/src/files/write/form-data.js b/src/files/write/form-data.js new file mode 100644 index 0000000..2a486d9 --- /dev/null +++ b/src/files/write/form-data.js @@ -0,0 +1,24 @@ +'use strict' + +const FormData = require('form-data') +const toAsyncIterable = require('../../lib/file-data-to-async-iterable') +const toStream = require('../../lib/iterable-to-readable-stream') + +exports.toFormData = (path, input) => { + input = toAsyncIterable(input) + const formData = new FormData() + + formData.append( + 'file', + // FIXME: add a `path` property to the stream so `form-data` doesn't set + // a Content-Length header that is only the sum of the size of the + // header/footer when knownLength option (below) is null. + Object.assign(toStream(input), { path }), + { + contentType: 'application/octet-stream', + knownLength: input.length // Send Content-Length header if known + } + ) + + return formData +} diff --git a/src/files/write/index.js b/src/files/write/index.js new file mode 100644 index 0000000..2901706 --- /dev/null +++ b/src/files/write/index.js @@ -0,0 +1,36 @@ +'use strict' + +const { objectToQuery } = require('../../lib/querystring') +const configure = require('../../lib/configure') +const { ok } = require('../../lib/fetch') +const { toFormData } = require('./form-data') + +module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => { + return async (path, input, options) => { + options = options || {} + + const qs = objectToQuery({ + arg: path, + 'stream-channels': true, + 'cid-version': options.cidVersion, + count: options.count, + create: options.create, + hash: options.hashAlg, + offset: options.offset, + parents: options.parents, + 'raw-leaves': options.rawLeaves, + truncate: options.truncate, + ...(options.qs || {}) + }) + + const url = `${apiUrl}${apiPath}/files/write${qs}` + const res = await ok(fetch(url, { + method: 'POST', + signal: options.signal, + headers: options.headers || headers, + body: await toFormData(path, input) + })) + + return res.text() + } +}) diff --git a/src/id.js b/src/id.js index fa3a3e4..a14e7e8 100644 --- a/src/id.js +++ b/src/id.js @@ -2,12 +2,14 @@ const configure = require('./lib/configure') const { ok } = require('./lib/fetch') +const { objectToQuery } = require('./lib/querystring') const toCamel = require('./lib/to-camel') module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => { return async options => { options = options || {} - const url = `${apiUrl}${apiPath}/id` + const qs = objectToQuery(options.qs) + const url = `${apiUrl}${apiPath}/id${qs}` const res = await ok(fetch(url, { signal: options.signal, headers: options.headers || headers diff --git a/src/index.js b/src/index.js index 5ab5aff..a747f5e 100644 --- a/src/index.js +++ b/src/index.js @@ -25,6 +25,11 @@ module.exports = config => { }, cat: callbackify(concatify(cat)), catPullStream: pullify.source(cat), + files: { + cp: callbackify(require('./files/cp')(config)), + mkdir: callbackify(require('./files/mkdir')(config)), + write: callbackify(require('./files/write')(config)) + }, ls: callbackify(collectify(ls)), lsPullStream: pullify.source(ls), id: callbackify(require('./id')(config)), diff --git a/src/lib/file-data-to-async-iterable.js b/src/lib/file-data-to-async-iterable.js new file mode 100644 index 0000000..ffa0f8d --- /dev/null +++ b/src/lib/file-data-to-async-iterable.js @@ -0,0 +1,54 @@ +'use strict' +/* eslint-env browser */ + +const toIterator = require('pull-stream-to-async-iterator') +const { Buffer } = require('buffer') +const blobToAsyncIterable = require('../lib/blob-to-async-iterable') + +/* +Transform one of: + +Buffer|ArrayBuffer|TypedArray +Blob|File +Iterable +AsyncIterable +PullStream + +Into: + +AsyncIterable +*/ +module.exports = function toAsyncIterable (input) { + // Buffer|ArrayBuffer|TypedArray|array of bytes + if (input[Symbol.iterator]) { + const buf = Buffer.from(input) + return Object.assign( + (async function * () { yield buf })(), // eslint-disable-line require-await + { length: buf.length } + ) + } + + // Blob|File + if (typeof Blob !== 'undefined' && input instanceof Blob) { + return Object.assign( + blobToAsyncIterable(input), + { length: input.size } + ) + } + + // AsyncIterable + if (input[Symbol.asyncIterator]) { + return (async function * () { + for await (const chunk of input) { + yield Buffer.from(chunk) + } + })() + } + + // PullStream + if (typeof input === 'function') { + return toIterator(input) + } + + throw new Error('Unexpected input: ' + typeof input) +} diff --git a/src/lib/form-data.browser.js b/src/lib/form-data.browser.js deleted file mode 100644 index 8b60885..0000000 --- a/src/lib/form-data.browser.js +++ /dev/null @@ -1,25 +0,0 @@ -'use strict' -/* eslint-env browser */ - -const isBuffer = require('is-buffer') - -exports.toFormData = async function toFormData (input) { - // If not an async iterator, this must conform to whatever FormData allows - if (!input[Symbol.asyncIterator]) { - const formData = new FormData() - formData.append('file', input) - return formData - } - - // In the browser there's _currently_ no streaming upload, buffer up our - // async iterator chunks and append a big Blob :( - - // One day, this will be browser streams - const formData = new FormData() - const bufs = [] - for await (const chunk of input) { - bufs.push(isBuffer(chunk) ? chunk.buffer : chunk) - } - formData.append('file', new Blob(bufs)) - return formData -} diff --git a/src/lib/form-data.js b/src/lib/form-data.js deleted file mode 100644 index ca3926b..0000000 --- a/src/lib/form-data.js +++ /dev/null @@ -1,25 +0,0 @@ -'use strict' - -// const toStream = require('async-iterator-to-stream') -const FormData = require('form-data') - -exports.toFormData = async function toFormData (input) { - // If not an async iterator, this must conform to whatever FormData allows - if (!input[Symbol.asyncIterator]) { - const formData = new FormData() - formData.append('file', input) - return formData - } - - // In Node.js, FormData can be passed a stream so no need to buffer - const formData = new FormData() - const bufs = [] - for await (const chunk of input) { - bufs.push(Buffer.from(chunk)) - } - // FIXME: the below does not work! It should do, but only the first chunk - // gets uploaded :( - // formData.append('file', toStream(input)) - formData.append('file', Buffer.concat(bufs)) - return formData -} diff --git a/src/lib/iterable-to-readable-stream.js b/src/lib/iterable-to-readable-stream.js new file mode 100644 index 0000000..f13d96d --- /dev/null +++ b/src/lib/iterable-to-readable-stream.js @@ -0,0 +1,26 @@ +'use strict' + +const { Readable } = require('stream') + +module.exports = function toStream (iterable) { + let reading = false + return new Readable({ + async read (size) { + if (reading) return + reading = true + + try { + while (true) { + const { value, done } = await iterable.next(size) + if (done) return this.push(null) + if (!this.push(value)) break + } + } catch (err) { + this.emit('error', err) + if (iterable.return) iterable.return() + } finally { + reading = false + } + } + }) +} diff --git a/src/lib/querystring.js b/src/lib/querystring.js index e0ffc9a..df35b0a 100644 --- a/src/lib/querystring.js +++ b/src/lib/querystring.js @@ -5,6 +5,8 @@ const QueryString = require('querystring') // Convert an object to a query string INCLUDING leading ? // Excludes null/undefined values exports.objectToQuery = obj => { + if (!obj) return '' + const qs = Object.entries(obj).reduce((obj, [key, value]) => { if (value != null) obj[key] = value return obj diff --git a/src/ls.js b/src/ls.js index d3107ef..7aa838e 100644 --- a/src/ls.js +++ b/src/ls.js @@ -9,7 +9,12 @@ module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => { return (path, options) => (async function * () { options = options || {} - const url = `${apiUrl}${apiPath}/ls${objectToQuery({ arg: path.toString() })}` + const qs = objectToQuery({ + arg: path.toString(), + ...(options.qs || {}) + }) + + const url = `${apiUrl}${apiPath}/ls${qs}` const res = await ok(fetch(url, { signal: options.signal, headers: options.headers || headers diff --git a/src/ping.js b/src/ping.js index 442b6d0..da96cf9 100644 --- a/src/ping.js +++ b/src/ping.js @@ -10,7 +10,12 @@ module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => { return (peerId, options) => (async function * () { options = options || {} - const qs = objectToQuery({ arg: peerId, count: options.count }) + const qs = objectToQuery({ + arg: peerId, + count: options.count, + ...(options.qs || {}) + }) + const url = `${apiUrl}${apiPath}/ping${qs}` const res = await ok(fetch(url, { signal: options.signal, diff --git a/src/pubsub/ls.js b/src/pubsub/ls.js index c65c17e..1bcec4f 100644 --- a/src/pubsub/ls.js +++ b/src/pubsub/ls.js @@ -2,12 +2,14 @@ const configure = require('../lib/configure') const { ok } = require('../lib/fetch') +const { objectToQuery } = require('../lib/querystring') module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => { return async (options) => { options = options || {} - const url = `${apiUrl}${apiPath}/pubsub/ls` + const qs = objectToQuery(options.qs) + const url = `${apiUrl}${apiPath}/pubsub/ls${qs}` const res = await ok(fetch(url, { signal: options.signal, headers: options.headers || headers diff --git a/src/pubsub/peers.js b/src/pubsub/peers.js index f738677..5ef202a 100644 --- a/src/pubsub/peers.js +++ b/src/pubsub/peers.js @@ -13,7 +13,12 @@ module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => { options = options || {} - const url = `${apiUrl}${apiPath}/pubsub/peers${objectToQuery({ arg: topic })}` + const qs = objectToQuery({ + arg: topic, + ...(options.qs || {}) + }) + + const url = `${apiUrl}${apiPath}/pubsub/peers${qs}` const res = await ok(fetch(url, { signal: options.signal, headers: options.headers || headers diff --git a/src/pubsub/publish.js b/src/pubsub/publish.js index 702f432..78d6f2b 100644 --- a/src/pubsub/publish.js +++ b/src/pubsub/publish.js @@ -2,13 +2,17 @@ const { Buffer } = require('buffer') const configure = require('../lib/configure') +const { objectToQuery } = require('../lib/querystring') const { ok } = require('../lib/fetch') module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => { return async (topic, data, options) => { options = options || {} - const url = `${apiUrl}${apiPath}/pubsub/pub?arg=${encodeURIComponent(topic)}&arg=${encodeBuffer(Buffer.from(data))}` + let qs = objectToQuery(options.qs) + qs = qs ? `&${qs.slice(1)}` : qs + + const url = `${apiUrl}${apiPath}/pubsub/pub?arg=${encodeURIComponent(topic)}&arg=${encodeBuffer(Buffer.from(data))}${qs}` const res = await ok(fetch(url, { method: 'POST', signal: options.signal, diff --git a/src/pubsub/subscribe.js b/src/pubsub/subscribe.js index 2dbe545..5023292 100644 --- a/src/pubsub/subscribe.js +++ b/src/pubsub/subscribe.js @@ -14,7 +14,12 @@ module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => { options = options || {} options.signal = subsTracker.subscribe(topic, handler, options.signal) - const qs = objectToQuery({ arg: topic, discover: options.discover }) + const qs = objectToQuery({ + arg: topic, + discover: options.discover, + ...(options.qs || {}) + }) + const url = `${apiUrl}${apiPath}/pubsub/sub${qs}` let res diff --git a/src/swarm/connect.js b/src/swarm/connect.js index 82cab96..401a253 100644 --- a/src/swarm/connect.js +++ b/src/swarm/connect.js @@ -9,7 +9,11 @@ module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => { addrs = Array.isArray(addrs) ? addrs : [addrs] options = options || {} - const qs = objectToQuery({ arg: addrs.map(a => a.toString()) }) + const qs = objectToQuery({ + arg: addrs.map(a => a.toString()), + ...(options.qs || {}) + }) + const url = `${apiUrl}${apiPath}/swarm/connect${qs}` const res = await ok(fetch(url, { signal: options.signal, diff --git a/src/swarm/peers.js b/src/swarm/peers.js index e9b706b..5db68c0 100644 --- a/src/swarm/peers.js +++ b/src/swarm/peers.js @@ -12,7 +12,8 @@ module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => { const qs = objectToQuery({ verbose: options.verbose, streams: options.streams, - latency: options.latency + latency: options.latency, + ...(options.qs || {}) }) const url = `${apiUrl}${apiPath}/swarm/peers${qs}` diff --git a/src/version.js b/src/version.js index 934ac60..782c4e4 100644 --- a/src/version.js +++ b/src/version.js @@ -2,12 +2,14 @@ const configure = require('./lib/configure') const { ok } = require('./lib/fetch') +const { objectToQuery } = require('./lib/querystring') const toCamel = require('./lib/to-camel') module.exports = configure(({ fetch, apiUrl, apiPath, headers }) => { return async options => { options = options || {} - const url = `${apiUrl}${apiPath}/version` + const qs = objectToQuery(options.qs) + const url = `${apiUrl}${apiPath}/version${qs}` const res = await ok(fetch(url, { signal: options.signal, headers: options.headers || headers diff --git a/test/interface.spec.js b/test/interface.spec.js index 0dde420..63f72b1 100644 --- a/test/interface.spec.js +++ b/test/interface.spec.js @@ -264,7 +264,52 @@ describe('interface-ipfs-core tests', () => { ] }) - tests.filesMFS(defaultCommon, { skip: { reason: 'LITE: not implemented yet' } }) + tests.filesMFS(defaultCommon, { + skip: [{ + name: 'write', + reason: 'LITE: depends on stat, stat not implemented yet' + }, + { + name: 'stat', + reason: 'LITE: not implemented yet' + }, + { + name: 'mv', + reason: 'LITE: not implemented yet' + }, + { + name: 'rm', + reason: 'LITE: not implemented yet' + }, + { + name: 'read', + reason: 'LITE: not implemented yet' + }, + { + name: 'readReadableStream', + reason: 'LITE: Node.js streams not supported' + }, + { + name: 'readPullStream', + reason: 'LITE: not implemented yet' + }, + { + name: 'ls', + reason: 'LITE: not implemented yet' + }, + { + name: 'lsReadableStream', + reason: 'LITE: Node.js streams not supported' + }, + { + name: 'lsPullStream', + reason: 'LITE: not implemented yet' + }, + { + name: 'flush', + reason: 'LITE: not implemented yet' + }] + }) tests.key(defaultCommon, { // skip: [ diff --git a/test/utils/core-adapter/files/args-to-srcs.js b/test/utils/core-adapter/files/args-to-srcs.js new file mode 100644 index 0000000..f720e06 --- /dev/null +++ b/test/utils/core-adapter/files/args-to-srcs.js @@ -0,0 +1,20 @@ +'use strict' + +module.exports = args => { + let options = {} + let sources = [] + + if (!Array.isArray(args[args.length - 1]) && typeof args[args.length - 1] === 'object') { + options = args.pop() + } + + if (args.length === 1 && Array.isArray(args[0])) { + // support ipfs.files.cp([src, dest], opts) + sources = args[0] + } else { + // support ipfs.files.cp(src, dest, opts) and ipfs.files.cp(src1, src2, dest, opts) + sources = args + } + + return { sources, options } +} diff --git a/test/utils/core-adapter/files/cp.js b/test/utils/core-adapter/files/cp.js new file mode 100644 index 0000000..6ddd7be --- /dev/null +++ b/test/utils/core-adapter/files/cp.js @@ -0,0 +1,8 @@ +'use strict' + +const toSources = require('./args-to-srcs') + +module.exports = ipfsLite => (...args) => { + const { sources, options } = toSources(args) + return ipfsLite.files.cp(...sources, options) +} diff --git a/test/utils/core-adapter/files/mkdir.js b/test/utils/core-adapter/files/mkdir.js new file mode 100644 index 0000000..2b15197 --- /dev/null +++ b/test/utils/core-adapter/files/mkdir.js @@ -0,0 +1,12 @@ +'use strict' + +module.exports = ipfsLite => (path, options) => { + options = options || {} + + if (options.p != null) { + options.parents = options.p + delete options.p + } + + return ipfsLite.files.mkdir(path, options) +} diff --git a/test/utils/core-adapter/index.js b/test/utils/core-adapter/index.js index bb8e67c..3589b9c 100644 --- a/test/utils/core-adapter/index.js +++ b/test/utils/core-adapter/index.js @@ -17,6 +17,11 @@ module.exports = ipfsLite => { put: callbackify(require('./block/put')(ipfsLite)), stat: ipfsLite.block.stat }, + files: { + cp: callbackify(require('./files/cp')(ipfsLite)), + mkdir: callbackify(require('./files/mkdir')(ipfsLite)), + write: ipfsLite.files.write + }, ls: callbackify(require('./ls')(ipfsLite)), lsPullStream: require('./ls-pull-stream')(ipfsLite), pubsub: {