diff --git a/package.json b/package.json index 85697a8..d9c0ac3 100644 --- a/package.json +++ b/package.json @@ -27,18 +27,21 @@ "license": "MIT", "dependencies": { "buffer": "^5.2.1", + "err-code": "^2.0.0", "is-buffer": "^2.0.3", "is-electron": "^2.2.0", "is-pull-stream": "0.0.0", "is-stream": "^2.0.0", "kind-of": "^6.0.2", + "pull-stream-to-async-iterator": "^1.0.2", "readable-stream": "^3.4.0" }, "devDependencies": { "aegir": "^20.0.0", + "async-iterator-all": "^1.0.0", "chai": "^4.2.0", "dirty-chai": "^2.0.1", - "electron": "^5.0.7", + "electron": "^6.0.6", "electron-mocha": "^8.0.3", "pull-stream": "^3.6.13" }, diff --git a/src/files/normalise-input.js b/src/files/normalise-input.js new file mode 100644 index 0000000..88b310d --- /dev/null +++ b/src/files/normalise-input.js @@ -0,0 +1,331 @@ +'use strict' + +const errCode = require('err-code') +const { Buffer } = require('buffer') +const pullStreamToIterable = require('pull-stream-to-async-iterator') +const { isSource } = require('is-pull-stream') +const globalThis = require('../globalthis') + +/* + * Transform one of: + * + * ``` + * Bytes (Buffer|ArrayBuffer|TypedArray) [single file] + * Bloby (Blob|File) [single file] + * String [single file] + * { path, content: Bytes } [single file] + * { path, content: Bloby } [single file] + * { path, content: String } [single file] + * { path, content: Iterable } [single file] + * { path, content: Iterable } [single file] + * { path, content: AsyncIterable } [single file] + * { path, content: PullStream } [single file] + * Iterable [single file] + * Iterable [single file] + * Iterable [multiple files] + * Iterable [multiple files] + * Iterable<{ path, content: Bytes }> [multiple files] + * Iterable<{ path, content: Bloby }> [multiple files] + * Iterable<{ path, content: String }> [multiple files] + * Iterable<{ path, content: Iterable }> [multiple files] + * Iterable<{ path, content: Iterable }> [multiple files] + * Iterable<{ path, content: AsyncIterable }> [multiple files] + * Iterable<{ path, content: PullStream }> [multiple files] + * AsyncIterable [single file] + * AsyncIterable [multiple files] + * AsyncIterable [multiple files] + * AsyncIterable<{ path, content: Bytes }> [multiple files] + * AsyncIterable<{ path, content: Bloby }> [multiple files] + * AsyncIterable<{ path, content: String }> [multiple files] + * AsyncIterable<{ path, content: Iterable }> [multiple files] + * AsyncIterable<{ path, content: Iterable }> [multiple files] + * AsyncIterable<{ path, content: AsyncIterable }> [multiple files] + * AsyncIterable<{ path, content: PullStream }> [multiple files] + * PullStream [single file] + * PullStream [multiple files] + * PullStream [multiple files] + * PullStream<{ path, content: Bytes }> [multiple files] + * PullStream<{ path, content: Bloby }> [multiple files] + * PullStream<{ path, content: String }> [multiple files] + * PullStream<{ path, content: Iterable }> [multiple files] + * PullStream<{ path, content: Iterable }> [multiple files] + * PullStream<{ path, content: AsyncIterable }> [multiple files] + * PullStream<{ path, content: PullStream }> [multiple files] + * ``` + * Into: + * + * ``` + * AsyncIterable<{ path, content: AsyncIterable }> + * ``` + * + * @param input Object + * @return AsyncInterable<{ path, content: AsyncIterable }> + */ +module.exports = function normaliseInput (input) { + // must give us something + if (input === null || input === undefined) { + throw errCode(new Error(`Unexpected input: ${input}`, 'ERR_UNEXPECTED_INPUT')) + } + + // String + if (typeof input === 'string' || input instanceof String) { + return (async function * () { // eslint-disable-line require-await + yield toFileObject(input) + })() + } + + // Buffer|ArrayBuffer|TypedArray + // Blob|File + if (isBytes(input) || isBloby(input)) { + return (async function * () { // eslint-disable-line require-await + yield toFileObject(input) + })() + } + + // Iterable + if (input[Symbol.iterator]) { + return (async function * () { // eslint-disable-line require-await + const iterator = input[Symbol.iterator]() + const first = iterator.next() + if (first.done) return iterator + + // Iterable + // Iterable + if (Number.isInteger(first.value) || isBytes(first.value)) { + yield toFileObject((function * () { + yield first.value + yield * iterator + })()) + return + } + + // Iterable + // Iterable + // Iterable<{ path, content }> + if (isFileObject(first.value) || isBloby(first.value) || typeof first.value === 'string') { + yield toFileObject(first.value) + for (const obj of iterator) { + yield toFileObject(obj) + } + return + } + + throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') + })() + } + + // AsyncIterable + if (input[Symbol.asyncIterator]) { + return (async function * () { + const iterator = input[Symbol.asyncIterator]() + const first = await iterator.next() + if (first.done) return iterator + + // AsyncIterable + if (isBytes(first.value)) { + yield toFileObject((async function * () { // eslint-disable-line require-await + yield first.value + yield * iterator + })()) + return + } + + // AsyncIterable + // AsyncIterable + // AsyncIterable<{ path, content }> + if (isFileObject(first.value) || isBloby(first.value) || typeof first.value === 'string') { + yield toFileObject(first.value) + for await (const obj of iterator) { + yield toFileObject(obj) + } + return + } + + throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') + })() + } + + // { path, content: ? } + // Note: Detected _after_ AsyncIterable because Node.js streams have a + // `path` property that passes this check. + if (isFileObject(input)) { + return (async function * () { // eslint-disable-line require-await + yield toFileObject(input) + })() + } + + // PullStream + if (isSource(input)) { + return (async function * () { + const iterator = pullStreamToIterable(input)[Symbol.asyncIterator]() + const first = await iterator.next() + if (first.done) return iterator + + // PullStream + if (isBytes(first.value)) { + yield toFileObject((async function * () { // eslint-disable-line require-await + yield first.value + yield * iterator + })()) + return + } + + // PullStream + // PullStream + // PullStream<{ path, content }> + if (isFileObject(first.value) || isBloby(first.value) || typeof first.value === 'string') { + yield toFileObject(first.value) + for await (const obj of iterator) { + yield toFileObject(obj) + } + return + } + + throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') + })() + } + + throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') +} + +function toFileObject (input) { + const obj = { path: input.path || '' } + + if (input.content) { + obj.content = toAsyncIterable(input.content) + } else if (!input.path) { // Not already a file object with path or content prop + obj.content = toAsyncIterable(input) + } + + return obj +} + +function toAsyncIterable (input) { + // Bytes | String + if (isBytes(input) || typeof input === 'string') { + return (async function * () { // eslint-disable-line require-await + yield toBuffer(input) + })() + } + + // Bloby + if (isBloby(input)) { + return blobToAsyncGenerator(input) + } + + // Iterator + if (input[Symbol.iterator]) { + return (async function * () { // eslint-disable-line require-await + const iterator = input[Symbol.iterator]() + const first = iterator.next() + if (first.done) return iterator + + // Iterable + if (Number.isInteger(first.value)) { + yield toBuffer(Array.from((function * () { + yield first.value + yield * iterator + })())) + return + } + + // Iterable + if (isBytes(first.value)) { + yield toBuffer(first.value) + for (const chunk of iterator) { + yield toBuffer(chunk) + } + return + } + + throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT') + })() + } + + // AsyncIterable + if (input[Symbol.asyncIterator]) { + return (async function * () { + for await (const chunk of input) { + yield toBuffer(chunk) + } + })() + } + + // PullStream + if (isSource(input)) { + return pullStreamToIterable(input) + } + + throw errCode(new Error(`Unexpected input: ${input}`, 'ERR_UNEXPECTED_INPUT')) +} + +function toBuffer (chunk) { + return isBytes(chunk) ? chunk : Buffer.from(chunk) +} + +function isBytes (obj) { + return Buffer.isBuffer(obj) || ArrayBuffer.isView(obj) || obj instanceof ArrayBuffer +} + +function isBloby (obj) { + return typeof globalThis.Blob !== 'undefined' && obj instanceof globalThis.Blob +} + +// An object with a path or content property +function isFileObject (obj) { + return typeof obj === 'object' && (obj.path || obj.content) +} + +function blobToAsyncGenerator (blob) { + if (typeof blob.stream === 'function') { + // firefox < 69 does not support blob.stream() + return streamBlob(blob) + } + + return readBlob(blob) +} + +async function * streamBlob (blob) { + const reader = blob.stream().getReader() + + while (true) { + const result = await reader.read() + + if (result.done) { + return + } + + yield result.value + } +} + +async function * readBlob (blob, options) { + options = options || {} + + const reader = new globalThis.FileReader() + const chunkSize = options.chunkSize || 1024 * 1024 + let offset = options.offset || 0 + + const getNextChunk = () => new Promise((resolve, reject) => { + reader.onloadend = e => { + const data = e.target.result + resolve(data.byteLength === 0 ? null : data) + } + reader.onerror = reject + + const end = offset + chunkSize + const slice = blob.slice(offset, end) + reader.readAsArrayBuffer(slice) + offset = end + }) + + while (true) { + const data = await getNextChunk() + + if (data == null) { + return + } + + yield Buffer.from(data) + } +} diff --git a/test/files/normalise-input.spec.js b/test/files/normalise-input.spec.js new file mode 100644 index 0000000..911364d --- /dev/null +++ b/test/files/normalise-input.spec.js @@ -0,0 +1,155 @@ +'use strict' + +/* eslint-env mocha */ +const chai = require('chai') +const dirtyChai = require('dirty-chai') +const normalise = require('../../src/files/normalise-input') +const { supportsFileReader } = require('../../src/supports') +const { Buffer } = require('buffer') +const all = require('async-iterator-all') +const pull = require('pull-stream') +const globalThis = require('../../src/globalthis') + +chai.use(dirtyChai) +const expect = chai.expect + +const STRING = 'hello world' +const BUFFER = Buffer.from(STRING) +const ARRAY = Array.from(BUFFER) +const TYPEDARRAY = Uint8Array.from(ARRAY) +let BLOB + +if (supportsFileReader) { + BLOB = new globalThis.Blob([ + STRING + ]) +} + +async function verifyNormalisation (input) { + expect(input.length).to.equal(1) + + if (!input[0].content[Symbol.asyncIterator] && !input[0].content[Symbol.iterator]) { + chai.assert.fail(`Content should have been an iterable or an async iterable`) + } + + expect(await all(input[0].content)).to.deep.equal([BUFFER]) + expect(input[0].path).to.equal('') +} + +async function testContent (input) { + const result = await all(normalise(input)) + + await verifyNormalisation(result) +} + +function iterableOf (thing) { + return [thing] +} + +function asyncIterableOf (thing) { + return (async function * () { // eslint-disable-line require-await + yield thing + }()) +} + +function pullStreamOf (thing) { + return pull.values([thing]) +} + +describe('normalise-input', function () { + function testInputType (content, name, isBytes) { + it(name, async function () { + await testContent(content) + }) + + if (isBytes) { + it(`Iterable<${name}>`, async function () { + await testContent(iterableOf(content)) + }) + + it(`AsyncIterable<${name}>`, async function () { + await testContent(asyncIterableOf(content)) + }) + + it(`PullStream<${name}>`, async function () { + await testContent(pullStreamOf(content)) + }) + } + + it(`{ path: '', content: ${name} }`, async function () { + await testContent({ path: '', content }) + }) + + if (isBytes) { + it(`{ path: '', content: Iterable<${name}> }`, async function () { + await testContent({ path: '', content: iterableOf(content) }) + }) + + it(`{ path: '', content: AsyncIterable<${name}> }`, async function () { + await testContent({ path: '', content: asyncIterableOf(content) }) + }) + + it(`{ path: '', content: PullStream<${name}> }`, async function () { + await testContent({ path: '', content: pullStreamOf(content) }) + }) + } + + it(`Iterable<{ path: '', content: ${name} }`, async function () { + await testContent(iterableOf({ path: '', content })) + }) + + it(`AsyncIterable<{ path: '', content: ${name} }`, async function () { + await testContent(asyncIterableOf({ path: '', content })) + }) + + it(`PullStream<{ path: '', content: ${name} }`, async function () { + await testContent(pullStreamOf({ path: '', content })) + }) + + if (isBytes) { + it(`Iterable<{ path: '', content: Iterable<${name}> }>`, async function () { + await testContent(iterableOf({ path: '', content: iterableOf(content) })) + }) + + it(`Iterable<{ path: '', content: AsyncIterable<${name}> }>`, async function () { + await testContent(iterableOf({ path: '', content: asyncIterableOf(content) })) + }) + + it(`AsyncIterable<{ path: '', content: Iterable<${name}> }>`, async function () { + await testContent(asyncIterableOf({ path: '', content: iterableOf(content) })) + }) + + it(`AsyncIterable<{ path: '', content: AsyncIterable<${name}> }>`, async function () { + await testContent(asyncIterableOf({ path: '', content: asyncIterableOf(content) })) + }) + + it(`PullStream<{ path: '', content: PullStream<${name}> }>`, async function () { + await testContent(pullStreamOf({ path: '', content: pullStreamOf(content) })) + }) + } + } + + describe('String', () => { + testInputType(STRING, 'String', false) + }) + + describe('Buffer', () => { + testInputType(BUFFER, 'Buffer', true) + }) + + describe('Blob', () => { + if (!supportsFileReader) { + return + } + + testInputType(BLOB, 'Blob', false) + }) + + describe('Iterable', () => { + testInputType(ARRAY, 'Iterable', false) + }) + + describe('TypedArray', () => { + testInputType(TYPEDARRAY, 'TypedArray', true) + }) +})