Skip to content

Commit e5b2509

Browse files
Alan Shawachingbrain
Alan Shaw
authored and
achingbrain
committed
feat: support pull streams (#8)
* feat: support pull streams This PR updates the `normaliseInput` function to accept pull streams. I've also made the following changes: 1. Update the docs for supported inputs * `Buffer|ArrayBuffer|TypedArray` is aliased as `Bytes` * `Blob|File` is aliased as `Bloby` * Added info for what a input "means" i.e. causes single/multiple files to be added 1. Peek the first item of an (async) iterator properly 1. Move file object check below `input[Symbol.asyncIterator]` check because Node.js streams have a path property that will false positive the `isFileObject` check 1. Fix `toFileObject` to allow objects with no `content` property 1. Simplify `toBuffer` to remove checks that `Buffer.from` already does License: MIT Signed-off-by: Alan Shaw <[email protected]> * fix: tests License: MIT Signed-off-by: Alan Shaw <[email protected]>
1 parent e51ff72 commit e5b2509

File tree

3 files changed

+213
-100
lines changed

3 files changed

+213
-100
lines changed

‎package.json

+1
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
"is-pull-stream": "0.0.0",
3434
"is-stream": "^2.0.0",
3535
"kind-of": "^6.0.2",
36+
"pull-stream-to-async-iterator": "^1.0.2",
3637
"readable-stream": "^3.4.0"
3738
},
3839
"devDependencies": {

‎src/files/normalise-input.js

+171-75
Original file line numberDiff line numberDiff line change
@@ -2,32 +2,53 @@
22

33
const errCode = require('err-code')
44
const { Buffer } = require('buffer')
5+
const pullStreamToIterable = require('pull-stream-to-async-iterator')
56

67
/*
78
* Transform one of:
89
*
910
* ```
10-
* Buffer|ArrayBuffer|TypedArray
11-
* Blob|File
12-
* { path, content: Blob }
13-
* { path, content: String }
14-
* { path, content: Iterable<Number> }
15-
* { path, content: Iterable<Buffer> }
16-
* { path, content: Iterable<Iterable<Number>> }
17-
* { path, content: AsyncIterable<Iterable<Number>> }
18-
* String
19-
* Iterable<Number>
20-
* Iterable<Buffer>
21-
* Iterable<Blob>
22-
* Iterable<{ path, content: Buffer }>
23-
* Iterable<{ path, content: Blob }>
24-
* Iterable<{ path, content: Iterable<Number> }>
25-
* Iterable<{ path, content: AsyncIterable<Buffer> }>
26-
* AsyncIterable<Buffer>
27-
* AsyncIterable<{ path, content: Buffer }>
28-
* AsyncIterable<{ path, content: Blob }>
29-
* AsyncIterable<{ path, content: Iterable<Buffer> }>
30-
* AsyncIterable<{ path, content: AsyncIterable<Buffer> }>
11+
* Bytes (Buffer|ArrayBuffer|TypedArray) [single file]
12+
* Bloby (Blob|File) [single file]
13+
* String [single file]
14+
* { path, content: Bytes } [single file]
15+
* { path, content: Bloby } [single file]
16+
* { path, content: String } [single file]
17+
* { path, content: Iterable<Number> } [single file]
18+
* { path, content: Iterable<Bytes> } [single file]
19+
* { path, content: AsyncIterable<Bytes> } [single file]
20+
* { path, content: PullStream<Bytes> } [single file]
21+
* Iterable<Number> [single file]
22+
* Iterable<Bytes> [single file]
23+
* Iterable<Bloby> [multiple files]
24+
* Iterable<String> [multiple files]
25+
* Iterable<{ path, content: Bytes }> [multiple files]
26+
* Iterable<{ path, content: Bloby }> [multiple files]
27+
* Iterable<{ path, content: String }> [multiple files]
28+
* Iterable<{ path, content: Iterable<Number> }> [multiple files]
29+
* Iterable<{ path, content: Iterable<Bytes> }> [multiple files]
30+
* Iterable<{ path, content: AsyncIterable<Bytes> }> [multiple files]
31+
* Iterable<{ path, content: PullStream<Bytes> }> [multiple files]
32+
* AsyncIterable<Bytes> [single file]
33+
* AsyncIterable<Bloby> [multiple files]
34+
* AsyncIterable<String> [multiple files]
35+
* AsyncIterable<{ path, content: Bytes }> [multiple files]
36+
* AsyncIterable<{ path, content: Bloby }> [multiple files]
37+
* AsyncIterable<{ path, content: String }> [multiple files]
38+
* AsyncIterable<{ path, content: Iterable<Number> }> [multiple files]
39+
* AsyncIterable<{ path, content: Iterable<Bytes> }> [multiple files]
40+
* AsyncIterable<{ path, content: AsyncIterable<Bytes> }> [multiple files]
41+
* AsyncIterable<{ path, content: PullStream<Bytes> }> [multiple files]
42+
* PullStream<Bytes> [single file]
43+
* PullStream<Bloby> [multiple files]
44+
* PullStream<String> [multiple files]
45+
* PullStream<{ path, content: Bytes }> [multiple files]
46+
* PullStream<{ path, content: Bloby }> [multiple files]
47+
* PullStream<{ path, content: String }> [multiple files]
48+
* PullStream<{ path, content: Iterable<Number> }> [multiple files]
49+
* PullStream<{ path, content: Iterable<Bytes> }> [multiple files]
50+
* PullStream<{ path, content: AsyncIterable<Bytes> }> [multiple files]
51+
* PullStream<{ path, content: PullStream<Bytes> }> [multiple files]
3152
* ```
3253
* Into:
3354
*
@@ -44,13 +65,6 @@ module.exports = function normaliseInput (input) {
4465
throw errCode(new Error(`Unexpected input: ${input}`, 'ERR_UNEXPECTED_INPUT'))
4566
}
4667

47-
// { path, content: ? }
48-
if (isFileObject(input)) {
49-
return (async function * () { // eslint-disable-line require-await
50-
yield toFileObject(input)
51-
})()
52-
}
53-
5468
// String
5569
if (typeof input === 'string' || input instanceof String) {
5670
return (async function * () { // eslint-disable-line require-await
@@ -68,76 +82,165 @@ module.exports = function normaliseInput (input) {
6882

6983
// Iterable<?>
7084
if (input[Symbol.iterator]) {
71-
// Iterable<Number>
72-
if (!isNaN(input[0])) {
73-
return (async function * () { // eslint-disable-line require-await
74-
yield toFileObject([input])
75-
})()
76-
}
77-
78-
// Iterable<Buffer>
79-
// Iterable<Blob>
8085
return (async function * () { // eslint-disable-line require-await
81-
for (const chunk of input) {
82-
yield toFileObject(chunk)
86+
const iterator = input[Symbol.iterator]()
87+
const first = iterator.next()
88+
if (first.done) return iterator
89+
90+
// Iterable<Number>
91+
// Iterable<Bytes>
92+
if (Number.isInteger(first.value) || isBytes(first.value)) {
93+
yield toFileObject((function * () {
94+
yield first.value
95+
yield * iterator
96+
})())
97+
return
8398
}
99+
100+
// Iterable<Bloby>
101+
// Iterable<String>
102+
// Iterable<{ path, content }>
103+
if (isFileObject(first.value) || isBloby(first.value) || typeof first.value === 'string') {
104+
yield toFileObject(first.value)
105+
for (const obj of iterator) {
106+
yield toFileObject(obj)
107+
}
108+
return
109+
}
110+
111+
throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT')
84112
})()
85113
}
86114

87115
// AsyncIterable<?>
88116
if (input[Symbol.asyncIterator]) {
117+
return (async function * () {
118+
const iterator = input[Symbol.asyncIterator]()
119+
const first = await iterator.next()
120+
if (first.done) return iterator
121+
122+
// AsyncIterable<Bytes>
123+
if (isBytes(first.value)) {
124+
yield toFileObject((async function * () { // eslint-disable-line require-await
125+
yield first.value
126+
yield * iterator
127+
})())
128+
return
129+
}
130+
131+
// AsyncIterable<Bloby>
132+
// AsyncIterable<String>
133+
// AsyncIterable<{ path, content }>
134+
if (isFileObject(first.value) || isBloby(first.value) || typeof first.value === 'string') {
135+
yield toFileObject(first.value)
136+
for await (const obj of iterator) {
137+
yield toFileObject(obj)
138+
}
139+
return
140+
}
141+
142+
throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT')
143+
})()
144+
}
145+
146+
// { path, content: ? }
147+
// Note: Detected _after_ AsyncIterable<?> because Node.js streams have a
148+
// `path` property that passes this check.
149+
if (isFileObject(input)) {
89150
return (async function * () { // eslint-disable-line require-await
90-
for await (const chunk of input) {
91-
yield toFileObject(chunk)
151+
yield toFileObject(input)
152+
})()
153+
}
154+
155+
// PullStream<?>
156+
if (typeof input === 'function') {
157+
return (async function * () {
158+
const iterator = pullStreamToIterable(input)[Symbol.asyncIterator]()
159+
const first = await iterator.next()
160+
if (first.done) return iterator
161+
162+
// PullStream<Bytes>
163+
if (isBytes(first.value)) {
164+
yield toFileObject((async function * () { // eslint-disable-line require-await
165+
yield first.value
166+
yield * iterator
167+
})())
168+
return
169+
}
170+
171+
// PullStream<Bloby>
172+
// PullStream<String>
173+
// PullStream<{ path, content }>
174+
if (isFileObject(first.value) || isBloby(first.value) || typeof first.value === 'string') {
175+
yield toFileObject(first.value)
176+
for await (const obj of iterator) {
177+
yield toFileObject(obj)
178+
}
179+
return
92180
}
181+
182+
throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT')
93183
})()
94184
}
95185

96186
throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT')
97187
}
98188

99189
function toFileObject (input) {
100-
return {
101-
path: input.path || '',
102-
content: toAsyncIterable(input.content || input)
190+
const obj = { path: input.path || '' }
191+
192+
if (input.content) {
193+
obj.content = toAsyncIterable(input.content)
194+
} else if (!input.path) { // Not already a file object with path or content prop
195+
obj.content = toAsyncIterable(input)
103196
}
197+
198+
return obj
104199
}
105200

106201
function toAsyncIterable (input) {
107-
// Buffer|ArrayBuffer|TypedArray|array of bytes
108-
if (isBytes(input)) {
109-
return (async function * () { // eslint-disable-line require-await
110-
yield toBuffer(input)
111-
})()
112-
}
113-
114-
if (typeof input === 'string' || input instanceof String) {
202+
// Bytes | String
203+
if (isBytes(input) || typeof input === 'string') {
115204
return (async function * () { // eslint-disable-line require-await
116205
yield toBuffer(input)
117206
})()
118207
}
119208

120-
// Blob|File
209+
// Bloby
121210
if (isBloby(input)) {
122211
return blobToAsyncGenerator(input)
123212
}
124213

125214
// Iterator<?>
126215
if (input[Symbol.iterator]) {
127-
if (!isNaN(input[0])) {
128-
return (async function * () { // eslint-disable-line require-await
129-
yield toBuffer(input)
130-
})()
131-
}
132-
133216
return (async function * () { // eslint-disable-line require-await
134-
for (const chunk of input) {
135-
yield toBuffer(chunk)
217+
const iterator = input[Symbol.iterator]()
218+
const first = iterator.next()
219+
if (first.done) return iterator
220+
221+
// Iterable<Number>
222+
if (Number.isInteger(first.value)) {
223+
yield toBuffer(Array.from((function * () {
224+
yield first.value
225+
yield * iterator
226+
})()))
227+
return
228+
}
229+
230+
// Iterable<Bytes>
231+
if (isBytes(first.value)) {
232+
yield toBuffer(first.value)
233+
for (const chunk of iterator) {
234+
yield toBuffer(chunk)
235+
}
236+
return
136237
}
238+
239+
throw errCode(new Error('Unexpected input: ' + typeof input), 'ERR_UNEXPECTED_INPUT')
137240
})()
138241
}
139242

140-
// AsyncIterable<?>
243+
// AsyncIterable<Bytes>
141244
if (input[Symbol.asyncIterator]) {
142245
return (async function * () {
143246
for await (const chunk of input) {
@@ -146,23 +249,16 @@ function toAsyncIterable (input) {
146249
})()
147250
}
148251

252+
// PullStream<Bytes>
253+
if (typeof input === 'function') {
254+
return pullStreamToIterable(input)
255+
}
256+
149257
throw errCode(new Error(`Unexpected input: ${input}`, 'ERR_UNEXPECTED_INPUT'))
150258
}
151259

152260
function toBuffer (chunk) {
153-
if (isBytes(chunk)) {
154-
return chunk
155-
}
156-
157-
if (typeof chunk === 'string' || chunk instanceof String) {
158-
return Buffer.from(chunk)
159-
}
160-
161-
if (Array.isArray(chunk)) {
162-
return Buffer.from(chunk)
163-
}
164-
165-
throw new Error('Unexpected input: ' + typeof chunk)
261+
return isBytes(chunk) ? chunk : Buffer.from(chunk)
166262
}
167263

168264
function isBytes (obj) {

0 commit comments

Comments
 (0)