fix: support uint8arraylists (#149)

To make parsing streams more flexible, support uint8arraylists

Author: achingbrain

packages/it-ndjson/package.json

@@ -54,5 +54,8 @@
     "aegir": "^45.0.8",
     "buffer": "^6.0.3",
     "it-all": "^3.0.0"
+  },
+  "dependencies": {
+    "uint8arraylist": "^2.4.8"
   }
 }

packages/it-ndjson/src/parse.ts

@@ -1,4 +1,6 @@
+import { isUint8ArrayList } from 'uint8arraylist'
 import { InvalidMessageLengthError } from './errors.js'
+import type { Uint8ArrayList } from 'uint8arraylist'
 
 export interface ParseOptions {
   /**
@@ -7,7 +9,7 @@ export interface ParseOptions {
   maxMessageLength?: number
 }
 
-export default async function * parse <T> (source: AsyncIterable<Uint8Array | string> | Iterable<Uint8Array | string>, opts: ParseOptions = {}): AsyncGenerator<T, void, undefined> {
+export default async function * parse <T> (source: AsyncIterable<Uint8Array | Uint8ArrayList | string> | Iterable<Uint8Array | Uint8ArrayList | string>, opts: ParseOptions = {}): AsyncGenerator<T, void, undefined> {
   const matcher = /\r?\n/
   const decoder = new TextDecoder('utf8')
   let buffer = ''
@@ -17,6 +19,10 @@ export default async function * parse <T> (source: AsyncIterable<Uint8Array | st
       chunk = new TextEncoder().encode(chunk)
     }
 
+    if (isUint8ArrayList(chunk)) {
+      chunk = chunk.subarray()
+    }
+
     buffer += decoder.decode(chunk, { stream: true })
 
     if (buffer.length > (opts?.maxMessageLength ?? buffer.length)) {

packages/it-ndjson/test/index.spec.ts

@@ -1,6 +1,7 @@
 import { Buffer } from 'buffer'
 import { expect } from 'aegir/chai'
 import all from 'it-all'
+import { Uint8ArrayList } from 'uint8arraylist'
 import * as ndjson from '../src/index.js'
 
 async function * toAsyncIterator <T> (array: T[]): AsyncIterable<T> {
@@ -74,6 +75,20 @@ describe('it-ndjson', () => {
     expect(results).to.deep.equal([{ id: 1 }, { id: 2 }, { id: 3 }])
   })
 
+  it('should split from Uint8ArrayLists', async () => {
+    const source = toAsyncIterator([new Uint8ArrayList(toUint8Array('{ "id": 1 }\n{ "i')), new Uint8ArrayList(toUint8Array('d": 2 }')), new Uint8ArrayList(toUint8Array('\n{"id":3}'))])
+    const results = await all(ndjson.parse(source))
+
+    expect(results).to.deep.equal([{ id: 1 }, { id: 2 }, { id: 3 }])
+  })
+
+  it('should split from Uint8ArrayLists with multiple chunks', async () => {
+    const source = toAsyncIterator([new Uint8ArrayList(toUint8Array('{ "id": 1 }\n{ "i'), toUint8Array('d": 2 }')), new Uint8ArrayList(toUint8Array('\n{"id":3}'))])
+    const results = await all(ndjson.parse(source))
+
+    expect(results).to.deep.equal([{ id: 1 }, { id: 2 }, { id: 3 }])
+  })
+
   it('should round trip', async () => {
     const input = '{"id":1}\n{"id":2}\n{"id":3}\n'
     const source = toAsyncIterator([input])