feat: add MFS read and write

License: MIT
Signed-off-by: Alan Shaw <[email protected]>

Author: Alan Shaw

API.md

@@ -8,9 +8,11 @@
 * [`cat`](#cat)
 * [`get`](#get)
 * [`id`](#id)
+* [`read`](#read)
 * [`start`](#start)
 * [`stop`](#stop)
 * [`version`](#version)
+* [`write`](#write)
 
 ## Getting started
 
@@ -219,7 +221,7 @@ Get file contents.
 | Name | Type | Description |
 |------|------|-------------|
 | path | `String`\|`Buffer`\|[`CID`](https://www.npmjs.com/package/cids) | IPFS path or CID to cat data from |
-| options | `Object` | (optional) options |
+| options | `Object` | (optional) options TODO |
 
 #### Returns
 
@@ -329,6 +331,36 @@ console.log(info)
 */
 ```
 
+## read
+
+Read data from an MFS (Mutable File System) path.
+
+### `node.read(path, [options])`
+
+#### Parameters
+
+| Name | Type | Description |
+|------|------|-------------|
+| path | `String` | MFS path of the _file_ to read (not a directory) |
+| options | `Object` | (optional) options |
+| options.offset | `Number` | Byte offset to begin reading at, default: `0` |
+| options.length | `Number` | Number of bytes to read, default: `undefined` (read all bytes) |
+
+#### Returns
+
+| Type | Description |
+|------|-------------|
+| `Iterator<Buffer>` | An iterator that can be used to consume all the data |
+
+#### Example
+
+```js
+let data = Buffer.alloc(0)
+for (const chunk of node.read('/path/to/mfs/file')) {
+  data = Buffer.concat([data, chunk])
+}
+```
+
 ## start
 
 Start the IPFS node.
@@ -386,3 +418,48 @@ const info = await node.version()
 console.log(info)
 // { version: '0.32.2', repo: 7, commit: '' }
 ```
+
+## write
+
+Write data to an MFS (Mutable File System) file.
+
+### `node.write(path, input, [options])`
+
+#### Parameters
+
+| Name | Type | Description |
+|------|------|-------------|
+| path | `String` | MFS path of the _file_ to write to |
+| input | `Buffer`\|`String`\|`Iterable`\|`Iterator` | Input data |
+| options | `Object` | (optional) options |
+| options.offset | `Number` | Byte offset to begin writing at, default: `0` |
+| options.length | `Number` | Number of bytes to write, default: `undefined` (write all bytes) |
+| options.create | `Boolean` | Create file if not exists, default: `true` |
+| options.parents | `Boolean` | Create parent directories if not exists, default: `false` |
+| options.truncate | `Boolean` | Truncate the file after writing, default: `false` |
+| options.rawLeaves | `Boolean` | Do not wrap leaf nodes in a protobuf, default: `false` |
+| options.cidVersion | `Number` | CID version to use when creating the node(s), default: 1 |
+
+#### Returns
+
+| Type | Description |
+|------|-------------|
+| `Promise` | Resolved when the write has been completed |
+
+#### Example
+
+Write a buffer/string:
+
+```js
+await node.write('/hello-world.txt', Buffer.from('hello world!'))
+```
+
+```js
+await node.write('/example/hello-world.txt', 'hello world!', { parents: true })
+```
+
+Write a Node.js stream:
+
+```js
+await node.write('/example.js', fs.createReadStream(__filename), { create: true })
+```

RATIONALE.md

@@ -57,10 +57,20 @@ Are the same options but they have been renamed to be less cryptic.
 
 The default value for CID codec is changed from `dag-pb` to `raw`. This is to encourage users of the API to add correctly formed data.
 
-### CID version
+### CID version 1
 
 Changed from 0 to 1. This will soon be the default for all new content added to IPFS anyway.
 
 ### Iterator input
 
 This makes adding multiple blocks significantly easier and allows us to easily integrate with [`js-unixfsv2-draft`](https://github.com/mikeal/js-unixfsv2-draft).
+
+## `write`
+
+### Create by default
+
+Similar to Node.js `fs.createWriteStream` or `fs.writeFile`, 99.9% of the time we want to create the file if it doesn't exist, why make this difficult?
+
+### CID version 1
+
+Changed from 0 to 1. This will soon be the default for all new content added to IPFS anyway.

README.md

@@ -61,7 +61,7 @@ const node = await ipfsx(new IPFS)
 // Add something to IPFS
 const { cid } = await node.add('hello world').first()
 
-// Stream content from IPFs using async iterators
+// Stream content from IPFS using async iterators
 let data = Buffer.alloc(0)
 for await (const chunk of node.cat(cid)) {
   data = Buffer.concat([data, chunk])
@@ -80,9 +80,11 @@ for await (const chunk of node.cat(cid)) {
 * [`cat`](https://github.com/alanshaw/ipfsx/blob/master/API.md#cat)
 * [`get`](https://github.com/alanshaw/ipfsx/blob/master/API.md#get)
 * [`id`](https://github.com/alanshaw/ipfsx/blob/master/API.md#id)
+* [`read`](https://github.com/alanshaw/ipfsx/blob/master/API.md#read)
 * [`start`](https://github.com/alanshaw/ipfsx/blob/master/API.md#start)
 * [`stop`](https://github.com/alanshaw/ipfsx/blob/master/API.md#stop)
 * [`version`](https://github.com/alanshaw/ipfsx/blob/master/API.md#version)
+* [`write`](https://github.com/alanshaw/ipfsx/blob/master/API.md#write)
 * TODO: more to come in upcoming releases!
 
 ## Contribute

examples/add/nodejs-stream.js

@@ -0,0 +1,26 @@
+
+const Os = require('os')
+const Path = require('path')
+const Fs = require('fs')
+const Ipfs = require('ipfs')
+const ipfsx = require('../../')
+
+async function main () {
+  const node = await ipfsx(new Ipfs({ repo: Path.join(Os.tmpdir(), `${Date.now()}`) }))
+
+  const { cid } = await node.add(Fs.createReadStream(__filename)).first()
+
+  let buffer = Buffer.alloc(0)
+  for await (const chunk of node.cat(cid)) {
+    buffer = Buffer.concat([buffer, chunk])
+  }
+
+  if (!buffer.equals(await Fs.promises.readFile(__filename))) {
+    throw new Error('something went wrong, data written was not equal to data read')
+  }
+
+  console.log('success! data written was equal to data read')
+  await node.stop()
+}
+
+main()

examples/block/stat.js

@@ -3,12 +3,16 @@ const Path = require('path')
 const Ipfs = require('ipfs')
 const ipfsx = require('../../')
 
-;(async () => {
+async function main () {
   const node = await ipfsx(new Ipfs({ repo: Path.join(Os.tmpdir(), `${Date.now()}`) }))
   const data = Buffer.from('hello world')
   const block = await node.block.put(data).first()
   const stats = await node.block.stat(block.cid)
 
   console.log(block.cid.toBaseEncodedString(), stats)
   // zb2rhj7crUKTQYRGCRATFaQ6YFLTde2YzdqbbhAASkL9uRDXn { size: 11 }
-})()
+
+  await node.stop()
+}
+
+main()

examples/get/directory.js

@@ -6,7 +6,7 @@ const ipfsx = require('../../')
 
 const rootDir = Path.resolve(__dirname, '..', '..')
 
-;(async () => {
+async function main () {
   const node = await ipfsx(new Ipfs({ repo: Path.join(Os.tmpdir(), `${Date.now()}`) }))
 
   const { cid, path } = await node.add([
@@ -30,4 +30,8 @@ const rootDir = Path.resolve(__dirname, '..', '..')
 
     console.log('Data:', data)
   }
-})()
+
+  await node.stop()
+}
+
+main()

examples/id.js

@@ -3,8 +3,22 @@ const Path = require('path')
 const Ipfs = require('ipfs')
 const ipfsx = require('../')
 
-;(async () => {
+async function main () {
   const node = await ipfsx(new Ipfs({ repo: Path.join(Os.tmpdir(), `${Date.now()}`) }))
   const info = await node.id()
   console.log(info)
-})()
+  /*
+  { id: 'QmTq6pfziFuG2Twi5FnVQwua5mNtSXbEXuExjYy6XZd5Qt',
+    publicKey:
+     'CAASpgIwggEiMA0GCSqGSIb3DQEBAQUAA4IBDwAwggEKAoIBAQCEgPFgwTrphILHGQo0AErm/F3Awwl6Pxa9TTlkJShEu0swQq6ENTcT34YmiJ/DiC+a+HmbUeisi0B+ebzhW3MnZSXWb5mCQC4yz22MChedWYSieQq0/jFGRezhAZnZ7PYFw0MEEHLaXHDuJRF3NvrddcFxBWBlk0mu9FMv75c/QUN5aqnnZ7fiVpKqO78W5xKXcOyCdxCAb1ul0YZWUp4HjIcn4g+Fw6lQEh/kLjubTB3rKH41dcOPhv41nUKSQIqAaDeakFQfADEz86+mUaW2xTL931mtWWeBLI36KaLNG5uSnGMhqykxqy9/9LzVTEMa8r6U7kMcfRl2m+xCVWlRAgMBAAE=',
+    addresses:
+     [ '/ip4/127.0.0.1/tcp/4002/ipfs/QmTq6pfziFuG2Twi5FnVQwua5mNtSXbEXuExjYy6XZd5Qt',
+       '/ip4/127.0.0.1/tcp/4003/ws/ipfs/QmTq6pfziFuG2Twi5FnVQwua5mNtSXbEXuExjYy6XZd5Qt',
+       '/ip4/192.168.0.17/tcp/4002/ipfs/QmTq6pfziFuG2Twi5FnVQwua5mNtSXbEXuExjYy6XZd5Qt' ],
+    agentVersion: 'js-ipfs/0.32.2',
+    protocolVersion: '9000' }
+  */
+  await node.stop()
+}
+
+main()

examples/read+write/iterator.js

@@ -0,0 +1,50 @@
+
+const Os = require('os')
+const Path = require('path')
+const Ipfs = require('ipfs')
+const { randomBytes } = require('crypto')
+const { inspect } = require('util')
+const ipfsx = require('../../')
+
+async function main () {
+  const node = await ipfsx(new Ipfs({ repo: Path.join(Os.tmpdir(), `${Date.now()}`) }))
+
+  const path = `/example-read+write-iterator-${Date.now()}.txt`
+  const { data, iterator } = randomData()
+
+  console.log('iterator[Symbol.iterable]=', iterator)
+
+  console.log(`writing to ${path}`)
+  await node.write(path, iterator)
+
+  console.log(`reading from ${path}`)
+  let buffer = Buffer.alloc(0)
+  for await (const chunk of node.read(path)) {
+    console.log(`read ${inspect(chunk)}`)
+    buffer = Buffer.concat([buffer, chunk])
+  }
+
+  if (!buffer.equals(data)) {
+    throw new Error('something went wrong, data written was not equal to data read')
+  }
+
+  console.log('success! data written was equal to data read')
+  await node.stop()
+}
+
+function randomData () {
+  const data = Array.from(Array(100000).fill(0), () => randomBytes(16))
+  const it = function * () {
+    for (var i = 0; i < data.length; i++) {
+      console.log(`writing ${inspect(data[i])}`)
+      yield data[i]
+    }
+  }
+  console.log(it)
+  return {
+    data: Buffer.concat(data),
+    iterator: (it)()
+  }
+}
+
+main()

examples/read+write/nodejs-stream.js

@@ -0,0 +1,28 @@
+
+const Os = require('os')
+const Path = require('path')
+const Fs = require('fs')
+const Ipfs = require('ipfs')
+const ipfsx = require('../../')
+
+async function main () {
+  const node = await ipfsx(new Ipfs({ repo: Path.join(Os.tmpdir(), `${Date.now()}`) }))
+
+  const path = `/example-read+write-nodejs-stream-${Date.now()}.js`
+
+  await node.write(path, Fs.createReadStream(__filename))
+
+  let buffer = Buffer.alloc(0)
+  for await (const chunk of node.read(path)) {
+    buffer = Buffer.concat([buffer, chunk])
+  }
+
+  if (!buffer.equals(await Fs.promises.readFile(__filename))) {
+    throw new Error('something went wrong, data written was not equal to data read')
+  }
+
+  console.log('success! data written was equal to data read')
+  await node.stop()
+}
+
+main()

examples/read+write/string.js

@@ -0,0 +1,31 @@
+
+const Os = require('os')
+const Path = require('path')
+const Ipfs = require('ipfs')
+const ipfsx = require('../../')
+
+async function main () {
+  const node = await ipfsx(new Ipfs({ repo: Path.join(Os.tmpdir(), `${Date.now()}`) }))
+
+  const now = new Date()
+  const path = `/example-read+write-string-${now.getTime()}.txt`
+  const data = `The date is now ${now.toISOString()}`
+
+  console.log(`writing "${data}" to ${path}`)
+  await node.write(path, data)
+
+  console.log(`reading from ${path}`)
+  let buffer = Buffer.alloc(0)
+  for await (const chunk of node.read(path)) {
+    buffer = Buffer.concat([buffer, chunk])
+  }
+
+  if (buffer.toString() !== data) {
+    throw new Error('something went wrong, data written was not equal to data read')
+  }
+
+  console.log('success! data written was equal to data read')
+  await node.stop()
+}
+
+main()

examples/version.js

@@ -3,9 +3,12 @@ const Path = require('path')
 const Ipfs = require('ipfs')
 const ipfsx = require('../')
 
-;(async () => {
+async function main () {
   const node = await ipfsx(new Ipfs({ repo: Path.join(Os.tmpdir(), `${Date.now()}`) }))
   const info = await node.version()
   console.log(info)
   // { version: '0.32.2', repo: 7, commit: '' }
-})()
+  await node.stop()
+}
+
+main()