feat: add fullPath option

When exporting a path with a subtree it's sometimes useful to get
information about everything in the path and not just the node
at the end.

This PR adds an option to return intermediate nodes in the export
stream.

It also documents the `maxDepth` option and lowers the sharding
threshold to make the shard tests more reliable in the browser.

Author: achingbrain

README.md

@@ -104,6 +104,91 @@ pull(
 )
 ```
 
+### `fullPath`
+
+If specified the exporter will emit an entry for every path component encountered.
+
+```javascript
+const exporter = require('ipfs-unixfs-exporter')
+const pull = require('pull-stream')
+const collect = require('pull-stream/sinks/collect')
+
+pull(
+  exporter('QmFoo.../bar/baz.txt', ipld, {
+    fullPath: true
+  })
+  collect((err, files) => {
+    console.info(files)
+
+    // [{
+    //   depth: 0,
+    //   name: 'QmFoo...',
+    //   path: 'QmFoo...',
+    //   size: ...
+    //   hash: Buffer
+    //   content: undefined
+    //   type: 'dir'
+    // }, {
+    //   depth: 1,
+    //   name: 'bar',
+    //   path: 'QmFoo.../bar',
+    //   size: ...
+    //   hash: Buffer
+    //   content: undefined
+    //   type: 'dir'
+    // }, {
+    //   depth: 2,
+    //   name: 'baz.txt',
+    //   path: 'QmFoo.../bar/baz.txt',
+    //   size: ...
+    //   hash: Buffer
+    //   content: <Pull stream>
+    //   type: 'file'
+    // }]
+    //
+  })
+)
+```
+
+### `maxDepth`
+
+If specified the exporter will only emit entries up to the specified depth.
+
+```javascript
+const exporter = require('ipfs-unixfs-exporter')
+const pull = require('pull-stream')
+const collect = require('pull-stream/sinks/collect')
+
+pull(
+  exporter('QmFoo.../bar/baz.txt', ipld, {
+    fullPath: true,
+    maxDepth: 1
+  })
+  collect((err, files) => {
+    console.info(files)
+
+    // [{
+    //   depth: 0,
+    //   name: 'QmFoo...',
+    //   path: 'QmFoo...',
+    //   size: ...
+    //   hash: Buffer
+    //   content: undefined
+    //   type: 'dir'
+    // }, {
+    //   depth: 1,
+    //   name: 'bar',
+    //   path: 'QmFoo.../bar',
+    //   size: ...
+    //   hash: Buffer
+    //   content: undefined
+    //   type: 'dir'
+    // }]
+    //
+  })
+)
+```
+
 [dag API]: https://github.com/ipfs/interface-ipfs-core/blob/master/SPEC/DAG.md
 [ipld-resolver instance]: https://github.com/ipld/js-ipld-resolver
 [UnixFS]: https://github.com/ipfs/specs/tree/master/unixfs

src/dir-flat.js

@@ -6,7 +6,7 @@ const cat = require('pull-cat')
 // Logic to export a unixfs directory.
 module.exports = dirExporter
 
-function dirExporter (cid, node, name, path, pathRest, resolve, size, dag, parent, depth) {
+function dirExporter (cid, node, name, path, pathRest, resolve, size, dag, parent, depth, options) {
   const accepts = pathRest[0]
 
   const dir = {
@@ -37,7 +37,7 @@ function dirExporter (cid, node, name, path, pathRest, resolve, size, dag, paren
   ]
 
   // place dir before if not specifying subtree
-  if (!pathRest.length) {
+  if (!pathRest.length || options.fullPath) {
     streams.unshift(pull.values([dir]))
   }
 

src/dir-hamt-sharded.js

@@ -6,7 +6,7 @@ const cat = require('pull-cat')
 // Logic to export a unixfs directory.
 module.exports = shardedDirExporter
 
-function shardedDirExporter (cid, node, name, path, pathRest, resolve, size, dag, parent, depth) {
+function shardedDirExporter (cid, node, name, path, pathRest, resolve, size, dag, parent, depth, options) {
   let dir
   if (!parent || (parent.path !== path)) {
     dir = {
@@ -49,7 +49,7 @@ function shardedDirExporter (cid, node, name, path, pathRest, resolve, size, dag
     )
   ]
 
-  if (!pathRest.length) {
+  if (!pathRest.length || options.fullPath) {
     streams.unshift(pull.values([dir]))
   }
 

src/file.js

@@ -7,7 +7,7 @@ const paramap = require('pull-paramap')
 const extractDataFromBlock = require('./extract-data-from-block')
 
 // Logic to export a single (possibly chunked) unixfs file.
-module.exports = (cid, node, name, path, pathRest, resolve, size, dag, parent, depth, offset, length) => {
+module.exports = (cid, node, name, path, pathRest, resolve, size, dag, parent, depth, options) => {
   const accepts = pathRest[0]
 
   if (accepts !== undefined && accepts !== path) {
@@ -24,6 +24,9 @@ module.exports = (cid, node, name, path, pathRest, resolve, size, dag, parent, d
 
   const fileSize = size || file.fileSize()
 
+  let offset = options.offset
+  let length = options.length
+
   if (offset < 0) {
     return pull.error(new Error('Offset must be greater than or equal to 0'))
   }

src/index.js

@@ -38,7 +38,8 @@ function pathBaseAndRest (path) {
 const defaultOptions = {
   maxDepth: Infinity,
   offset: undefined,
-  length: undefined
+  length: undefined,
+  fullPath: false
 }
 
 module.exports = (path, dag, options) => {
@@ -70,7 +71,7 @@ module.exports = (path, dag, options) => {
       return {
         depth: node.depth,
         name: node.name,
-        path: finalPathFor(node),
+        path: options.fullPath ? node.path : finalPathFor(node),
         size: node.size,
         hash: node.multihash,
         content: node.content,

src/raw.js

@@ -4,7 +4,7 @@ const pull = require('pull-stream')
 const extractDataFromBlock = require('./extract-data-from-block')
 
 // Logic to export a single raw block
-module.exports = (cid, node, name, path, pathRest, resolve, size, dag, parent, depth, offset, length) => {
+module.exports = (cid, node, name, path, pathRest, resolve, size, dag, parent, depth, options) => {
   const accepts = pathRest[0]
 
   if (accepts !== undefined && accepts !== path) {
@@ -13,6 +13,9 @@ module.exports = (cid, node, name, path, pathRest, resolve, size, dag, parent, d
 
   size = size || node.length
 
+  let offset = options.offset
+  let length = options.length
+
   if (offset < 0) {
     return pull.error(new Error('Offset must be greater than or equal to 0'))
   }

src/resolve.js

@@ -4,6 +4,7 @@ const UnixFS = require('ipfs-unixfs')
 const pull = require('pull-stream')
 const paramap = require('pull-paramap')
 const CID = require('cids')
+const waterfall = require('async/waterfall')
 
 const resolvers = {
   directory: require('./dir-flat'),
@@ -34,30 +35,37 @@ function createResolver (dag, options, depth, parent) {
       }
 
       if (item.object) {
-        return cb(null, resolveItem(null, item.object, item, options.offset, options.length))
+        return cb(null, resolveItem(null, item.object, item, options))
       }
 
       const cid = new CID(item.multihash)
 
-      dag.get(cid, (err, node) => {
-        if (err) {
-          return cb(err)
-        }
-
-        // const name = item.fromPathRest ? item.name : item.path
-        cb(null, resolveItem(cid, node.value, item, options.offset, options.length))
-      })
+      waterfall([
+        (done) => dag.get(cid, done),
+        (node, done) => done(null, resolveItem(cid, node.value, item, options))
+      ], cb)
     }),
     pull.flatten(),
     pull.filter(Boolean),
     pull.filter((node) => node.depth <= options.maxDepth)
   )
 
-  function resolveItem (cid, node, item, offset, length) {
-    return resolve(cid, node, item.name, item.path, item.pathRest, item.size, dag, item.parent || parent, item.depth, offset, length)
+  function resolveItem (cid, node, item, options) {
+    return resolve({
+      cid,
+      node,
+      name: item.name,
+      path: item.path,
+      pathRest: item.pathRest,
+      size: item.size,
+      dag,
+      parentNode: item.parent || parent,
+      depth: item.depth,
+      options
+    })
   }
 
-  function resolve (cid, node, name, path, pathRest, size, dag, parentNode, depth, offset, length) {
+  function resolve ({ cid, node, name, path, pathRest, size, dag, parentNode, depth, options }) {
     let type
 
     try {
@@ -67,11 +75,14 @@ function createResolver (dag, options, depth, parent) {
     }
 
     const nodeResolver = resolvers[type]
+
     if (!nodeResolver) {
       return pull.error(new Error('Unkown node type ' + type))
     }
+
     const resolveDeep = createResolver(dag, options, depth, node)
-    return nodeResolver(cid, node, name, path, pathRest, resolveDeep, size, dag, parentNode, depth, offset, length)
+
+    return nodeResolver(cid, node, name, path, pathRest, resolveDeep, size, dag, parentNode, depth, options)
   }
 }
 

test/exporter-sharded.spec.js

@@ -15,7 +15,7 @@ const randomBytes = require('./helpers/random-bytes')
 const exporter = require('../src')
 const importer = require('ipfs-unixfs-importer')
 
-const SHARD_SPLIT_THRESHOLD = 1000
+const SHARD_SPLIT_THRESHOLD = 10
 
 describe('exporter sharded', () => {
   let ipld
@@ -49,7 +49,8 @@ describe('exporter sharded', () => {
           }))
         ),
         importer(ipld, {
-          wrap: true
+          wrap: true,
+          shardSplitThreshold: SHARD_SPLIT_THRESHOLD
         }),
         pull.collect(cb)
       ),

test/exporter-subtree.spec.js

@@ -143,6 +143,50 @@ describe('exporter subtree', () => {
       }
     ], done)
   })
+
+  it('exports all components of a path', (done) => {
+    const content = randomBytes(ONE_MEG)
+
+    waterfall([
+      (cb) => pull(
+        pull.values([{
+          path: './200Bytes.txt',
+          content: randomBytes(ONE_MEG)
+        }, {
+          path: './level-1/200Bytes.txt',
+          content
+        }, {
+          path: './level-1/level-2'
+        }, {
+          path: './level-1/level-2/200Bytes.txt',
+          content
+        }]),
+        importer(ipld),
+        pull.collect(cb)
+      ),
+      (files, cb) => cb(null, files.pop().multihash),
+      (buf, cb) => cb(null, new CID(buf)),
+      (cid, cb) => pull(
+        exporter(`${cid.toBaseEncodedString()}/level-1/level-2/200Bytes.txt`, ipld, {
+          fullPath: true
+        }),
+        pull.collect((err, files) => cb(err, { cid, files }))
+      ),
+      ({ cid, files }, cb) => {
+        expect(files.length).to.equal(4)
+        expect(files[0].path).to.equal(cid.toBaseEncodedString())
+        expect(files[0].name).to.equal(cid.toBaseEncodedString())
+        expect(files[1].path).to.equal(`${cid.toBaseEncodedString()}/level-1`)
+        expect(files[1].name).to.equal('level-1')
+        expect(files[2].path).to.equal(`${cid.toBaseEncodedString()}/level-1/level-2`)
+        expect(files[2].name).to.equal('level-2')
+        expect(files[3].path).to.equal(`${cid.toBaseEncodedString()}/level-1/level-2/200Bytes.txt`)
+        expect(files[3].name).to.equal('200Bytes.txt')
+
+        cb()
+      }
+    ], done)
+  })
 })
 
 function fileEql (f1, f2, done) {

test/exporter.spec.js

@@ -60,7 +60,7 @@ describe('exporter', () => {
     })
   }
 
-  function addTestFile ({file, strategy = 'balanced', path = '/foo', maxChunkSize, rawLeaves}, cb) {
+  function addTestFile ({ file, strategy = 'balanced', path = '/foo', maxChunkSize, rawLeaves }, cb) {
     pull(
       pull.values([{
         path,
@@ -79,8 +79,8 @@ describe('exporter', () => {
     )
   }
 
-  function addAndReadTestFile ({file, offset, length, strategy = 'balanced', path = '/foo', maxChunkSize, rawLeaves}, cb) {
-    addTestFile({file, strategy, path, maxChunkSize, rawLeaves}, (error, multihash) => {
+  function addAndReadTestFile ({ file, offset, length, strategy = 'balanced', path = '/foo', maxChunkSize, rawLeaves }, cb) {
+    addTestFile({ file, strategy, path, maxChunkSize, rawLeaves }, (error, multihash) => {
       if (error) {
         return cb(error)
       }
@@ -100,7 +100,7 @@ describe('exporter', () => {
     })
   }
 
-  function addTestDirectory ({directory, strategy = 'balanced', maxChunkSize}, callback) {
+  function addTestDirectory ({ directory, strategy = 'balanced', maxChunkSize }, callback) {
     const input = push()
     const dirName = path.basename(directory)
 
@@ -293,7 +293,8 @@ describe('exporter', () => {
         content: randomBytes(100),
         links: [
           new DAGLink('', file.node.size, file.cid)
-        ]}, cb),
+        ]
+      }, cb),
       (result, cb) => {
         pull(
           exporter(result.cid, ipld, {