feat: support unixfs metadata and formatting it (#14)

package.json

@@ -33,14 +33,14 @@
     "is-electron": "^2.2.0",
     "is-pull-stream": "0.0.0",
     "is-stream": "^2.0.0",
-    "it-glob": "0.0.4",
+    "it-glob": "0.0.7",
     "kind-of": "^6.0.2",
     "pull-stream-to-async-iterator": "^1.0.2",
     "readable-stream": "^3.4.0"
   },
   "devDependencies": {
     "aegir": "^20.3.0",
-    "async-iterator-all": "^1.0.0",
+    "it-all": "^1.0.1",
     "chai": "^4.2.0",
     "chai-as-promised": "^7.1.1",
     "dirty-chai": "^2.0.1",

src/files/format-mode.js

@@ -0,0 +1,66 @@
+'use strict'
+
+const S_ISUID = parseInt('4000', 8) //   set UID bit
+const S_ISGID = parseInt('2000', 8) //   set-group-ID bit (see below)
+const S_ISVTX = parseInt('1000', 8) //   sticky bit (see below)
+// const S_IRWXU = parseInt('700', 8) //    mask for file owner permissions
+const S_IRUSR = parseInt('400', 8) //    owner has read permission
+const S_IWUSR = parseInt('200', 8) //    owner has write permission
+const S_IXUSR = parseInt('100', 8) //    owner has execute permission
+// const S_IRWXG = parseInt('70', 8) //     mask for group permissions
+const S_IRGRP = parseInt('40', 8) //     group has read permission
+const S_IWGRP = parseInt('20', 8) //     group has write permission
+const S_IXGRP = parseInt('10', 8) //     group has execute permission
+// const S_IRWXO = parseInt('7', 8) //      mask for permissions for others (not in group)
+const S_IROTH = parseInt('4', 8) //      others have read permission
+const S_IWOTH = parseInt('2', 8) //      others have write permission
+const S_IXOTH = parseInt('1', 8) //      others have execute permission
+
+function checkPermission (mode, perm, type, output) {
+  if ((mode & perm) === perm) {
+    output.push(type)
+  } else {
+    output.push('-')
+  }
+}
+
+function formatMode (mode, isDirectory) {
+  const output = []
+
+  if (isDirectory) {
+    output.push('d')
+  } else {
+    output.push('-')
+  }
+
+  checkPermission(mode, S_IRUSR, 'r', output)
+  checkPermission(mode, S_IWUSR, 'w', output)
+
+  if ((mode & S_ISUID) === S_ISUID) {
+    output.push('s')
+  } else {
+    checkPermission(mode, S_IXUSR, 'x', output)
+  }
+
+  checkPermission(mode, S_IRGRP, 'r', output)
+  checkPermission(mode, S_IWGRP, 'w', output)
+
+  if ((mode & S_ISGID) === S_ISGID) {
+    output.push('s')
+  } else {
+    checkPermission(mode, S_IXGRP, 'x', output)
+  }
+
+  checkPermission(mode, S_IROTH, 'r', output)
+  checkPermission(mode, S_IWOTH, 'w', output)
+
+  if ((mode & S_ISVTX) === S_ISVTX) {
+    output.push('t')
+  } else {
+    checkPermission(mode, S_IXOTH, 'x', output)
+  }
+
+  return output.join('')
+}
+
+module.exports = formatMode

src/files/format-mtime.js

@@ -0,0 +1,19 @@
+'use strict'
+
+function formatMtime (mtime) {
+  if (mtime === undefined) {
+    return '-'
+  }
+
+  return new Date(mtime * 1000).toLocaleDateString(Intl.DateTimeFormat().resolvedOptions().locale, {
+    year: 'numeric',
+    month: 'short',
+    day: 'numeric',
+    hour: '2-digit',
+    minute: '2-digit',
+    second: '2-digit',
+    timeZoneName: 'short'
+  })
+}
+
+module.exports = formatMtime

src/files/glob-source.js

@@ -15,6 +15,10 @@ const kindOf = require('kind-of')
 * @param {Boolean} [options.hidden] Include .dot files in matched paths
 * @param {Array<String>} [options.ignore] Glob paths to ignore
 * @param {Boolean} [options.followSymlinks] follow symlinks
+* @param {Boolean} [options.preserveMode] preserve mode
+* @param {Boolean} [options.preserveMtime] preserve mtime
+* @param {Boolean} [options.mode] mode to use - if preserveMode is true this will be ignored
+* @param {Boolean} [options.mtime] mtime to use - if preserveMtime is true this will be ignored
 * @yields {Object} File objects in the form `{ path: String, content: AsyncIterator<Buffer> }`
 */
 module.exports = async function * globSource (paths, options) {
@@ -47,21 +51,49 @@ module.exports = async function * globSource (paths, options) {
     const stat = await fs.stat(absolutePath)
     const prefix = Path.dirname(absolutePath)
 
-    for await (const entry of toGlobSource({ path, type: stat.isDirectory() ? 'dir' : 'file', prefix }, globSourceOptions)) {
-      yield entry
+    let mode = options.mode
+
+    if (options.preserveMode) {
+      mode = stat.mode
+    }
+
+    let mtime = options.mtime
+
+    if (options.preserveMtime) {
+      mtime = parseInt(stat.mtimeMs / 1000)
+    }
+
+    if (stat.isDirectory()) {
+      yield {
+        path: `/${Path.basename(path)}`,
+        mode,
+        mtime
+      }
     }
+
+    yield * toGlobSource({
+      path,
+      type: stat.isDirectory() ? 'dir' : 'file',
+      prefix,
+      mode,
+      mtime,
+      preserveMode: options.preserveMode,
+      preserveMtime: options.preserveMtime
+    }, globSourceOptions)
   }
 }
 
-async function * toGlobSource ({ path, type, prefix }, options) {
+async function * toGlobSource ({ path, type, prefix, mode, mtime, preserveMode, preserveMtime }, options) {
   options = options || {}
 
   const baseName = Path.basename(path)
 
   if (type === 'file') {
     yield {
-      path: baseName.replace(prefix, ''),
-      content: fs.createReadStream(Path.isAbsolute(path) ? path : Path.join(process.cwd(), path))
+      path: `/${baseName.replace(prefix, '')}`,
+      content: fs.createReadStream(Path.isAbsolute(path) ? path : Path.join(process.cwd(), path)),
+      mode,
+      mtime
     }
 
     return
@@ -77,15 +109,29 @@ async function * toGlobSource ({ path, type, prefix }, options) {
 
   const globOptions = Object.assign({}, options.glob, {
     cwd: path,
-    nodir: true,
+    nodir: false,
     realpath: false,
     absolute: true
   })
 
   for await (const p of glob(path, '**/*', globOptions)) {
+    const stat = await fs.stat(p)
+
+    if (preserveMode || preserveMtime) {
+      if (preserveMode) {
+        mode = stat.mode
+      }
+
+      if (preserveMtime) {
+        mtime = parseInt(stat.mtimeMs / 1000)
+      }
+    }
+
     yield {
       path: toPosix(p.replace(prefix, '')),
-      content: fs.createReadStream(p)
+      content: stat.isFile() ? fs.createReadStream(p) : undefined,
+      mode,
+      mtime
     }
   }
 }

src/files/normalise-input.js

@@ -211,7 +211,11 @@ module.exports = function normaliseInput (input) {
 }
 
 function toFileObject (input) {
-  const obj = { path: input.path || '' }
+  const obj = {
+    path: input.path || '',
+    mode: input.mode,
+    mtime: input.mtime
+  }
 
   if (input.content) {
     obj.content = toAsyncIterable(input.content)

test/files/format-mode.spec.js

@@ -0,0 +1,58 @@
+'use strict'
+
+/* eslint-env mocha */
+const chai = require('chai')
+const dirtyChai = require('dirty-chai')
+const formatMode = require('../../src/files/format-mode')
+
+chai.use(dirtyChai)
+const expect = chai.expect
+
+describe('format-mode', function () {
+  it('formats mode for directories', function () {
+    expect(formatMode(parseInt('0777', 8), true)).to.equal('drwxrwxrwx')
+  })
+
+  it('formats mode for files', function () {
+    expect(formatMode(parseInt('0777', 8), false)).to.equal('-rwxrwxrwx')
+  })
+
+  it('setgid, setuid and stick bit', function () {
+    expect(formatMode(parseInt('1777', 8), false)).to.equal('-rwxrwxrwt')
+    expect(formatMode(parseInt('2777', 8), false)).to.equal('-rwxrwsrwx')
+    expect(formatMode(parseInt('4777', 8), false)).to.equal('-rwsrwxrwx')
+    expect(formatMode(parseInt('5777', 8), false)).to.equal('-rwsrwxrwt')
+    expect(formatMode(parseInt('6777', 8), false)).to.equal('-rwsrwsrwx')
+    expect(formatMode(parseInt('7777', 8), false)).to.equal('-rwsrwsrwt')
+  })
+
+  it('formats user', function () {
+    expect(formatMode(parseInt('0100', 8), false)).to.equal('---x------')
+    expect(formatMode(parseInt('0200', 8), false)).to.equal('--w-------')
+    expect(formatMode(parseInt('0300', 8), false)).to.equal('--wx------')
+    expect(formatMode(parseInt('0400', 8), false)).to.equal('-r--------')
+    expect(formatMode(parseInt('0500', 8), false)).to.equal('-r-x------')
+    expect(formatMode(parseInt('0600', 8), false)).to.equal('-rw-------')
+    expect(formatMode(parseInt('0700', 8), false)).to.equal('-rwx------')
+  })
+
+  it('formats group', function () {
+    expect(formatMode(parseInt('0010', 8), false)).to.equal('------x---')
+    expect(formatMode(parseInt('0020', 8), false)).to.equal('-----w----')
+    expect(formatMode(parseInt('0030', 8), false)).to.equal('-----wx---')
+    expect(formatMode(parseInt('0040', 8), false)).to.equal('----r-----')
+    expect(formatMode(parseInt('0050', 8), false)).to.equal('----r-x---')
+    expect(formatMode(parseInt('0060', 8), false)).to.equal('----rw----')
+    expect(formatMode(parseInt('0070', 8), false)).to.equal('----rwx---')
+  })
+
+  it('formats other', function () {
+    expect(formatMode(parseInt('0001', 8), false)).to.equal('---------x')
+    expect(formatMode(parseInt('0002', 8), false)).to.equal('--------w-')
+    expect(formatMode(parseInt('0003', 8), false)).to.equal('--------wx')
+    expect(formatMode(parseInt('0004', 8), false)).to.equal('-------r--')
+    expect(formatMode(parseInt('0005', 8), false)).to.equal('-------r-x')
+    expect(formatMode(parseInt('0006', 8), false)).to.equal('-------rw-')
+    expect(formatMode(parseInt('0007', 8), false)).to.equal('-------rwx')
+  })
+})

test/files/format-mtime.spec.js

@@ -0,0 +1,15 @@
+'use strict'
+
+/* eslint-env mocha */
+const chai = require('chai')
+const dirtyChai = require('dirty-chai')
+const formatMtime = require('../../src/files/format-mtime')
+
+chai.use(dirtyChai)
+const expect = chai.expect
+
+describe('format-mtime', function () {
+  it('formats mtime', function () {
+    expect((new Date(formatMtime(0))).getTime()).to.equal(0)
+  })
+})

test/files/glob-source.spec.js

@@ -5,99 +5,107 @@ const chai = require('chai')
 const dirtyChai = require('dirty-chai')
 const chaiAsPromised = require('chai-as-promised')
 const globSource = require('../../src/files/glob-source')
-const all = require('async-iterator-all')
+const all = require('it-all')
 const path = require('path')
 const {
   isNode
 } = require('../../src/env')
+const fs = require('fs')
 
 chai.use(dirtyChai)
 chai.use(chaiAsPromised)
 const expect = chai.expect
 
-describe('glob-source', () => {
-  it('single file, relative path', async function () {
-    if (!isNode) {
-      return this.skip()
-    }
+function fixture (file) {
+  return path.resolve(path.join(__dirname, '..', 'fixtures', file))
+}
 
-    const result = await all(globSource(path.relative(process.cwd(), path.join(__dirname, '..', 'fixtures', 'file-0.html'))))
+function findMode (file) {
+  return fs.statSync(fixture(file)).mode
+}
 
-    expect(result.length).to.equal(1)
-    expect(result[0].path).to.equal('file-0.html')
-  })
+function findMtime (file) {
+  return parseInt(fs.statSync(fixture(file)).mtimeMs / 1000)
+}
 
-  it('directory, relative path', async function () {
+describe('glob-source', () => {
+  it('single file, relative path', async function () {
     if (!isNode) {
       return this.skip()
     }
 
-    const result = await all(globSource(path.relative(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir')), {
-      recursive: true
-    }))
+    const result = await all(globSource(fixture('file-0.html')))
 
-    expect(result.length).to.equal(3)
-    expect(result[0].path).to.equal('/dir/file-1.txt')
-    expect(result[1].path).to.equal('/dir/file-2.js')
-    expect(result[2].path).to.equal('/dir/file-3.css')
+    expect(result.length).to.equal(1)
+    expect(result[0].path).to.equal('/file-0.html')
   })
 
   it('single file, absolute path', async function () {
     if (!isNode) {
       return this.skip()
     }
 
-    const result = await all(globSource(path.resolve(process.cwd(), path.join(__dirname, '..', 'fixtures', 'file-0.html'))))
+    const result = await all(globSource(fixture('file-0.html')))
 
     expect(result.length).to.equal(1)
-    expect(result[0].path).to.equal('file-0.html')
+    expect(result[0].path).to.equal('/file-0.html')
   })
 
   it('directory, relative path', async function () {
     if (!isNode) {
       return this.skip()
     }
 
-    const result = await all(globSource(path.resolve(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir')), {
+    const result = await all(globSource(fixture('/dir'), {
       recursive: true
     }))
 
-    expect(result.length).to.equal(3)
-    expect(result[0].path).to.equal('/dir/file-1.txt')
-    expect(result[1].path).to.equal('/dir/file-2.js')
-    expect(result[2].path).to.equal('/dir/file-3.css')
+    expect(result).to.have.lengthOf(6)
+    expect(result).to.have.nested.property('[0].path', '/dir')
+    expect(result).to.not.have.nested.property('[0].content')
+    expect(result).to.have.nested.property('[1].path', '/dir/file-1.txt')
+    expect(result).to.have.nested.property('[2].path', '/dir/file-2.js')
+    expect(result).to.have.nested.property('[3].path', '/dir/file-3.css')
+    expect(result).to.have.nested.property('[4].path', '/dir/nested-dir')
+    expect(result).to.have.nested.property('[5].path', '/dir/nested-dir/other.txt')
   })
 
   it('directory, hidden files', async function () {
     if (!isNode) {
       return this.skip()
     }
 
-    const result = await all(globSource(path.resolve(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir')), {
+    const result = await all(globSource(fixture('/dir'), {
       recursive: true,
       hidden: true
     }))
 
-    expect(result.length).to.equal(4)
-    expect(result[0].path).to.equal('/dir/.hidden.txt')
-    expect(result[1].path).to.equal('/dir/file-1.txt')
-    expect(result[2].path).to.equal('/dir/file-2.js')
-    expect(result[3].path).to.equal('/dir/file-3.css')
+    expect(result).to.have.lengthOf(7)
+    expect(result).to.have.nested.property('[0].path', '/dir')
+    expect(result).to.have.nested.property('[1].path', '/dir/.hidden.txt')
+    expect(result).to.have.nested.property('[2].path', '/dir/file-1.txt')
+    expect(result).to.have.nested.property('[3].path', '/dir/file-2.js')
+    expect(result).to.have.nested.property('[4].path', '/dir/file-3.css')
+    expect(result).to.have.nested.property('[5].path', '/dir/nested-dir')
+    expect(result).to.have.nested.property('[6].path', '/dir/nested-dir/other.txt')
   })
 
   it('directory, ignore files', async function () {
     if (!isNode) {
       return this.skip()
     }
 
-    const result = await all(globSource(path.resolve(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir')), {
+    const result = await all(globSource(fixture('/dir'), {
       recursive: true,
       ignore: ['**/file-1.txt']
     }))
 
-    expect(result.length).to.equal(2)
-    expect(result[0].path).to.equal('/dir/file-2.js')
-    expect(result[1].path).to.equal('/dir/file-3.css')
+    expect(result).to.have.lengthOf(5)
+    expect(result).to.have.nested.property('[0].path', '/dir')
+    expect(result).to.have.nested.property('[1].path', '/dir/file-2.js')
+    expect(result).to.have.nested.property('[2].path', '/dir/file-3.css')
+    expect(result).to.have.nested.property('[3].path', '/dir/nested-dir')
+    expect(result).to.have.nested.property('[4].path', '/dir/nested-dir/other.txt')
   })
 
   it('multiple paths', async function () {
@@ -106,20 +114,120 @@ describe('glob-source', () => {
     }
 
     const result = await all(globSource([
-      path.relative(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir', 'file-1.txt')),
-      path.relative(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir', 'file-2.js'))
+      fixture('/dir/file-1.txt'),
+      fixture('/dir/file-2.js')
     ]))
 
-    expect(result.length).to.equal(2)
-    expect(result[0].path).to.equal('file-1.txt')
-    expect(result[1].path).to.equal('file-2.js')
+    expect(result).to.have.lengthOf(2)
+    expect(result).to.have.nested.property('[0].path', '/file-1.txt')
+    expect(result).to.have.nested.property('[1].path', '/file-2.js')
   })
 
   it('requires recursive flag for directory', async function () {
     if (!isNode) {
       return this.skip()
     }
 
-    await expect(all(globSource(path.resolve(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir'))))).to.be.rejectedWith(/recursive option not set/)
+    await expect(all(globSource(fixture('/dir')))).to.be.rejectedWith(/recursive option not set/)
+  })
+
+  it('preserves mode for directories', async function () {
+    if (!isNode) {
+      return this.skip()
+    }
+
+    const result = await all(globSource(fixture('/dir'), {
+      preserveMode: true,
+      recursive: true
+    }))
+
+    expect(result).to.have.lengthOf(6)
+    expect(result).to.have.nested.property('[0].path', '/dir')
+    expect(result).to.have.nested.property('[0].mode', findMode('/dir'))
+    expect(result).to.have.nested.property('[1].path', '/dir/file-1.txt')
+    expect(result).to.have.nested.property('[1].mode', findMode('/dir/file-1.txt'))
+    expect(result).to.have.nested.property('[2].path', '/dir/file-2.js')
+    expect(result).to.have.nested.property('[2].mode', findMode('/dir/file-2.js'))
+    expect(result).to.have.nested.property('[3].path', '/dir/file-3.css')
+    expect(result).to.have.nested.property('[3].mode', findMode('/dir/file-3.css'))
+    expect(result).to.have.nested.property('[4].path', '/dir/nested-dir')
+    expect(result).to.have.nested.property('[4].mode', findMode('/dir/nested-dir'))
+    expect(result).to.have.nested.property('[5].path', '/dir/nested-dir/other.txt')
+    expect(result).to.have.nested.property('[5].mode', findMode('/dir/nested-dir/other.txt'))
+  })
+
+  it('overrides mode for directories', async function () {
+    if (!isNode) {
+      return this.skip()
+    }
+
+    const result = await all(globSource(fixture('/dir'), {
+      recursive: true,
+      mode: 5
+    }))
+
+    expect(result).to.have.lengthOf(6)
+    expect(result).to.have.nested.property('[0].path', '/dir')
+    expect(result).to.have.nested.property('[0].mode', 5)
+    expect(result).to.have.nested.property('[1].path', '/dir/file-1.txt')
+    expect(result).to.have.nested.property('[1].mode', 5)
+    expect(result).to.have.nested.property('[2].path', '/dir/file-2.js')
+    expect(result).to.have.nested.property('[2].mode', 5)
+    expect(result).to.have.nested.property('[3].path', '/dir/file-3.css')
+    expect(result).to.have.nested.property('[3].mode', 5)
+    expect(result).to.have.nested.property('[4].path', '/dir/nested-dir')
+    expect(result).to.have.nested.property('[4].mode', 5)
+    expect(result).to.have.nested.property('[5].path', '/dir/nested-dir/other.txt')
+    expect(result).to.have.nested.property('[5].mode', 5)
+  })
+
+  it('preserves mtime for directories', async function () {
+    if (!isNode) {
+      return this.skip()
+    }
+
+    const result = await all(globSource(fixture('/dir'), {
+      preserveMtime: true,
+      recursive: true
+    }))
+
+    expect(result).to.have.lengthOf(6)
+    expect(result).to.have.nested.property('[0].path', '/dir')
+    expect(result).to.have.nested.property('[0].mtime', findMtime('/dir'))
+    expect(result).to.have.nested.property('[1].path', '/dir/file-1.txt')
+    expect(result).to.have.nested.property('[1].mtime', findMtime('/dir/file-1.txt'))
+    expect(result).to.have.nested.property('[2].path', '/dir/file-2.js')
+    expect(result).to.have.nested.property('[2].mtime', findMtime('/dir/file-2.js'))
+    expect(result).to.have.nested.property('[3].path', '/dir/file-3.css')
+    expect(result).to.have.nested.property('[3].mtime', findMtime('/dir/file-3.css'))
+    expect(result).to.have.nested.property('[4].path', '/dir/nested-dir')
+    expect(result).to.have.nested.property('[4].mtime', findMtime('/dir/nested-dir'))
+    expect(result).to.have.nested.property('[5].path', '/dir/nested-dir/other.txt')
+    expect(result).to.have.nested.property('[5].mtime', findMtime('/dir/nested-dir/other.txt'))
+  })
+
+  it('overrides mtime for directories', async function () {
+    if (!isNode) {
+      return this.skip()
+    }
+
+    const result = await all(globSource(fixture('/dir'), {
+      recursive: true,
+      mtime: 5
+    }))
+
+    expect(result).to.have.lengthOf(6)
+    expect(result).to.have.nested.property('[0].path', '/dir')
+    expect(result).to.have.nested.property('[0].mtime', 5)
+    expect(result).to.have.nested.property('[1].path', '/dir/file-1.txt')
+    expect(result).to.have.nested.property('[1].mtime', 5)
+    expect(result).to.have.nested.property('[2].path', '/dir/file-2.js')
+    expect(result).to.have.nested.property('[2].mtime', 5)
+    expect(result).to.have.nested.property('[3].path', '/dir/file-3.css')
+    expect(result).to.have.nested.property('[3].mtime', 5)
+    expect(result).to.have.nested.property('[4].path', '/dir/nested-dir')
+    expect(result).to.have.nested.property('[4].mtime', 5)
+    expect(result).to.have.nested.property('[5].path', '/dir/nested-dir/other.txt')
+    expect(result).to.have.nested.property('[5].mtime', 5)
   })
 })

test/files/normalise-input.spec.js

@@ -6,7 +6,7 @@ const dirtyChai = require('dirty-chai')
 const normalise = require('../../src/files/normalise-input')
 const { supportsFileReader } = require('../../src/supports')
 const { Buffer } = require('buffer')
-const all = require('async-iterator-all')
+const all = require('it-all')
 const pull = require('pull-stream')
 const Readable2 = require('readable-stream-2')
 const Readable3 = require('readable-stream')