feat: support unixfs metadata and formatting it (#14)

Adds mode and mtime properties to normalised .add inputs, also adds functions for turning metadata into strings for CLI use.

BREAKING CHANGE:

In order to support metadata on intermediate directories, globSource in this module will now emit directories and files where previously it only emitted files.

Author: hugomrdias

package.json

@@ -31,16 +31,14 @@
     "fs-extra": "^8.1.0",
     "is-buffer": "^2.0.3",
     "is-electron": "^2.2.0",
-    "is-pull-stream": "0.0.0",
-    "is-stream": "^2.0.0",
-    "it-glob": "0.0.4",
-    "kind-of": "^6.0.2",
-    "pull-stream-to-async-iterator": "^1.0.2",
-    "readable-stream": "^3.4.0"
+    "it-glob": "0.0.7",
+    "ky": "^0.15.0",
+    "ky-universal": "^0.3.0",
+    "stream-to-it": "^0.2.0"
   },
   "devDependencies": {
     "aegir": "^20.3.0",
-    "async-iterator-all": "^1.0.0",
+    "it-all": "^1.0.1",
     "chai": "^4.2.0",
     "chai-as-promised": "^7.1.1",
     "dirty-chai": "^2.0.1",

src/files/format-mode.js

@@ -0,0 +1,66 @@
+'use strict'
+
+const S_ISUID = parseInt('4000', 8) //   set UID bit
+const S_ISGID = parseInt('2000', 8) //   set-group-ID bit (see below)
+const S_ISVTX = parseInt('1000', 8) //   sticky bit (see below)
+// const S_IRWXU = parseInt('700', 8) //    mask for file owner permissions
+const S_IRUSR = parseInt('400', 8) //    owner has read permission
+const S_IWUSR = parseInt('200', 8) //    owner has write permission
+const S_IXUSR = parseInt('100', 8) //    owner has execute permission
+// const S_IRWXG = parseInt('70', 8) //     mask for group permissions
+const S_IRGRP = parseInt('40', 8) //     group has read permission
+const S_IWGRP = parseInt('20', 8) //     group has write permission
+const S_IXGRP = parseInt('10', 8) //     group has execute permission
+// const S_IRWXO = parseInt('7', 8) //      mask for permissions for others (not in group)
+const S_IROTH = parseInt('4', 8) //      others have read permission
+const S_IWOTH = parseInt('2', 8) //      others have write permission
+const S_IXOTH = parseInt('1', 8) //      others have execute permission
+
+function checkPermission (mode, perm, type, output) {
+  if ((mode & perm) === perm) {
+    output.push(type)
+  } else {
+    output.push('-')
+  }
+}
+
+function formatMode (mode, isDirectory) {
+  const output = []
+
+  if (isDirectory) {
+    output.push('d')
+  } else {
+    output.push('-')
+  }
+
+  checkPermission(mode, S_IRUSR, 'r', output)
+  checkPermission(mode, S_IWUSR, 'w', output)
+
+  if ((mode & S_ISUID) === S_ISUID) {
+    output.push('s')
+  } else {
+    checkPermission(mode, S_IXUSR, 'x', output)
+  }
+
+  checkPermission(mode, S_IRGRP, 'r', output)
+  checkPermission(mode, S_IWGRP, 'w', output)
+
+  if ((mode & S_ISGID) === S_ISGID) {
+    output.push('s')
+  } else {
+    checkPermission(mode, S_IXGRP, 'x', output)
+  }
+
+  checkPermission(mode, S_IROTH, 'r', output)
+  checkPermission(mode, S_IWOTH, 'w', output)
+
+  if ((mode & S_ISVTX) === S_ISVTX) {
+    output.push('t')
+  } else {
+    checkPermission(mode, S_IXOTH, 'x', output)
+  }
+
+  return output.join('')
+}
+
+module.exports = formatMode

src/files/format-mtime.js

@@ -0,0 +1,19 @@
+'use strict'
+
+function formatMtime (mtime) {
+  if (mtime === undefined) {
+    return '-'
+  }
+
+  return new Date(mtime * 1000).toLocaleDateString(Intl.DateTimeFormat().resolvedOptions().locale, {
+    year: 'numeric',
+    month: 'short',
+    day: 'numeric',
+    hour: '2-digit',
+    minute: '2-digit',
+    second: '2-digit',
+    timeZoneName: 'short'
+  })
+}
+
+module.exports = formatMtime

src/files/glob-source.js

@@ -15,6 +15,10 @@ const kindOf = require('kind-of')
 * @param {Boolean} [options.hidden] Include .dot files in matched paths
 * @param {Array<String>} [options.ignore] Glob paths to ignore
 * @param {Boolean} [options.followSymlinks] follow symlinks
+* @param {Boolean} [options.preserveMode] preserve mode
+* @param {Boolean} [options.preserveMtime] preserve mtime
+* @param {Boolean} [options.mode] mode to use - if preserveMode is true this will be ignored
+* @param {Boolean} [options.mtime] mtime to use - if preserveMtime is true this will be ignored
 * @yields {Object} File objects in the form `{ path: String, content: AsyncIterator<Buffer> }`
 */
 module.exports = async function * globSource (paths, options) {
@@ -47,21 +51,49 @@ module.exports = async function * globSource (paths, options) {
     const stat = await fs.stat(absolutePath)
     const prefix = Path.dirname(absolutePath)
 
-    for await (const entry of toGlobSource({ path, type: stat.isDirectory() ? 'dir' : 'file', prefix }, globSourceOptions)) {
-      yield entry
+    let mode = options.mode
+
+    if (options.preserveMode) {
+      mode = stat.mode
+    }
+
+    let mtime = options.mtime
+
+    if (options.preserveMtime) {
+      mtime = parseInt(stat.mtimeMs / 1000)
+    }
+
+    if (stat.isDirectory()) {
+      yield {
+        path: `/${Path.basename(path)}`,
+        mode,
+        mtime
+      }
     }
+
+    yield * toGlobSource({
+      path,
+      type: stat.isDirectory() ? 'dir' : 'file',
+      prefix,
+      mode,
+      mtime,
+      preserveMode: options.preserveMode,
+      preserveMtime: options.preserveMtime
+    }, globSourceOptions)
   }
 }
 
-async function * toGlobSource ({ path, type, prefix }, options) {
+async function * toGlobSource ({ path, type, prefix, mode, mtime, preserveMode, preserveMtime }, options) {
   options = options || {}
 
   const baseName = Path.basename(path)
 
   if (type === 'file') {
     yield {
-      path: baseName.replace(prefix, ''),
-      content: fs.createReadStream(Path.isAbsolute(path) ? path : Path.join(process.cwd(), path))
+      path: `/${baseName.replace(prefix, '')}`,
+      content: fs.createReadStream(Path.isAbsolute(path) ? path : Path.join(process.cwd(), path)),
+      mode,
+      mtime
     }
 
     return
@@ -77,15 +109,29 @@ async function * toGlobSource ({ path, type, prefix }, options) {
 
   const globOptions = Object.assign({}, options.glob, {
     cwd: path,
-    nodir: true,
+    nodir: false,
     realpath: false,
     absolute: true
   })
 
   for await (const p of glob(path, '**/*', globOptions)) {
+    const stat = await fs.stat(p)
+
+    if (preserveMode || preserveMtime) {
+      if (preserveMode) {
+        mode = stat.mode
+      }
+
+      if (preserveMtime) {
+        mtime = parseInt(stat.mtimeMs / 1000)
+      }
+    }
+
     yield {
       path: toPosix(p.replace(prefix, '')),
-      content: fs.createReadStream(p)
+      content: stat.isFile() ? fs.createReadStream(p) : undefined,
+      mode,
+      mtime
     }
   }
 }

src/files/normalise-input.js

@@ -211,7 +211,11 @@ module.exports = function normaliseInput (input) {
 }
 
 function toFileObject (input) {
-  const obj = { path: input.path || '' }
+  const obj = {
+    path: input.path || '',
+    mode: input.mode,
+    mtime: input.mtime
+  }
 
   if (input.content) {
     obj.content = toAsyncIterable(input.content)

test/files/format-mode.spec.js

@@ -0,0 +1,58 @@
+'use strict'
+
+/* eslint-env mocha */
+const chai = require('chai')
+const dirtyChai = require('dirty-chai')
+const formatMode = require('../../src/files/format-mode')
+
+chai.use(dirtyChai)
+const expect = chai.expect
+
+describe('format-mode', function () {
+  it('formats mode for directories', function () {
+    expect(formatMode(parseInt('0777', 8), true)).to.equal('drwxrwxrwx')
+  })
+
+  it('formats mode for files', function () {
+    expect(formatMode(parseInt('0777', 8), false)).to.equal('-rwxrwxrwx')
+  })
+
+  it('setgid, setuid and stick bit', function () {
+    expect(formatMode(parseInt('1777', 8), false)).to.equal('-rwxrwxrwt')
+    expect(formatMode(parseInt('2777', 8), false)).to.equal('-rwxrwsrwx')
+    expect(formatMode(parseInt('4777', 8), false)).to.equal('-rwsrwxrwx')
+    expect(formatMode(parseInt('5777', 8), false)).to.equal('-rwsrwxrwt')
+    expect(formatMode(parseInt('6777', 8), false)).to.equal('-rwsrwsrwx')
+    expect(formatMode(parseInt('7777', 8), false)).to.equal('-rwsrwsrwt')
+  })
+
+  it('formats user', function () {
+    expect(formatMode(parseInt('0100', 8), false)).to.equal('---x------')
+    expect(formatMode(parseInt('0200', 8), false)).to.equal('--w-------')
+    expect(formatMode(parseInt('0300', 8), false)).to.equal('--wx------')
+    expect(formatMode(parseInt('0400', 8), false)).to.equal('-r--------')
+    expect(formatMode(parseInt('0500', 8), false)).to.equal('-r-x------')
+    expect(formatMode(parseInt('0600', 8), false)).to.equal('-rw-------')
+    expect(formatMode(parseInt('0700', 8), false)).to.equal('-rwx------')
+  })
+
+  it('formats group', function () {
+    expect(formatMode(parseInt('0010', 8), false)).to.equal('------x---')
+    expect(formatMode(parseInt('0020', 8), false)).to.equal('-----w----')
+    expect(formatMode(parseInt('0030', 8), false)).to.equal('-----wx---')
+    expect(formatMode(parseInt('0040', 8), false)).to.equal('----r-----')
+    expect(formatMode(parseInt('0050', 8), false)).to.equal('----r-x---')
+    expect(formatMode(parseInt('0060', 8), false)).to.equal('----rw----')
+    expect(formatMode(parseInt('0070', 8), false)).to.equal('----rwx---')
+  })
+
+  it('formats other', function () {
+    expect(formatMode(parseInt('0001', 8), false)).to.equal('---------x')
+    expect(formatMode(parseInt('0002', 8), false)).to.equal('--------w-')
+    expect(formatMode(parseInt('0003', 8), false)).to.equal('--------wx')
+    expect(formatMode(parseInt('0004', 8), false)).to.equal('-------r--')
+    expect(formatMode(parseInt('0005', 8), false)).to.equal('-------r-x')
+    expect(formatMode(parseInt('0006', 8), false)).to.equal('-------rw-')
+    expect(formatMode(parseInt('0007', 8), false)).to.equal('-------rwx')
+  })
+})

test/files/format-mtime.spec.js

@@ -0,0 +1,15 @@
+'use strict'
+
+/* eslint-env mocha */
+const chai = require('chai')
+const dirtyChai = require('dirty-chai')
+const formatMtime = require('../../src/files/format-mtime')
+
+chai.use(dirtyChai)
+const expect = chai.expect
+
+describe('format-mtime', function () {
+  it('formats mtime', function () {
+    expect((new Date(formatMtime(0))).getTime()).to.equal(0)
+  })
+})