Add metadata support (#63)

* chore: dep updated

* feat: adds touch and chmod commands and metadata

Adds UnixFSv1.5 metadata support to mfs, including displaying it
when listing dirs and statting files.

Also adds `touch` and `chmod` commands to manipulate metadata in
a similar way to the unix shell.

* chore: update deps

* chore: update deps

* test: add cli tests

* fix: add missing dep

* fix: downgrade repo

* fix: fix tests after hashOnly turned to onlyHash

* test: add tests for http interface

* fix: use multipart pr

* chore: remove unecessary browser overrides

* chore: update ipfs-utils dep

* fix: fix up tests, add support for timespecs

* chore: use multipart pr

* fix: support optional mtimes

Author: achingbrain

package.json

@@ -5,20 +5,23 @@
   "leadMaintainer": "Alex Potsides <[email protected]>",
   "main": "src/index.js",
   "browser": {
-    "fs": false,
-    "@hapi/joi": "joi-browser"
+    "@hapi/joi": "joi-browser",
+    "fs": false
   },
   "scripts": {
     "test": "aegir test",
     "test:node": "aegir test -t node",
+    "test:cli": "aegir test -t node -f test/cli/**/*.js",
+    "test:core": "aegir test -t node -f test/core/**/*.js",
+    "test:http": "aegir test -t node -f test/http/**/*.js",
     "test:browser": "aegir test -t browser",
     "test:webworker": "aegir test -t webworker",
     "build": "aegir build",
     "lint": "aegir lint",
     "release": "aegir release",
     "release-minor": "aegir release --type minor",
     "release-major": "aegir release --type major",
-    "coverage": "aegir coverage",
+    "coverage": "nyc --reporter=text --reporter=lcov npm run test:node",
     "dep-check": "aegir dep-check"
   },
   "repository": {
@@ -38,36 +41,46 @@
   },
   "homepage": "https://github.com/ipfs/js-ipfs-mfs#readme",
   "devDependencies": {
+    "@hapi/hapi": "^18.4.0",
     "aegir": "^20.0.0",
-    "async-iterator-all": "^1.0.0",
     "chai": "^4.2.0",
+    "chai-as-promised": "^7.1.1",
+    "delay": "^4.3.0",
     "detect-node": "^2.0.4",
     "detect-webworker": "^1.0.0",
     "dirty-chai": "^2.0.1",
+    "form-data": "^3.0.0",
     "ipfs-block-service": "~0.16.0",
-    "ipfs-repo": "~0.27.0",
+    "ipfs-repo": "^0.30.1",
     "ipld": "~0.25.0",
-    "memdown": "^4.0.0",
-    "temp-write": "^4.0.0"
+    "it-all": "^1.0.1",
+    "memdown": "^5.1.0",
+    "nyc": "^15.0.0",
+    "sinon": "^8.0.4",
+    "stream-to-promise": "^2.2.0",
+    "temp-write": "^4.0.0",
+    "yargs": "^15.0.2",
+    "yargs-promise": "^1.1.0"
   },
   "dependencies": {
     "@hapi/boom": "^7.4.2",
     "@hapi/joi": "^15.1.0",
-    "async-iterator-last": "^1.0.0",
-    "cids": "~0.7.1",
+    "cids": "^0.7.1",
     "debug": "^4.1.0",
     "err-code": "^2.0.0",
-    "hamt-sharding": "~0.0.2",
-    "interface-datastore": "~0.7.0",
-    "ipfs-multipart": "~0.2.0",
-    "ipfs-unixfs": "~0.1.16",
-    "ipfs-unixfs-exporter": "~0.38.0",
-    "ipfs-unixfs-importer": "~0.40.0",
-    "ipld-dag-pb": "~0.18.0",
+    "hamt-sharding": "^1.0.0",
+    "interface-datastore": "^0.8.0",
+    "ipfs-multipart": "^0.3.0",
+    "ipfs-unixfs": "^0.3.0",
+    "ipfs-unixfs-exporter": "^0.40.0",
+    "ipfs-unixfs-importer": "^0.43.0",
+    "ipfs-utils": "^0.4.2",
+    "ipld-dag-pb": "^0.18.0",
+    "it-last": "^1.0.1",
     "joi-browser": "^13.4.0",
     "mortice": "^2.0.0",
-    "multicodec": "~0.5.3",
-    "multihashes": "~0.4.14",
+    "multicodec": "^1.0.0",
+    "multihashes": "^0.4.14",
     "once": "^1.4.0",
     "pull-stream": "^3.6.9"
   },

src/cli/chmod.js

@@ -0,0 +1,80 @@
+'use strict'
+
+const {
+  asBoolean,
+  asOctal
+} = require('./utils')
+
+module.exports = {
+  command: 'chmod [mode] [path]',
+
+  describe: 'Change file modes',
+
+  builder: {
+    path: {
+      type: 'string',
+      describe: 'The MFS path to change the mode of'
+    },
+    mode: {
+      type: 'int',
+      coerce: asOctal,
+      describe: 'The mode to use'
+    },
+    recursive: {
+      alias: 'r',
+      type: 'boolean',
+      default: false,
+      coerce: asBoolean,
+      describe: 'Whether to change modes recursively'
+    },
+    codec: {
+      alias: 'c',
+      type: 'string',
+      default: 'dag-pb',
+      describe: 'If intermediate directories are created, use this codec to create them (experimental)'
+    },
+    'hash-alg': {
+      alias: 'h',
+      type: 'string',
+      default: 'sha2-256',
+      describe: 'Hash function to use. Will set CID version to 1 if used'
+    },
+    flush: {
+      alias: 'f',
+      type: 'boolean',
+      default: true,
+      coerce: asBoolean,
+      describe: 'Flush the changes to disk immediately'
+    },
+    'shard-split-threshold': {
+      type: 'number',
+      default: 1000,
+      describe: 'If a directory has more links than this, it will be transformed into a hamt-sharded-directory'
+    }
+  },
+
+  handler (argv) {
+    const {
+      path,
+      mode,
+      getIpfs,
+      recursive,
+      codec,
+      hashAlg,
+      flush,
+      shardSplitThreshold
+    } = argv
+
+    argv.resolve((async () => {
+      const ipfs = await getIpfs()
+
+      return ipfs.files.chmod(path, mode, {
+        recursive,
+        format: codec,
+        hashAlg,
+        flush,
+        shardSplitThreshold
+      })
+    })())
+  }
+}

src/cli/cp.js

@@ -17,18 +17,25 @@ module.exports = {
       coerce: asBoolean,
       describe: 'Create any non-existent intermediate directories'
     },
-    format: {
-      alias: 'h',
+    codec: {
+      alias: 'c',
       type: 'string',
       default: 'dag-pb',
-      describe: 'If intermediate directories are created, use this format to create them (experimental)'
+      describe: 'If intermediate directories are created, use this codec to create them (experimental)'
     },
     'hash-alg': {
       alias: 'h',
       type: 'string',
       default: 'sha2-256',
       describe: 'Hash function to use. Will set CID version to 1 if used'
     },
+    flush: {
+      alias: 'f',
+      type: 'boolean',
+      default: true,
+      coerce: asBoolean,
+      describe: 'Flush the changes to disk immediately'
+    },
     'shard-split-threshold': {
       type: 'number',
       default: 1000,
@@ -42,7 +49,8 @@ module.exports = {
       dest,
       getIpfs,
       parents,
-      format,
+      codec,
+      flush,
       hashAlg,
       shardSplitThreshold
     } = argv
@@ -51,7 +59,8 @@ module.exports = {
       const ipfs = await getIpfs()
       return ipfs.files.cp(source, dest, {
         parents,
-        format,
+        format: codec,
+        flush,
         hashAlg,
         shardSplitThreshold
       })

src/cli/index.js

@@ -1,9 +1,5 @@
 'use strict'
 
-const {
-  print
-} = require('./utils')
-
 const command = {
   command: 'files <command>',
 
@@ -14,7 +10,7 @@ const command = {
   },
 
   handler (argv) {
-    print('Type `jsipfs files --help` for more instructions')
+    argv.print('Type `jsipfs files --help` for more instructions')
   }
 }
 

src/cli/ls.js

@@ -4,12 +4,13 @@ const pull = require('pull-stream/pull')
 const onEnd = require('pull-stream/sinks/on-end')
 const through = require('pull-stream/throughs/through')
 const {
-  print,
   asBoolean
 } = require('./utils')
 const {
   FILE_SEPARATOR
 } = require('../core/utils/constants')
+const formatMode = require('ipfs-utils/src/files/format-mode')
+const formatMtime = require('ipfs-utils/src/files/format-mtime')
 
 module.exports = {
   command: 'ls [path]',
@@ -43,18 +44,15 @@ module.exports = {
       getIpfs,
       long,
       sort,
-      cidBase
+      cidBase,
+      print
     } = argv
 
     argv.resolve((async () => {
       const ipfs = await getIpfs()
       return new Promise((resolve, reject) => {
         if (sort) {
-          ipfs.files.ls(path || FILE_SEPARATOR, {
-            long,
-            sort,
-            cidBase
-          })
+          ipfs.files.ls(path || FILE_SEPARATOR)
             .then(files => {
               // https://github.com/ipfs/go-ipfs/issues/5181
               if (sort) {
@@ -64,8 +62,8 @@ module.exports = {
               }
 
               if (long) {
-                files.forEach(link => {
-                  print(`${link.name}\t${link.hash}\t${link.size}`)
+                files.forEach(file => {
+                  print(`${formatMode(file.mode, file.type === 1)}\t${formatMtime(file.mtime)}\t${file.name}\t${file.hash}\t${file.size}`)
                 })
               } else {
                 files.forEach(link => print(link.name))
@@ -85,7 +83,7 @@ module.exports = {
           }),
           through(file => {
             if (long) {
-              print(`${file.name}\t${file.hash}\t${file.size}`)
+              print(`${formatMode(file.mode, file.type === 1)}\t${formatMtime(file.mtime)}\t${file.name}\t${file.hash}\t${file.size}`)
             } else {
               print(file.name)
             }

src/cli/mkdir.js

@@ -1,7 +1,9 @@
 'use strict'
 
 const {
-  asBoolean
+  asBoolean,
+  asOctal,
+  asDateFromSeconds
 } = require('./utils')
 
 module.exports = {
@@ -23,9 +25,17 @@ module.exports = {
       default: 0,
       describe: 'Cid version to use. (experimental).'
     },
+    codec: {
+      alias: 'c',
+      type: 'string',
+      default: 'dag-pb',
+      describe: 'If intermediate directories are created, use this codec to create them (experimental)'
+    },
     'hash-alg': {
+      alias: 'h',
       type: 'string',
-      describe: 'Hash function to use. Will set Cid version to 1 if used. (experimental).'
+      default: 'sha2-256',
+      describe: 'Hash function to use. Will set CID version to 1 if used'
     },
     flush: {
       alias: 'f',
@@ -38,6 +48,16 @@ module.exports = {
       type: 'number',
       default: 1000,
       describe: 'If a directory has more links than this, it will be transformed into a hamt-sharded-directory'
+    },
+    mode: {
+      type: 'number',
+      coerce: asOctal,
+      describe: 'Mode to apply to the new directory'
+    },
+    mtime: {
+      type: 'date',
+      coerce: asDateFromSeconds,
+      describe: 'Mtime to apply to the new directory in seconds'
     }
   },
 
@@ -47,9 +67,12 @@ module.exports = {
       getIpfs,
       parents,
       cidVersion,
+      codec,
       hashAlg,
       flush,
-      shardSplitThreshold
+      shardSplitThreshold,
+      mode,
+      mtime
     } = argv
 
     argv.resolve((async () => {
@@ -58,9 +81,12 @@ module.exports = {
       return ipfs.files.mkdir(path, {
         parents,
         cidVersion,
+        format: codec,
         hashAlg,
         flush,
-        shardSplitThreshold
+        shardSplitThreshold,
+        mode,
+        mtime
       })
     })())
   }