feat: support sharded directories

For big directories we need to support HAMT sharded directories.

Removes DAG traversal code and uses the importer/exporter from
UnixFS exclusivley to do DAG manipulations.

Transform directories into hamt-sharded directories when the number
of DAGLinks reaches a certain point, otherwise reading them from
the block store becomes too expensive.

License: MIT
Signed-off-by: achingbrain <[email protected]>

Author: achingbrain

package.json

@@ -55,7 +55,8 @@
     "interface-datastore": "~0.6.0",
     "ipfs-multipart": "~0.1.0",
     "ipfs-unixfs": "~0.1.16",
-    "ipfs-unixfs-engine": "~0.34.0",
+    "ipfs-unixfs-importer": "~0.37.0",
+    "ipfs-unixfs-exporter": "~0.35.0",
     "ipld-dag-pb": "~0.15.0",
     "is-pull-stream": "~0.0.0",
     "is-stream": "^1.1.0",

src/cli/cp.js

@@ -23,11 +23,16 @@ module.exports = {
       default: 'dag-pb',
       describe: 'If intermediate directories are created, use this format to create them (experimental)'
     },
-    hash: {
+    'hash-alg': {
       alias: 'h',
       type: 'string',
       default: 'sha2-256',
-      describe: 'Hash function to use. Will set Cid version to 1 if used. (experimental)'
+      describe: 'Hash function to use. Will set CID version to 1 if used'
+    },
+    'shard-split-threshold': {
+      type: 'number',
+      default: 1000,
+      describe: 'If a directory has more links than this, it will be transformed into a hamt-sharded-directory'
     }
   },
 
@@ -38,14 +43,16 @@ module.exports = {
       ipfs,
       parents,
       format,
-      hash
+      hashAlg,
+      shardSplitThreshold
     } = argv
 
     argv.resolve(
       ipfs.files.cp(source, dest, {
         parents,
         format,
-        hashAlg: hash
+        hashAlg,
+        shardSplitThreshold
       })
     )
   }

src/cli/ls.js

@@ -28,8 +28,7 @@ module.exports = {
       coerce: asBoolean,
       describe: 'Do not sort; list entries in directory order.'
     },
-    cidBase: {
-      alias: 'cid-base',
+    'cid-base': {
       default: 'base58btc',
       describe: 'CID base to use.'
     }

src/cli/mkdir.js

@@ -17,13 +17,13 @@ module.exports = {
       coerce: asBoolean,
       describe: 'No error if existing, make parent directories as needed.'
     },
-    cidVersion: {
-      alias: ['cid-ver', 'cid-version'],
+    'cid-version': {
+      alias: ['cid-ver'],
       type: 'number',
       default: 0,
       describe: 'Cid version to use. (experimental).'
     },
-    hashAlg: {
+    'hash-alg': {
       type: 'string',
       describe: 'Hash function to use. Will set Cid version to 1 if used. (experimental).'
     },
@@ -33,6 +33,11 @@ module.exports = {
       default: true,
       coerce: asBoolean,
       describe: 'Flush the changes to disk immediately'
+    },
+    'shard-split-threshold': {
+      type: 'number',
+      default: 1000,
+      describe: 'If a directory has more links than this, it will be transformed into a hamt-sharded-directory'
     }
   },
 
@@ -43,15 +48,17 @@ module.exports = {
       parents,
       cidVersion,
       hashAlg,
-      flush
+      flush,
+      shardSplitThreshold
     } = argv
 
     argv.resolve(
       ipfs.files.mkdir(path, {
         parents,
         cidVersion,
         hashAlg,
-        flush
+        flush,
+        shardSplitThreshold
       })
     )
   }

src/cli/mv.js

@@ -23,6 +23,11 @@ module.exports = {
       default: false,
       coerce: asBoolean,
       describe: 'Remove directories recursively'
+    },
+    'shard-split-threshold': {
+      type: 'number',
+      default: 1000,
+      describe: 'If a directory has more links than this, it will be transformed into a hamt-sharded-directory'
     }
   },
 
@@ -32,13 +37,15 @@ module.exports = {
       dest,
       ipfs,
       parents,
-      recursive
+      recursive,
+      shardSplitThreshold
     } = argv
 
     argv.resolve(
       ipfs.files.mv(source, dest, {
         parents,
-        recursive
+        recursive,
+        shardSplitThreshold
       })
     )
   }

src/cli/stat.js

@@ -35,15 +35,14 @@ Type: <type>`,
       coerce: asBoolean,
       describe: 'Print only size. Implies \'--format=<cumulsize>\'. Conflicts with other format options.'
     },
-    withLocal: {
+    'with-local': {
       alias: 'l',
       type: 'boolean',
       default: false,
       coerce: asBoolean,
       describe: 'Compute the amount of the dag that is local, and if possible the total size'
     },
-    cidBase: {
-      alias: 'cid-base',
+    'cid-base': {
       default: 'base58btc',
       describe: 'CID base to use.'
     }

src/cli/write.js

@@ -39,14 +39,14 @@ module.exports = {
       coerce: asBoolean,
       describe: 'Truncate the file after writing'
     },
-    rawLeaves: {
+    'raw-leaves': {
       alias: 'r',
       type: 'boolean',
       default: false,
       coerce: asBoolean,
       describe: 'Whether to write leaf nodes as raw UnixFS nodes'
     },
-    reduceSingleLeafToSelf: {
+    'reduce-single-leaf-to-self': {
       type: 'boolean',
       default: false,
       coerce: asBoolean,
@@ -64,20 +64,25 @@ module.exports = {
       type: 'string',
       default: 'balanced'
     },
-    cidVersion: {
-      alias: ['cid-ver', 'cid-version'],
+    'cid-version': {
+      alias: ['cid-ver'],
       type: 'number',
       default: 0,
-      describe: 'Cid version to use. (experimental).'
+      describe: 'Cid version to use'
     },
-    hashAlg: {
+    'hash-alg': {
       alias: 'h',
       type: 'string',
       default: 'sha2-256'
     },
     format: {
       type: 'string',
       default: 'dag-pb'
+    },
+    'shard-split-threshold': {
+      type: 'number',
+      default: 1000,
+      describe: 'If a directory has more links than this, it will be transformed into a hamt-sharded-directory'
     }
   },
 
@@ -97,7 +102,8 @@ module.exports = {
       parents,
       progress,
       strategy,
-      flush
+      flush,
+      shardSplitThreshold
     } = argv
 
     argv.resolve(
@@ -114,7 +120,8 @@ module.exports = {
         parents,
         progress,
         strategy,
-        flush
+        flush,
+        shardSplitThreshold
       })
     )
   }