feat: refs-local reconstructs CIDs

Related to #2415

Author: AuHau

package.json

@@ -99,11 +99,11 @@
     "ipfs-bitswap": "^0.26.0",
     "ipfs-block": "~0.8.1",
     "ipfs-block-service": "~0.16.0",
-    "ipfs-http-client": "^38.2.0",
+    "ipfs-http-client": "github:ipfs/js-ipfs-http-client#auhau/feat/multihash_keys_in_datastore",
     "ipfs-http-response": "~0.3.1",
     "ipfs-mfs": "^0.13.0",
     "ipfs-multipart": "^0.2.0",
-    "ipfs-repo": "^0.28.0",
+    "ipfs-repo": "github:ipfs/js-ipfs-repo#auhau/feat/multihash_keys_in_datastore",
     "ipfs-unixfs": "~0.1.16",
     "ipfs-unixfs-exporter": "^0.38.0",
     "ipfs-unixfs-importer": "^0.40.0",
@@ -203,7 +203,7 @@
     "execa": "^2.0.4",
     "form-data": "^2.5.1",
     "hat": "0.0.3",
-    "interface-ipfs-core": "^0.117.2",
+    "interface-ipfs-core": "github:ipfs/interface-js-ipfs-core#auhau/feat/multihash_keys_in_datastore",
     "ipfs-interop": "^0.1.1",
     "ipfsd-ctl": "^0.47.2",
     "libp2p-websocket-star": "~0.10.2",

src/cli/commands/refs-local.js

@@ -5,12 +5,22 @@ module.exports = {
 
   describe: 'List all local references.',
 
-  handler ({ getIpfs, print, resolve }) {
+  builder (yargs) {
+    return yargs
+      .option('multihash', {
+        desc: 'Shows base32 encoded multihashes instead of reconstructed CIDs',
+        type: 'boolean',
+        default: false
+      })
+      .epilog('CIDs are reconstructed therefore they might differ from those under which the blocks were originally stored.')
+  },
+
+  handler ({ getIpfs, print, resolve, multihash }) {
     resolve((async () => {
       const ipfs = await getIpfs()
 
       return new Promise((resolve, reject) => {
-        const stream = ipfs.refs.localReadableStream()
+        const stream = ipfs.refs.localReadableStream({ multihash })
 
         stream.on('error', reject)
         stream.on('end', resolve)

src/core/components/files-regular/refs-local-pull-stream.js

@@ -1,26 +1,18 @@
 'use strict'
 
-const CID = require('cids')
-const base32 = require('base32.js')
+const { keyToCid } = require('ipfs-repo/src/blockstore-utils')
 const itToPull = require('async-iterator-to-pull-stream')
 
 module.exports = function (self) {
-  return () => {
+  return ({ multihash }) => {
     return itToPull((async function * () {
-      for await (const result of self._repo.blocks.query({ keysOnly: true })) {
-        yield dsKeyToRef(result.key)
+      for await (const { key: k } of self._repo.blocks.query({ keysOnly: true })) {
+        try {
+          yield { ref: multihash ? k.toString().substr(1) : keyToCid(k).toString() }
+        } catch (err) {
+          yield { err: `Could not convert block with key '${k.toString()}' to CID: ${err.message}` }
+        }
       }
     })())
   }
 }
-
-function dsKeyToRef (key) {
-  try {
-    // Block key is of the form /<base32 encoded string>
-    const decoder = new base32.Decoder()
-    const buff = Buffer.from(decoder.write(key.toString().slice(1)).finalize())
-    return { ref: new CID(buff).toString() }
-  } catch (err) {
-    return { err: `Could not convert block with key '${key}' to CID: ${err.message}` }
-  }
-}

src/core/components/files-regular/refs-local-readable-stream.js

@@ -3,7 +3,7 @@
 const toStream = require('pull-stream-to-stream')
 
 module.exports = function (self) {
-  return (ipfsPath, options) => {
-    return toStream.source(self.refs.localPullStream())
+  return (options) => {
+    return toStream.source(self.refs.localPullStream(options))
   }
 }

src/core/components/files-regular/refs-local.js

@@ -4,9 +4,16 @@ const promisify = require('promisify-es6')
 const pull = require('pull-stream')
 
 module.exports = function (self) {
-  return promisify((callback) => {
+  return promisify((options, callback) => {
+    if (typeof options === 'function') {
+      callback = options
+      options = {}
+    }
+
+    options = options || {}
+
     pull(
-      self.refs.localPullStream(),
+      self.refs.localPullStream(options),
       pull.collect((err, values) => {
         if (err) {
           return callback(err)

src/http/api/resources/files-regular.js

@@ -37,7 +37,7 @@ exports.parseKey = (request, h) => {
   const { arg } = request.query
 
   if (!arg) {
-    throw Boom.badRequest("Argument 'key' is required")
+    throw Boom.badRequest('Argument \'key\' is required')
   }
 
   const isArray = Array.isArray(arg)
@@ -235,7 +235,7 @@ exports.add = {
     )
       .then(() => {
         if (!filesParsed) {
-          throw new Error("File argument 'data' is required.")
+          throw new Error('File argument \'data\' is required.')
         }
       })
       .catch(err => {
@@ -342,10 +342,19 @@ exports.refs = {
 }
 
 exports.refs.local = {
+  validate: {
+    query: Joi.object().keys({
+      multihash: Joi.boolean().default(false),
+    }).unknown()
+  },
+
   // main route handler
   handler (request, h) {
     const { ipfs } = request.server.app
-    const source = ipfs.refs.localPullStream()
+
+    const multihash = request.query.multihash
+
+    const source = ipfs.refs.localPullStream({ multihash })
     return sendRefsReplyStream(request, h, 'local refs', source)
   }
 }

test/cli/refs-local.js

@@ -18,7 +18,7 @@ describe('refs-local', () => runOnAndOff((thing) => {
     const out = await ipfs('refs-local')
     const lines = out.split('\n')
 
-    expect(lines.includes('QmPkWYfSLCEBLZu7BZt4kigGDMe3cpogMbeVf97gN2xJDN')).to.eql(true)
-    expect(lines.includes('QmUhUuiTKkkK8J6JZ9zmj8iNHPuNfGYcszgRumzhHBxEEU')).to.eql(true)
+    expect(lines.includes('bafkreicjl7v3vyyv4zlryihez5xhunqmriry6styhil7z5lhd3r4prnz6y')).to.eql(true)
+    expect(lines.includes('bafkreidj5bovvm25wszvajfshj7m7m2efpswcs6dsz7giz52ovlquxc4o4')).to.eql(true)
   })
 }))

test/fixtures/go-ipfs-repo/version

@@ -1 +1 @@
-7
+8