feat: support loading arbitrary ipld formats in the http client (#3073)

Adds support for handling more than just `dag-pb`, `dag-cbor` and
`raw` codecs in the client.

Also adds docs on how to configure the IPFS node you are sending
the data to as it needs to know how to handle the formats too.

Author: achingbrain

README.md

@@ -58,6 +58,7 @@
     - [URL source](#url-source)
       - [`urlSource(url)`](#urlsourceurl)
       - [Example](#example-1)
+  - [IPLD Formats](#ipld-formats)
   - [Running the daemon with the right port](#running-the-daemon-with-the-right-port)
   - [Importing the module and usage](#importing-the-module-and-usage)
   - [Importing a sub-module and usage](#importing-a-sub-module-and-usage)
@@ -95,6 +96,8 @@ All core API methods take _additional_ `options` specific to the HTTP API:
 
 * `headers` - An object or [Headers](https://developer.mozilla.org/en-US/docs/Web/API/Headers) instance that can be used to set custom HTTP headers. Note that this option can also be [configured globally](#custom-headers) via the constructor options.
 * `searchParams` - An object or [`URLSearchParams`](https://developer.mozilla.org/en-US/docs/Web/API/URLSearchParams) instance that can be used to add additional query parameters to the query string sent with each request.
+* `ipld.formats` - An array of additional [IPLD formats](https://github.com/ipld/interface-ipld-format) to support
+* `ipld.loadFormat` an async function that takes the name of an [IPLD format](https://github.com/ipld/interface-ipld-format) as a string and should return the implementation of that codec
 
 ### Instance Utils
 
@@ -191,6 +194,53 @@ for await (const file of ipfs.add(urlSource('https://ipfs.io/images/ipfs-logo.sv
 */
 ```
 
+### IPLD Formats
+
+By default an instance of the client supports the following [IPLD formats](https://github.com/ipld/interface-ipld-format), which are enough to do all core IPFS operations:
+
+  * [dag-pb](https://github.com/ipld/specs/blob/master/block-layer/codecs/dag-pb.md)
+  * [dag-cbor](https://github.com/ipld/specs/blob/master/block-layer/codecs/dag-cbor.md)
+  * [raw](https://github.com/ipld/specs/issues/223)
+
+If your application requires support for extra codecs, you can configure them as follows:
+
+1. Configure the [IPLD layer](https://github.com/ipfs/js-ipfs/blob/master/packages/ipfs/docs/MODULE.md#optionsipld) of your IPFS daemon to support the codec. This step is necessary so the node knows how to prepare data received over HTTP to be passed to IPLD for serialization:
+    ```javascript
+    const ipfs = require('ipfs')
+
+    const node = await ipfs({
+      ipld: {
+        // either specify them as part of the `formats` list
+        formats: [
+          require('my-format')
+        ],
+
+        // or supply a function to load them dynamically
+        loadFormat: async (format) => {
+          return require(format)
+        }
+      }
+    })
+2. Configure your IPFS HTTP API Client to support the codec. This is necessary so that the client can send the data to the IPFS node over HTTP:
+    ```javascript
+    const ipfsHttpClient = require('ipfs-http-client')
+
+    const client = ipfsHttpClient({
+      url: 'http://127.0.0.1:5002',
+      ipld: {
+        // either specify them as part of the `formats` list
+        formats: [
+          require('my-format')
+        ],
+
+        // or supply a function to load them dynamically
+        loadFormat: async (format) => {
+          return require(format)
+        }
+      }
+    })
+    ```
+
 ### Running the daemon with the right port
 
 To interact with the API, you need to have a local daemon running. It needs to be open on the right port. `5001` is the default, and is used in the examples below, but it can be set to whatever you need.

src/dag/put.js

@@ -1,15 +1,30 @@
 'use strict'
 
 const dagCBOR = require('ipld-dag-cbor')
+const dagPB = require('ipld-dag-pb')
+const ipldRaw = require('ipld-raw')
 const CID = require('cids')
 const multihash = require('multihashes')
 const configure = require('../lib/configure')
 const multipartRequest = require('../lib/multipart-request')
 const toUrlSearchParams = require('../lib/to-url-search-params')
 const anySignal = require('any-signal')
 const AbortController = require('abort-controller')
+const multicodec = require('multicodec')
+
+module.exports = configure((api, opts) => {
+  const formats = {
+    [multicodec.DAG_PB]: dagPB,
+    [multicodec.DAG_CBOR]: dagCBOR,
+    [multicodec.RAW]: ipldRaw
+  }
+
+  const ipldOptions = (opts && opts.ipld) || {}
+  const configuredFormats = (ipldOptions && ipldOptions.formats) || []
+  configuredFormats.forEach(format => {
+    formats[format.codec] = format
+  })
 
-module.exports = configure(api => {
   return async (dagNode, options = {}) => {
     if (options.cid && (options.format || options.hashAlg)) {
       throw new Error('Failed to put DAG node. Provide either `cid` OR `format` and `hashAlg` options')
@@ -34,17 +49,25 @@ module.exports = configure(api => {
       ...options
     }
 
-    let serialized
+    const number = multicodec.getNumber(options.format)
+    let format = formats[number]
+
+    if (!format) {
+      if (opts && opts.ipld && opts.ipld.loadFormat) {
+        format = await opts.ipld.loadFormat(options.format)
+      }
 
-    if (options.format === 'dag-cbor') {
-      serialized = dagCBOR.util.serialize(dagNode)
-    } else if (options.format === 'dag-pb') {
-      serialized = dagNode.serialize()
-    } else {
-      // FIXME Hopefully already serialized...can we use IPLD to serialise instead?
-      serialized = dagNode
+      if (!format) {
+        throw new Error('Format unsupported - please add support using the options.ipld.formats or options.ipld.loadFormat options')
+      }
+    }
+
+    if (!format.util || !format.util.serialize) {
+      throw new Error('Format does not support utils.serialize function')
     }
 
+    const serialized = format.util.serialize(dagNode)
+
     // allow aborting requests on body errors
     const controller = new AbortController()
     const signal = anySignal([controller.signal, options.signal])

test/dag.spec.js

@@ -8,6 +8,7 @@ const { expect } = require('interface-ipfs-core/src/utils/mocha')
 const { DAGNode } = require('ipld-dag-pb')
 const CID = require('cids')
 const f = require('./utils/factory')()
+const ipfsHttpClient = require('../src')
 
 let ipfs
 
@@ -27,7 +28,6 @@ describe('.dag', function () {
     cid = cid.toV0()
     expect(cid.codec).to.equal('dag-pb')
     cid = cid.toBaseEncodedString('base58btc')
-    // expect(cid).to.equal('bafybeig3t3eugdchignsgkou3ly2mmy4ic4gtfor7inftnqn3yq4ws3a5u')
     expect(cid).to.equal('Qmd7xRhW5f29QuBFtqu3oSD27iVy35NRB91XFjmKFhtgMr')
 
     const result = await ipfs.dag.get(cid)
@@ -48,11 +48,54 @@ describe('.dag', function () {
     expect(result.value).to.deep.equal(cbor)
   })
 
+  it('should be able to put and get a DAG node with format raw', async () => {
+    const node = Buffer.from('some data')
+    let cid = await ipfs.dag.put(node, { format: 'raw', hashAlg: 'sha2-256' })
+
+    expect(cid.codec).to.equal('raw')
+    cid = cid.toBaseEncodedString('base32')
+    expect(cid).to.equal('bafkreiata6mq425fzikf5m26temcvg7mizjrxrkn35swuybmpah2ajan5y')
+
+    const result = await ipfs.dag.get(cid)
+
+    expect(result.value).to.deep.equal(node)
+  })
+
   it('should error when missing DAG resolver for multicodec from requested CID', async () => {
     const block = await ipfs.block.put(Buffer.from([0, 1, 2, 3]), {
       cid: new CID('z8mWaJ1dZ9fH5EetPuRsj8jj26pXsgpsr')
     })
 
     await expect(ipfs.dag.get(block.cid)).to.be.rejectedWith('Missing IPLD format "git-raw"')
   })
+
+  it('should error when putting node with esoteric format', () => {
+    const node = Buffer.from('some data')
+
+    return expect(ipfs.dag.put(node, { format: 'git-raw', hashAlg: 'sha2-256' })).to.eventually.be.rejectedWith(/Format unsupported/)
+  })
+
+  it('should attempt to load an unsupported format', async () => {
+    let askedToLoadFormat
+    const ipfs2 = ipfsHttpClient({
+      url: `http://${ipfs.apiHost}:${ipfs.apiPort}`,
+      ipld: {
+        loadFormat: (format) => {
+          askedToLoadFormat = format === 'git-raw'
+          return {
+            util: {
+              serialize: (buf) => buf
+            }
+          }
+        }
+      }
+    })
+
+    const node = Buffer.from('some data')
+
+    // error is from go-ipfs, this means the client serialized it ok
+    await expect(ipfs2.dag.put(node, { format: 'git-raw', hashAlg: 'sha2-256' })).to.eventually.be.rejectedWith(/no parser for format "git-raw"/)
+
+    expect(askedToLoadFormat).to.be.true()
+  })
 })