diff --git a/src/core/components/index.js b/src/core/components/index.js index e664e55c55..1abb7f4499 100644 --- a/src/core/components/index.js +++ b/src/core/components/index.js @@ -9,6 +9,20 @@ exports.bitswap = { exports.config = require('./config') exports.id = require('./id') exports.init = require('./init') +exports.object = { + data: require('./object/data'), + get: require('./object/get'), + links: require('./object/links'), + new: require('./object/new'), + patch: { + addLink: require('./object/patch/add-link'), + appendData: require('./object/patch/append-data'), + rmLink: require('./object/patch/rm-link'), + setData: require('./object/patch/set-data') + }, + put: require('./object/put'), + stat: require('./object/stat') +} exports.ping = require('./ping') exports.start = require('./start') exports.stop = require('./stop') @@ -17,6 +31,5 @@ exports.version = require('./version') exports.legacy = { // TODO: these will be removed as the new API is completed dag: require('./dag'), libp2p: require('./libp2p'), - object: require('./object'), pin: require('./pin') } diff --git a/src/core/components/init.js b/src/core/components/init.js index 642b9e63db..e5b6788f63 100644 --- a/src/core/components/init.js +++ b/src/core/components/init.js @@ -102,7 +102,20 @@ module.exports = ({ }) const dag = Commands.legacy.dag({ _ipld: ipld, _preload: preload }) - const object = Commands.legacy.object({ _ipld: ipld, _preload: preload, dag, _gcLock: gcLock }) + const object = { + data: Commands.object.data({ ipld, preload }), + get: Commands.object.get({ ipld, preload }), + links: Commands.object.links({ dag }), + new: Commands.object.new({ ipld, preload }), + patch: { + addLink: Commands.object.patch.addLink({ ipld, gcLock, preload }), + appendData: Commands.object.patch.appendData({ ipld, gcLock, preload }), + rmLink: Commands.object.patch.rmLink({ ipld, gcLock, preload }), + setData: Commands.object.patch.setData({ ipld, gcLock, preload }) + }, + put: Commands.object.put({ ipld, gcLock, preload }), + stat: Commands.object.stat({ ipld, preload }) + } const pinManager = new PinManager(repo, dag) await pinManager.load() @@ -134,6 +147,7 @@ module.exports = ({ initOptions: options, ipld, keychain, + object, peerInfo, pinManager, preload, @@ -276,6 +290,7 @@ function createApi ({ initOptions, ipld, keychain, + object, peerInfo, pinManager, preload, @@ -302,6 +317,7 @@ function createApi ({ config: Commands.config({ repo }), id: Commands.id({ peerInfo }), init: () => { throw new AlreadyInitializedError() }, + object, start, version: Commands.version({ repo }) } diff --git a/src/core/components/object.js b/src/core/components/object.js deleted file mode 100644 index 1f7e3f7cbe..0000000000 --- a/src/core/components/object.js +++ /dev/null @@ -1,302 +0,0 @@ -'use strict' - -const callbackify = require('callbackify') -const dagPB = require('ipld-dag-pb') -const DAGNode = dagPB.DAGNode -const DAGLink = dagPB.DAGLink -const CID = require('cids') -const mh = require('multihashes') -const multicodec = require('multicodec') -const Unixfs = require('ipfs-unixfs') -const errCode = require('err-code') - -function normalizeMultihash (multihash, enc) { - if (typeof multihash === 'string') { - if (enc === 'base58' || !enc) { - return multihash - } - - return Buffer.from(multihash, enc) - } else if (Buffer.isBuffer(multihash)) { - return multihash - } else if (CID.isCID(multihash)) { - return multihash.buffer - } else { - throw new Error('unsupported multihash') - } -} - -function parseBuffer (buf, encoding) { - switch (encoding) { - case 'json': - return parseJSONBuffer(buf) - case 'protobuf': - return parseProtoBuffer(buf) - default: - throw new Error(`unkown encoding: ${encoding}`) - } -} - -function parseJSONBuffer (buf) { - let data - let links - - try { - const parsed = JSON.parse(buf.toString()) - - links = (parsed.Links || []).map((link) => { - return new DAGLink( - link.Name || link.name, - link.Size || link.size, - mh.fromB58String(link.Hash || link.hash || link.multihash) - ) - }) - data = Buffer.from(parsed.Data) - } catch (err) { - throw new Error('failed to parse JSON: ' + err) - } - - return new DAGNode(data, links) -} - -function parseProtoBuffer (buf) { - return dagPB.util.deserialize(buf) -} - -function findLinks (node, links = []) { - for (const key in node) { - const val = node[key] - - if (key === '/' && Object.keys(node).length === 1) { - try { - links.push(new DAGLink('', 0, new CID(val))) - continue - } catch (_) { - // not a CID - } - } - - if (CID.isCID(val)) { - links.push(new DAGLink('', 0, val)) - - continue - } - - if (Array.isArray(val)) { - findLinks(val, links) - } - - if (typeof val === 'object' && !(val instanceof String)) { - findLinks(val, links) - } - } - - return links -} - -module.exports = function object (self) { - async function editAndSave (multihash, edit, options) { - options = options || {} - - const node = await self.object.get(multihash, options) - - // edit applies the edit func passed to - // editAndSave - const cid = await self._ipld.put(edit(node), multicodec.DAG_PB, { - cidVersion: 0, - hashAlg: multicodec.SHA2_256 - }) - - if (options.preload !== false) { - self._preload(cid) - } - - return cid - } - - return { - new: callbackify.variadic(async (template, options) => { - options = options || {} - - // allow options in the template position - if (template && typeof template !== 'string') { - options = template - template = null - } - - let data - - if (template) { - if (template === 'unixfs-dir') { - data = (new Unixfs('directory')).marshal() - } else { - throw new Error('unknown template') - } - } else { - data = Buffer.alloc(0) - } - - const node = new DAGNode(data) - - const cid = await self._ipld.put(node, multicodec.DAG_PB, { - cidVersion: 0, - hashAlg: multicodec.SHA2_256 - }) - - if (options.preload !== false) { - self._preload(cid) - } - - return cid - }), - put: callbackify.variadic(async (obj, options) => { - options = options || {} - - const encoding = options.enc - let node - - if (Buffer.isBuffer(obj)) { - if (encoding) { - node = await parseBuffer(obj, encoding) - } else { - node = new DAGNode(obj) - } - } else if (DAGNode.isDAGNode(obj)) { - // already a dag node - node = obj - } else if (typeof obj === 'object') { - node = new DAGNode(obj.Data, obj.Links) - } else { - throw new Error('obj not recognized') - } - - const release = await self._gcLock.readLock() - - try { - const cid = await self._ipld.put(node, multicodec.DAG_PB, { - cidVersion: 0, - hashAlg: multicodec.SHA2_256 - }) - - if (options.preload !== false) { - self._preload(cid) - } - - return cid - } finally { - release() - } - }), - - get: callbackify.variadic(async (multihash, options) => { // eslint-disable-line require-await - options = options || {} - - let mh, cid - - try { - mh = normalizeMultihash(multihash, options.enc) - } catch (err) { - throw errCode(err, 'ERR_INVALID_MULTIHASH') - } - - try { - cid = new CID(mh) - } catch (err) { - throw errCode(err, 'ERR_INVALID_CID') - } - - if (options.cidVersion === 1) { - cid = cid.toV1() - } - - if (options.preload !== false) { - self._preload(cid) - } - - return self._ipld.get(cid) - }), - - data: callbackify.variadic(async (multihash, options) => { - options = options || {} - - const node = await self.object.get(multihash, options) - - return node.Data - }), - - links: callbackify.variadic(async (multihash, options) => { - options = options || {} - - const cid = new CID(multihash) - const result = await self.dag.get(cid, options) - - if (cid.codec === 'raw') { - return [] - } - - if (cid.codec === 'dag-pb') { - return result.value.Links - } - - if (cid.codec === 'dag-cbor') { - return findLinks(result) - } - - throw new Error(`Cannot resolve links from codec ${cid.codec}`) - }), - - stat: callbackify.variadic(async (multihash, options) => { - options = options || {} - - const node = await self.object.get(multihash, options) - const serialized = dagPB.util.serialize(node) - const cid = await dagPB.util.cid(serialized, { - cidVersion: 0 - }) - - const blockSize = serialized.length - const linkLength = node.Links.reduce((a, l) => a + l.Tsize, 0) - - return { - Hash: cid.toBaseEncodedString(), - NumLinks: node.Links.length, - BlockSize: blockSize, - LinksSize: blockSize - node.Data.length, - DataSize: node.Data.length, - CumulativeSize: blockSize + linkLength - } - }), - - patch: { - addLink: callbackify.variadic(async (multihash, link, options) => { // eslint-disable-line require-await - return editAndSave(multihash, (node) => { - node.addLink(link) - - return node - }, options) - }), - - rmLink: callbackify.variadic(async (multihash, linkRef, options) => { // eslint-disable-line require-await - return editAndSave(multihash, (node) => { - node.rmLink(linkRef.Name || linkRef.name) - - return node - }, options) - }), - - appendData: callbackify.variadic(async (multihash, data, options) => { // eslint-disable-line require-await - return editAndSave(multihash, (node) => { - const newData = Buffer.concat([node.Data, data]) - - return new DAGNode(newData, node.Links) - }, options) - }), - - setData: callbackify.variadic(async (multihash, data, options) => { // eslint-disable-line require-await - return editAndSave(multihash, (node) => { - return new DAGNode(data, node.Links) - }, options) - }) - } - } -} diff --git a/src/core/components/object/data.js b/src/core/components/object/data.js new file mode 100644 index 0000000000..e7066f3d74 --- /dev/null +++ b/src/core/components/object/data.js @@ -0,0 +1,9 @@ +'use strict' + +module.exports = ({ ipld, preload }) => { + const get = require('./get')({ ipld, preload }) + return async function data (multihash, options) { + const node = await get(multihash, options) + return node.Data + } +} diff --git a/src/core/components/object/get.js b/src/core/components/object/get.js new file mode 100644 index 0000000000..394235cc6c --- /dev/null +++ b/src/core/components/object/get.js @@ -0,0 +1,48 @@ +'use strict' + +const CID = require('cids') +const errCode = require('err-code') + +function normalizeMultihash (multihash, enc) { + if (typeof multihash === 'string') { + if (enc === 'base58' || !enc) { + return multihash + } + return Buffer.from(multihash, enc) + } else if (Buffer.isBuffer(multihash)) { + return multihash + } else if (CID.isCID(multihash)) { + return multihash.buffer + } + throw new Error('unsupported multihash') +} + +module.exports = ({ ipld, preload }) => { + return async function get (multihash, options) { // eslint-disable-line require-await + options = options || {} + + let mh, cid + + try { + mh = normalizeMultihash(multihash, options.enc) + } catch (err) { + throw errCode(err, 'ERR_INVALID_MULTIHASH') + } + + try { + cid = new CID(mh) + } catch (err) { + throw errCode(err, 'ERR_INVALID_CID') + } + + if (options.cidVersion === 1) { + cid = cid.toV1() + } + + if (options.preload !== false) { + preload(cid) + } + + return ipld.get(cid) + } +} diff --git a/src/core/components/object/links.js b/src/core/components/object/links.js new file mode 100644 index 0000000000..8e6a58f177 --- /dev/null +++ b/src/core/components/object/links.js @@ -0,0 +1,58 @@ +'use strict' + +const dagPB = require('ipld-dag-pb') +const DAGLink = dagPB.DAGLink +const CID = require('cids') + +function findLinks (node, links = []) { + for (const key in node) { + const val = node[key] + + if (key === '/' && Object.keys(node).length === 1) { + try { + links.push(new DAGLink('', 0, new CID(val))) + continue + } catch (_) { + // not a CID + } + } + + if (CID.isCID(val)) { + links.push(new DAGLink('', 0, val)) + continue + } + + if (Array.isArray(val)) { + findLinks(val, links) + } + + if (val && typeof val === 'object') { + findLinks(val, links) + } + } + + return links +} + +module.exports = ({ dag }) => { + return async function links (multihash, options) { + options = options || {} + + const cid = new CID(multihash) + const result = await dag.get(cid, options) + + if (cid.codec === 'raw') { + return [] + } + + if (cid.codec === 'dag-pb') { + return result.value.Links + } + + if (cid.codec === 'dag-cbor') { + return findLinks(result) + } + + throw new Error(`Cannot resolve links from codec ${cid.codec}`) + } +} diff --git a/src/core/components/object/new.js b/src/core/components/object/new.js new file mode 100644 index 0000000000..4d6e6291b0 --- /dev/null +++ b/src/core/components/object/new.js @@ -0,0 +1,43 @@ +'use strict' + +const dagPB = require('ipld-dag-pb') +const DAGNode = dagPB.DAGNode +const multicodec = require('multicodec') +const Unixfs = require('ipfs-unixfs') + +module.exports = ({ ipld, preload }) => { + return async function _new (template, options) { + options = options || {} + + // allow options in the template position + if (template && typeof template !== 'string') { + options = template + template = null + } + + let data + + if (template) { + if (template === 'unixfs-dir') { + data = (new Unixfs('directory')).marshal() + } else { + throw new Error('unknown template') + } + } else { + data = Buffer.alloc(0) + } + + const node = new DAGNode(data) + + const cid = await ipld.put(node, multicodec.DAG_PB, { + cidVersion: 0, + hashAlg: multicodec.SHA2_256 + }) + + if (options.preload !== false) { + preload(cid) + } + + return cid + } +} diff --git a/src/core/components/object/patch/add-link.js b/src/core/components/object/patch/add-link.js new file mode 100644 index 0000000000..2cdd990749 --- /dev/null +++ b/src/core/components/object/patch/add-link.js @@ -0,0 +1,12 @@ +'use strict' + +module.exports = ({ ipld, gcLock, preload }) => { + const get = require('../get')({ ipld, preload }) + const put = require('../put')({ ipld, gcLock, preload }) + + return async function addLink (multihash, link, options) { + const node = await get(multihash, options) + node.addLink(link) + return put(node, options) + } +} diff --git a/src/core/components/object/patch/append-data.js b/src/core/components/object/patch/append-data.js new file mode 100644 index 0000000000..511d79feb3 --- /dev/null +++ b/src/core/components/object/patch/append-data.js @@ -0,0 +1,14 @@ +'use strict' + +const { DAGNode } = require('ipld-dag-pb') + +module.exports = ({ ipld, gcLock, preload }) => { + const get = require('../get')({ ipld, preload }) + const put = require('../put')({ ipld, gcLock, preload }) + + return async function appendData (multihash, data, options) { + const node = await get(multihash, options) + const newData = Buffer.concat([node.Data, data]) + return put(new DAGNode(newData, node.Links), options) + } +} diff --git a/src/core/components/object/patch/rm-link.js b/src/core/components/object/patch/rm-link.js new file mode 100644 index 0000000000..bd3033a06b --- /dev/null +++ b/src/core/components/object/patch/rm-link.js @@ -0,0 +1,12 @@ +'use strict' + +module.exports = ({ ipld, gcLock, preload }) => { + const get = require('../get')({ ipld, preload }) + const put = require('../put')({ ipld, gcLock, preload }) + + return async function rmLink (multihash, linkRef, options) { + const node = await get(multihash, options) + node.rmLink(linkRef.Name || linkRef.name) + return put(node, options) + } +} diff --git a/src/core/components/object/patch/set-data.js b/src/core/components/object/patch/set-data.js new file mode 100644 index 0000000000..7693a5b5ba --- /dev/null +++ b/src/core/components/object/patch/set-data.js @@ -0,0 +1,13 @@ +'use strict' + +const { DAGNode } = require('ipld-dag-pb') + +module.exports = ({ ipld, gcLock, preload }) => { + const get = require('../get')({ ipld, preload }) + const put = require('../put')({ ipld, gcLock, preload }) + + return async function setData (multihash, data, options) { + const node = await get(multihash, options) + return put(new DAGNode(data, node.Links), options) + } +} diff --git a/src/core/components/object/put.js b/src/core/components/object/put.js new file mode 100644 index 0000000000..2a8a195f53 --- /dev/null +++ b/src/core/components/object/put.js @@ -0,0 +1,85 @@ +'use strict' + +const dagPB = require('ipld-dag-pb') +const DAGNode = dagPB.DAGNode +const DAGLink = dagPB.DAGLink +const mh = require('multihashes') +const multicodec = require('multicodec') + +function parseBuffer (buf, encoding) { + switch (encoding) { + case 'json': + return parseJSONBuffer(buf) + case 'protobuf': + return parseProtoBuffer(buf) + default: + throw new Error(`unkown encoding: ${encoding}`) + } +} + +function parseJSONBuffer (buf) { + let data + let links + + try { + const parsed = JSON.parse(buf.toString()) + + links = (parsed.Links || []).map((link) => { + return new DAGLink( + link.Name || link.name, + link.Size || link.size, + mh.fromB58String(link.Hash || link.hash || link.multihash) + ) + }) + data = Buffer.from(parsed.Data) + } catch (err) { + throw new Error('failed to parse JSON: ' + err) + } + + return new DAGNode(data, links) +} + +function parseProtoBuffer (buf) { + return dagPB.util.deserialize(buf) +} + +module.exports = ({ ipld, gcLock, preload }) => { + return async function put (obj, options) { + options = options || {} + + const encoding = options.enc + let node + + if (Buffer.isBuffer(obj)) { + if (encoding) { + node = await parseBuffer(obj, encoding) + } else { + node = new DAGNode(obj) + } + } else if (DAGNode.isDAGNode(obj)) { + // already a dag node + node = obj + } else if (typeof obj === 'object') { + node = new DAGNode(obj.Data, obj.Links) + } else { + throw new Error('obj not recognized') + } + + const release = await gcLock.readLock() + + try { + const cid = await ipld.put(node, multicodec.DAG_PB, { + cidVersion: 0, + hashAlg: multicodec.SHA2_256 + }) + + if (options.preload !== false) { + preload(cid) + } + + return cid + } finally { + release() + } + } +} diff --git a/src/core/components/object/stat.js b/src/core/components/object/stat.js new file mode 100644 index 0000000000..ea2f06c72c --- /dev/null +++ b/src/core/components/object/stat.js @@ -0,0 +1,28 @@ +'use strict' + +const dagPB = require('ipld-dag-pb') + +module.exports = ({ ipld, preload }) => { + const get = require('./get')({ ipld, preload }) + return async function stat (multihash, options) { + options = options || {} + + const node = await get(multihash, options) + const serialized = dagPB.util.serialize(node) + const cid = await dagPB.util.cid(serialized, { + cidVersion: 0 + }) + + const blockSize = serialized.length + const linkLength = node.Links.reduce((a, l) => a + l.Tsize, 0) + + return { + Hash: cid.toBaseEncodedString(), + NumLinks: node.Links.length, + BlockSize: blockSize, + LinksSize: blockSize - node.Data.length, + DataSize: node.Data.length, + CumulativeSize: blockSize + linkLength + } + } +} diff --git a/src/core/components/start.js b/src/core/components/start.js index 0fb44f22aa..5234820fa8 100644 --- a/src/core/components/start.js +++ b/src/core/components/start.js @@ -108,7 +108,20 @@ function createApi ({ repo }) { const dag = Commands.legacy.dag({ _ipld: ipld, _preload: preload }) - const object = Commands.legacy.object({ _ipld: ipld, _preload: preload, dag, _gcLock: gcLock }) + const object = { + data: Commands.object.data({ ipld, preload }), + get: Commands.object.get({ ipld, preload }), + links: Commands.object.links({ dag }), + new: Commands.object.new({ ipld, preload }), + patch: { + addLink: Commands.object.patch.addLink({ ipld, gcLock, preload }), + appendData: Commands.object.patch.appendData({ ipld, gcLock, preload }), + rmLink: Commands.object.patch.rmLink({ ipld, gcLock, preload }), + setData: Commands.object.patch.setData({ ipld, gcLock, preload }) + }, + put: Commands.object.put({ ipld, gcLock, preload }), + stat: Commands.object.stat({ ipld, preload }) + } const pin = Commands.legacy.pin({ _ipld: ipld, _preload: preload, object, _repo: repo, _pinManager: pinManager }) const add = Commands.add({ ipld, dag, preload, pin, gcLock, constructorOptions }) @@ -123,6 +136,7 @@ function createApi ({ ipns, keychain, libp2p, + object, peerInfo, preload, print, diff --git a/src/core/components/stop.js b/src/core/components/stop.js index 1e70484005..9be69ec1d6 100644 --- a/src/core/components/stop.js +++ b/src/core/components/stop.js @@ -77,7 +77,20 @@ function createApi ({ repo }) { const dag = Commands.legacy.dag({ _ipld: ipld, _preload: preload }) - const object = Commands.legacy.object({ _ipld: ipld, _preload: preload, dag, _gcLock: gcLock }) + const object = { + data: Commands.object.data({ ipld, preload }), + get: Commands.object.get({ ipld, preload }), + links: Commands.object.links({ dag }), + new: Commands.object.new({ ipld, preload }), + patch: { + addLink: Commands.object.patch.addLink({ ipld, gcLock, preload }), + appendData: Commands.object.patch.appendData({ ipld, gcLock, preload }), + rmLink: Commands.object.patch.rmLink({ ipld, gcLock, preload }), + setData: Commands.object.patch.setData({ ipld, gcLock, preload }) + }, + put: Commands.object.put({ ipld, gcLock, preload }), + stat: Commands.object.stat({ ipld, preload }) + } const pin = Commands.legacy.pin({ _ipld: ipld, _preload: preload, object, _repo: repo, _pinManager: pinManager }) const add = Commands.add({ ipld, dag, preload, pin, gcLock, constructorOptions }) @@ -89,6 +102,7 @@ function createApi ({ initOptions, ipld, keychain, + object, peerInfo, pinManager, preload,