Skip to content

Commit 95fc373

Browse files
vmxvmx
committed
WIP: all in using js-multiformats for everying
 - One test suite is commented out - bignumber.js was added as dependency to work around a type bug - some types should work, but don't, hence there are too many `// @ts-ignores` - webworker tests are broken, due to a bug in js-multiformats
1 parent 3e0b91f commit 95fc373

38 files changed

+794
-707
lines changed

‎packages/ipfs-unixfs-exporter/package.json

+4-6
Original file line numberDiff line numberDiff line change
@@ -43,15 +43,10 @@
4343
"events": "^3.3.0",
4444
"ipfs-core-types": "^0.3.1",
4545
"ipfs-unixfs-importer": "^7.0.1",
46-
"ipld": "^0.29.0",
47-
"ipld-block": "^0.11.1",
48-
"ipld-dag-pb": "^0.22.1",
49-
"ipld-in-memory": "^8.0.0",
5046
"it-all": "^1.0.5",
5147
"it-buffer-stream": "^2.0.0",
5248
"it-first": "^1.0.6",
5349
"merge-options": "^3.0.4",
54-
"multicodec": "^3.0.1",
5550
"native-abort-controller": "^1.0.3",
5651
"nyc": "^15.0.0",
5752
"readable-stream": "^3.6.0",
@@ -61,11 +56,14 @@
6156
"util": "^0.12.3"
6257
},
6358
"dependencies": {
64-
"cids": "^1.1.5",
59+
"@ipld/dag-cbor": "^3.0.0",
60+
"@ipld/dag-pb": "^0.0.1",
6561
"err-code": "^3.0.1",
6662
"hamt-sharding": "^2.0.0",
6763
"ipfs-unixfs": "^4.0.1",
6864
"it-last": "^1.0.5",
65+
"multicodec": "^3.0.1",
66+
"multiformats": "^4.6.1",
6967
"multihashing-async": "^2.1.0"
7068
},
7169
"types": "dist/src/index.d.ts",

‎packages/ipfs-unixfs-exporter/src/index.js

+20-16
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,13 @@
11
'use strict'
22

33
const errCode = require('err-code')
4-
const CID = require('cids')
4+
const CID = require('multiformats/cid')
55
const resolve = require('./resolvers')
66
const last = require('it-last')
77

88
/**
99
* @typedef {import('ipfs-unixfs').UnixFS} UnixFS
10-
* @typedef {import('ipld-dag-pb').DAGNode} DAGNode
11-
* @typedef {import('ipld')} IPLD
10+
* @typedef {import('ipfs-unixfs-importer/src/types').BlockAPI} BlockAPI
1211
* @typedef {import('./types').ExporterOptions} ExporterOptions
1312
* @typedef {import('./types').UnixFSFile} UnixFSFile
1413
* @typedef {import('./types').UnixFSDirectory} UnixFSDirectory
@@ -31,15 +30,19 @@ const toPathComponents = (path = '') => {
3130
*/
3231
const cidAndRest = (path) => {
3332
if (path instanceof Uint8Array) {
33+
console.log('vmx: index: path:', path)
3434
return {
35-
cid: new CID(path),
35+
// @ts-ignore
36+
cid: CID.decode(path),
3637
toResolve: []
3738
}
3839
}
3940

40-
if (CID.isCID(path)) {
41+
// @ts-ignore
42+
const cid = CID.asCID(path)
43+
if (cid) {
4144
return {
42-
cid: path,
45+
cid,
4346
toResolve: []
4447
}
4548
}
@@ -52,7 +55,8 @@ const cidAndRest = (path) => {
5255
const output = toPathComponents(path)
5356

5457
return {
55-
cid: new CID(output[0]),
58+
// @ts-ignore
59+
cid: CID.parse(output[0]),
5660
toResolve: output.slice(1)
5761
}
5862
}
@@ -62,10 +66,10 @@ const cidAndRest = (path) => {
6266

6367
/**
6468
* @param {string | CID} path
65-
* @param {IPLD} ipld
69+
* @param {BlockAPI} blockService
6670
* @param {ExporterOptions} [options]
6771
*/
68-
async function * walkPath (path, ipld, options = {}) {
72+
async function * walkPath (path, blockService, options = {}) {
6973
let {
7074
cid,
7175
toResolve
@@ -75,7 +79,7 @@ async function * walkPath (path, ipld, options = {}) {
7579
const startingDepth = toResolve.length
7680

7781
while (true) {
78-
const result = await resolve(cid, name, entryPath, toResolve, startingDepth, ipld, options)
82+
const result = await resolve(cid, name, entryPath, toResolve, startingDepth, blockService, options)
7983

8084
if (!result.entry && !result.next) {
8185
throw errCode(new Error(`Could not resolve ${path}`), 'ERR_NOT_FOUND')
@@ -99,11 +103,11 @@ async function * walkPath (path, ipld, options = {}) {
99103

100104
/**
101105
* @param {string | CID} path
102-
* @param {IPLD} ipld
106+
* @param {BlockAPI} blockService
103107
* @param {ExporterOptions} [options]
104108
*/
105-
async function exporter (path, ipld, options = {}) {
106-
const result = await last(walkPath(path, ipld, options))
109+
async function exporter (path, blockService, options = {}) {
110+
const result = await last(walkPath(path, blockService, options))
107111

108112
if (!result) {
109113
throw errCode(new Error(`Could not resolve ${path}`), 'ERR_NOT_FOUND')
@@ -114,11 +118,11 @@ async function exporter (path, ipld, options = {}) {
114118

115119
/**
116120
* @param {string | CID} path
117-
* @param {IPLD} ipld
121+
* @param {BlockAPI} blockService
118122
* @param {ExporterOptions} [options]
119123
*/
120-
async function * recursive (path, ipld, options = {}) {
121-
const node = await exporter(path, ipld, options)
124+
async function * recursive (path, blockService, options = {}) {
125+
const node = await exporter(path, blockService, options)
122126

123127
if (!node) {
124128
return

‎packages/ipfs-unixfs-exporter/src/resolvers/dag-cbor.js

+12-8
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,9 @@
11
'use strict'
22

3-
const CID = require('cids')
3+
const CID = require('multiformats/cid')
44
const errCode = require('err-code')
5+
// @ts-ignore
6+
const dagCbor = require('@ipld/dag-cbor')
57

68
/**
79
* @typedef {import('../types').Resolver} Resolver
@@ -10,9 +12,9 @@ const errCode = require('err-code')
1012
/**
1113
* @type {Resolver}
1214
*/
13-
const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options) => {
14-
const object = await ipld.get(cid, options)
15-
const block = await ipld.get(new CID(1, 'raw', cid.multihash))
15+
const resolve = async (cid, name, path, toResolve, resolve, depth, blockService, options) => {
16+
const block = await blockService.get(cid)
17+
const object = dagCbor.decode(block.bytes)
1618
let subObject = object
1719
let subPath = path
1820

@@ -24,22 +26,24 @@ const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options
2426
toResolve.shift()
2527
subPath = `${subPath}/${prop}`
2628

27-
if (CID.isCID(subObject[prop])) {
29+
// @ts-ignore
30+
const subObjectCid = CID.asCID(subObject[prop])
31+
if (subObjectCid) {
2832
return {
2933
entry: {
3034
type: 'object',
3135
name,
3236
path,
3337
cid,
34-
node: block,
38+
node: block.bytes,
3539
depth,
3640
size: block.length,
3741
content: async function * () {
3842
yield object
3943
}
4044
},
4145
next: {
42-
cid: subObject[prop],
46+
cid: subObjectCid,
4347
name: prop,
4448
path: subPath,
4549
toResolve
@@ -60,7 +64,7 @@ const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options
6064
name,
6165
path,
6266
cid,
63-
node: block,
67+
node: block.bytes,
6468
depth,
6569
size: block.length,
6670
content: async function * () {

‎packages/ipfs-unixfs-exporter/src/resolvers/identity.js

+6-5
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33
const errCode = require('err-code')
44
const extractDataFromBlock = require('../utils/extract-data-from-block')
55
const validateOffsetAndLength = require('../utils/validate-offset-and-length')
6-
const mh = require('multihashing-async').multihash
6+
const mh = require('multiformats/hashes/digest')
77

88
/**
99
* @typedef {import('../types').ExporterOptions} ExporterOptions
@@ -32,22 +32,23 @@ const rawContent = (node) => {
3232
/**
3333
* @type {Resolver}
3434
*/
35-
const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options) => {
35+
const resolve = async (cid, name, path, toResolve, resolve, depth, blockService, options) => {
3636
if (toResolve.length) {
3737
throw errCode(new Error(`No link named ${path} found in raw node ${cid.toString()}`), 'ERR_NOT_FOUND')
3838
}
39-
40-
const buf = await mh.decode(cid.multihash)
39+
// @ts-ignore
40+
const buf = await mh.decode(cid.multihash.bytes)
4141

4242
return {
4343
entry: {
4444
type: 'identity',
4545
name,
4646
path,
47+
// @ts-ignore
4748
cid,
4849
content: rawContent(buf.digest),
4950
depth,
50-
size: buf.length,
51+
size: buf.digest.length,
5152
node: buf.digest
5253
}
5354
}

‎packages/ipfs-unixfs-exporter/src/resolvers/index.js

+11-10
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
'use strict'
22

33
const errCode = require('err-code')
4+
const multicodec = require('multicodec')
45

56
/**
6-
* @typedef {import('cids')} CID
7-
* @typedef {import('ipld')} IPLD
7+
* @typedef {import('../').BlockAPI} BlockAPI
88
* @typedef {import('../types').ExporterOptions} ExporterOptions
99
* @typedef {import('../types').UnixFSEntry} UnixFSEntry
1010
* @typedef {import('../types').Resolver} Resolver
@@ -15,23 +15,24 @@ const errCode = require('err-code')
1515
* @type {{ [ key: string ]: Resolver }}
1616
*/
1717
const resolvers = {
18-
'dag-pb': require('./unixfs-v1'),
19-
raw: require('./raw'),
20-
'dag-cbor': require('./dag-cbor'),
21-
identity: require('./identity')
18+
[multicodec.DAG_PB]: require('./unixfs-v1'),
19+
[multicodec.RAW]: require('./raw'),
20+
[multicodec.DAG_CBOR]: require('./dag-cbor'),
21+
[multicodec.IDENTITY]: require('./identity')
2222
}
2323

2424
/**
2525
* @type {Resolve}
2626
*/
27-
function resolve (cid, name, path, toResolve, depth, ipld, options) {
28-
const resolver = resolvers[cid.codec]
27+
function resolve (cid, name, path, toResolve, depth, blockService, options) {
28+
const resolver = resolvers[cid.code]
2929

3030
if (!resolver) {
31-
throw errCode(new Error(`No resolver for codec ${cid.codec}`), 'ERR_NO_RESOLVER')
31+
// @ts-ignore - TODO vmx 2021-03-05: Add proper typing
32+
throw errCode(new Error(`No resolver for codec ${multicodec.getName(cid.code)}`), 'ERR_NO_RESOLVER')
3233
}
3334

34-
return resolver(cid, name, path, toResolve, resolve, depth, ipld, options)
35+
return resolver(cid, name, path, toResolve, resolve, depth, blockService, options)
3536
}
3637

3738
module.exports = resolve

‎packages/ipfs-unixfs-exporter/src/resolvers/raw.js

+8-5
Original file line numberDiff line numberDiff line change
@@ -30,23 +30,26 @@ const rawContent = (node) => {
3030
/**
3131
* @type {import('../types').Resolver}
3232
*/
33-
const resolve = async (cid, name, path, toResolve, resolve, depth, ipld, options) => {
33+
const resolve = async (cid, name, path, toResolve, resolve, depth, blockService, options) => {
3434
if (toResolve.length) {
3535
throw errCode(new Error(`No link named ${path} found in raw node ${cid.toString()}`), 'ERR_NOT_FOUND')
3636
}
3737

38-
const buf = await ipld.get(cid, options)
38+
const block = await blockService.get(cid, options)
3939

4040
return {
4141
entry: {
4242
type: 'raw',
4343
name,
4444
path,
45+
// @ts-ignore
4546
cid,
46-
content: rawContent(buf),
47+
content: rawContent(block.bytes),
4748
depth,
48-
size: buf.length,
49-
node: buf
49+
// TODO vmx 2021-03-24: Check if `size` is needed
50+
size: 0,
51+
// @ts-ignore
52+
node: block.bytes
5053
}
5154
}
5255
}

‎packages/ipfs-unixfs-exporter/src/resolvers/unixfs-v1/content/directory.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -9,7 +9,7 @@
99
/**
1010
* @type {UnixfsV1Resolver}
1111
*/
12-
const directoryContent = (cid, node, unixfs, path, resolve, depth, ipld) => {
12+
const directoryContent = (cid, node, unixfs, path, resolve, depth, blockService) => {
1313
/**
1414
* @param {ExporterOptions} [options]
1515
* @returns {UnixfsV1DirectoryContent}
@@ -20,7 +20,7 @@ const directoryContent = (cid, node, unixfs, path, resolve, depth, ipld) => {
2020
const links = node.Links.slice(offset, length)
2121

2222
for (const link of links) {
23-
const result = await resolve(link.Hash, link.Name, `${path}/${link.Name}`, [], depth + 1, ipld, options)
23+
const result = await resolve(link.Hash, link.Name, `${path}/${link.Name}`, [], depth + 1, blockService, options)
2424

2525
if (result.entry) {
2626
yield result.entry

0 commit comments

Comments
 (0)