Skip to content

Commit 3f6f22f

Browse files
achingbrainachingbrain
authored
feat: store pins in datastore instead of a DAG (#2771)
Adds a `.pins` datastore to `ipfs-repo` and uses that to store pins as cbor binary keyed by multihash. ### Format As stored in the datastore, each pin has several fields: ```javascript { codec: // optional Number, the codec from the CID that this multihash was pinned with, if omitted, treated as 'dag-pb' version: // optional Number, the version number from the CID that this multihash was pinned with, if omitted, treated as v0 depth: // Number Infinity = recursive pin, 0 = direct, 1+ = pinned to a depth comments: // optional String user-friendly description of the pin metadata: // optional Object, user-defined data for the pin } ``` Notes: `.codec` and `.version` are stored so we can recreate the original CID when listing pins. ### Metadata The intention is for us to be able to add extra fields that have technical meaning to the root of the object, and the user can store application-specific data in the `metadata` field. ### CLI ```console $ ipfs pin add bafyfoo --metadata key1=value1,key2=value2 $ ipfs pin add bafyfoo --metadata-format=json --metadata '{"key1":"value1","key2":"value2"}' $ ipfs pin list bafyfoo $ ipfs pin list -l CID Name Type Metadata bafyfoo My pin Recursive {"key1":"value1","key2":"value2"} $ ipfs pin metadata Qmfoo --format=json {"key1":"value1","key2":"value2"} ``` ### HTTP API * '/api/v0/pin/add' route adds new `metadata` argument, accepts a json string * '/api/v0/pin/metadata' returns metadata as json ### Core API * `ipfs.pin.addAll` accepts and returns an async iterator * `ipfs.pin.rmAll` accepts and returns an async iterator ```javascript // pass a cid or IPFS Path with options const { cid } = await ipfs.pin.add(new CID('/ipfs/Qmfoo'), { recursive: false, metadata: { key: 'value }, timeout: 2000 })) // pass an iterable of CIDs const [{ cid: cid1 }, { cid: cid2 }] = await all(ipfs.pin.addAll([ new CID('/ipfs/Qmfoo'), new CID('/ipfs/Qmbar') ], { timeout: '2s' })) // pass an iterable of objects with options const [{ cid: cid1 }, { cid: cid2 }] = await all(ipfs.pin.addAll([ { cid: new CID('/ipfs/Qmfoo'), recursive: true, comments: 'A recursive pin' }, { cid: new CID('/ipfs/Qmbar'), recursive: false, comments: 'A direct pin' } ], { timeout: '2s' })) ``` * ipfs.pin.rmAll accepts and returns an async generator (other input types are available) ```javascript // pass an IPFS Path or CID const { cid } = await ipfs.rm(new CID('/ipfs/Qmfoo/file.txt')) // pass options const { cid } = await all(ipfs.rm(new CID('/ipfs/Qmfoo'), { recursive: true })) // pass an iterable of CIDs or objects with options const [{ cid }] = await all(ipfs.rmAll([{ cid: new CID('/ipfs/Qmfoo'), recursive: true }])) ``` Bonus: Lets us pipe the output of one command into another: ```javascript await pipe( ipfs.pin.ls({ type: 'recursive' }), (source) => ipfs.pin.rmAll(source) ) // or await all(ipfs.pin.rmAll(ipfs.pin.ls({ type: 'recursive'}))) ``` BREAKING CHANGES: * pins are now stored in a datastore, a repo migration will occur on startup * All deps of this module now use Uint8Arrays in place of node Buffers
1 parent 78dfe51 commit 3f6f22f

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

46 files changed

+230
-146
lines changed

‎package.json

+15-14
Original file line numberDiff line numberDiff line change
@@ -43,38 +43,39 @@
4343
"abort-controller": "^3.0.0",
4444
"any-signal": "^1.1.0",
4545
"bignumber.js": "^9.0.0",
46-
"buffer": "^5.6.0",
47-
"cids": "^0.8.3",
46+
"cids": "^1.0.0",
4847
"debug": "^4.1.0",
4948
"form-data": "^3.0.0",
5049
"ipfs-core-utils": "^0.3.2",
5150
"ipfs-utils": "^3.0.0",
52-
"ipld-block": "^0.9.2",
53-
"ipld-dag-cbor": "^0.16.0",
54-
"ipld-dag-pb": "^0.19.0",
55-
"ipld-raw": "^5.0.0",
51+
"ipld-block": "^0.10.0",
52+
"ipld-dag-cbor": "^0.17.0",
53+
"ipld-dag-pb": "^0.20.0",
54+
"ipld-raw": "^6.0.0",
5655
"iso-url": "^0.4.7",
5756
"it-last": "^1.0.1",
57+
"it-map": "^1.0.2",
5858
"it-tar": "^1.2.2",
5959
"it-to-buffer": "^1.0.0",
6060
"it-to-stream": "^0.1.1",
6161
"merge-options": "^2.0.0",
62-
"multiaddr": "^7.4.3",
63-
"multiaddr-to-uri": "^5.1.0",
64-
"multibase": "^1.0.1",
65-
"multicodec": "^1.0.0",
66-
"multihashes": "^1.0.1",
62+
"multiaddr": "^8.0.0",
63+
"multiaddr-to-uri": "^6.0.0",
64+
"multibase": "^3.0.0",
65+
"multicodec": "^2.0.0",
66+
"multihashes": "^3.0.1",
6767
"nanoid": "^3.0.2",
6868
"node-fetch": "^2.6.0",
6969
"parse-duration": "^0.4.4",
70-
"stream-to-it": "^0.2.1"
70+
"stream-to-it": "^0.2.1",
71+
"uint8arrays": "^1.1.0"
7172
},
7273
"devDependencies": {
73-
"aegir": "^23.0.0",
74+
"aegir": "^26.0.0",
7475
"cross-env": "^7.0.0",
7576
"go-ipfs": "^0.6.0",
7677
"interface-ipfs-core": "^0.139.1",
77-
"ipfsd-ctl": "^5.0.0",
78+
"ipfsd-ctl": "^6.0.0",
7879
"it-all": "^1.0.1",
7980
"it-concat": "^1.0.0",
8081
"it-pipe": "^1.1.0",

‎src/block/get.js

+1-2
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
const Block = require('ipld-block')
44
const CID = require('cids')
5-
const { Buffer } = require('buffer')
65
const configure = require('../lib/configure')
76
const toUrlSearchParams = require('../lib/to-url-search-params')
87

@@ -20,6 +19,6 @@ module.exports = configure(api => {
2019
headers: options.headers
2120
})
2221

23-
return new Block(Buffer.from(await res.arrayBuffer()), cid)
22+
return new Block(new Uint8Array(await res.arrayBuffer()), cid)
2423
}
2524
})

‎src/config/replace.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
'use strict'
22

3-
const { Buffer } = require('buffer')
3+
const uint8ArrayFromString = require('uint8arrays/from-string')
44
const multipartRequest = require('../lib/multipart-request')
55
const configure = require('../lib/configure')
66
const toUrlSearchParams = require('../lib/to-url-search-params')
@@ -18,7 +18,7 @@ module.exports = configure(api => {
1818
signal,
1919
searchParams: toUrlSearchParams(options),
2020
...(
21-
await multipartRequest(Buffer.from(JSON.stringify(config)), controller, options.headers)
21+
await multipartRequest(uint8ArrayFromString(JSON.stringify(config)), controller, options.headers)
2222
)
2323
})
2424

‎src/dag/get.js

+4-4
Original file line numberDiff line numberDiff line change
@@ -18,16 +18,16 @@ module.exports = configure((api, options) => {
1818
return async (cid, options = {}) => {
1919
const resolved = await dagResolve(cid, options)
2020
const block = await getBlock(resolved.cid, options)
21-
const dagResolver = resolvers[block.cid.codec]
21+
const dagResolver = resolvers[resolved.cid.codec]
2222

2323
if (!dagResolver) {
2424
throw Object.assign(
25-
new Error(`Missing IPLD format "${block.cid.codec}"`),
26-
{ missingMulticodec: cid.codec }
25+
new Error(`Missing IPLD format "${resolved.cid.codec}"`),
26+
{ missingMulticodec: resolved.cid.codec }
2727
)
2828
}
2929

30-
if (block.cid.codec === 'raw' && !resolved.remPath) {
30+
if (resolved.cid.codec === 'raw' && !resolved.remPath) {
3131
resolved.remainderPath = '/'
3232
}
3333

‎src/dht/find-peer.js

+1-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
'use strict'
22

3-
const { Buffer } = require('buffer')
43
const CID = require('cids')
54
const multiaddr = require('multiaddr')
65
const configure = require('../lib/configure')
@@ -13,7 +12,7 @@ module.exports = configure(api => {
1312
timeout: options.timeout,
1413
signal: options.signal,
1514
searchParams: toUrlSearchParams({
16-
arg: `${Buffer.isBuffer(peerId) ? new CID(peerId) : peerId}`,
15+
arg: `${peerId instanceof Uint8Array ? new CID(peerId) : peerId}`,
1716
...options
1817
}),
1918
headers: options.headers

‎src/dht/get.js

+4-3
Original file line numberDiff line numberDiff line change
@@ -1,25 +1,26 @@
11
'use strict'
22

3-
const { Buffer } = require('buffer')
43
const configure = require('../lib/configure')
54
const toUrlSearchParams = require('../lib/to-url-search-params')
65
const { Value } = require('./response-types')
6+
const uint8ArrayToString = require('uint8arrays/to-string')
7+
const uint8ArrayFromString = require('uint8arrays/from-string')
78

89
module.exports = configure(api => {
910
return async function get (key, options = {}) {
1011
const res = await api.post('dht/get', {
1112
timeout: options.timeout,
1213
signal: options.signal,
1314
searchParams: toUrlSearchParams({
14-
arg: Buffer.isBuffer(key) ? key.toString() : key,
15+
arg: key instanceof Uint8Array ? uint8ArrayToString(key) : key,
1516
...options
1617
}),
1718
headers: options.headers
1819
})
1920

2021
for await (const message of res.ndjson()) {
2122
if (message.Type === Value) {
22-
return Buffer.from(message.Extra, 'base64')
23+
return uint8ArrayFromString(message.Extra, 'base64pad')
2324
}
2425
}
2526

‎src/files/read.js

+1-4
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
'use strict'
22

3-
const { Buffer } = require('buffer')
43
const toIterable = require('stream-to-it/source')
54
const configure = require('../lib/configure')
65
const toUrlSearchParams = require('../lib/to-url-search-params')
@@ -18,8 +17,6 @@ module.exports = configure(api => {
1817
headers: options.headers
1918
})
2019

21-
for await (const chunk of toIterable(res.body)) {
22-
yield Buffer.from(chunk)
23-
}
20+
yield * toIterable(res.body)
2421
}
2522
})

‎src/get.js

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,18 @@
11
'use strict'
22

33
const Tar = require('it-tar')
4-
const { Buffer } = require('buffer')
54
const CID = require('cids')
65
const configure = require('./lib/configure')
76
const toUrlSearchParams = require('./lib/to-url-search-params')
7+
const map = require('it-map')
88

99
module.exports = configure(api => {
1010
return async function * get (path, options = {}) {
1111
const res = await api.post('get', {
1212
timeout: options.timeout,
1313
signal: options.signal,
1414
searchParams: toUrlSearchParams({
15-
arg: `${Buffer.isBuffer(path) ? new CID(path) : path}`,
15+
arg: `${path instanceof Uint8Array ? new CID(path) : path}`,
1616
...options
1717
}),
1818
headers: options.headers
@@ -28,7 +28,7 @@ module.exports = configure(api => {
2828
} else {
2929
yield {
3030
path: header.name,
31-
content: body
31+
content: map(body, (chunk) => chunk.slice()) // convert bl to Buffer/Uint8Array
3232
}
3333
}
3434
}

‎src/index.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
'use strict'
22
/* eslint-env browser */
3-
const { Buffer } = require('buffer')
3+
44
const CID = require('cids')
55
const multiaddr = require('multiaddr')
66
const multibase = require('multibase')
@@ -56,6 +56,6 @@ function ipfsClient (options = {}) {
5656
}
5757
}
5858

59-
Object.assign(ipfsClient, { Buffer, CID, multiaddr, multibase, multicodec, multihash, globSource, urlSource })
59+
Object.assign(ipfsClient, { CID, multiaddr, multibase, multicodec, multihash, globSource, urlSource })
6060

6161
module.exports = ipfsClient

‎src/ls.js

+1-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
'use strict'
22

3-
const { Buffer } = require('buffer')
43
const CID = require('cids')
54
const configure = require('./lib/configure')
65
const toUrlSearchParams = require('./lib/to-url-search-params')
@@ -11,7 +10,7 @@ module.exports = configure(api => {
1110
timeout: options.timeout,
1211
signal: options.signal,
1312
searchParams: toUrlSearchParams({
14-
arg: `${Buffer.isBuffer(path) ? new CID(path) : path}`,
13+
arg: `${path instanceof Uint8Array ? new CID(path) : path}`,
1514
...options
1615
}),
1716
headers: options.headers

‎src/object/data.js

+2-3
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
'use strict'
22

3-
const { Buffer } = require('buffer')
43
const CID = require('cids')
54
const configure = require('../lib/configure')
65
const toUrlSearchParams = require('../lib/to-url-search-params')
@@ -11,13 +10,13 @@ module.exports = configure(api => {
1110
timeout: options.timeout,
1211
signal: options.signal,
1312
searchParams: toUrlSearchParams({
14-
arg: `${Buffer.isBuffer(cid) ? new CID(cid) : cid}`,
13+
arg: `${cid instanceof Uint8Array ? new CID(cid) : cid}`,
1514
...options
1615
}),
1716
headers: options.headers
1817
})
1918
const data = await res.arrayBuffer()
2019

21-
return Buffer.from(data)
20+
return new Uint8Array(data, data.byteOffset, data.byteLength)
2221
}
2322
})

‎src/object/get.js

+3-3
Original file line numberDiff line numberDiff line change
@@ -1,18 +1,18 @@
11
'use strict'
22

3-
const { Buffer } = require('buffer')
43
const CID = require('cids')
54
const { DAGNode, DAGLink } = require('ipld-dag-pb')
65
const configure = require('../lib/configure')
76
const toUrlSearchParams = require('../lib/to-url-search-params')
7+
const uint8ArrayFromString = require('uint8arrays/from-string')
88

99
module.exports = configure(api => {
1010
return async (cid, options = {}) => {
1111
const res = await api.post('object/get', {
1212
timeout: options.timeout,
1313
signal: options.signal,
1414
searchParams: toUrlSearchParams({
15-
arg: `${Buffer.isBuffer(cid) ? new CID(cid) : cid}`,
15+
arg: `${cid instanceof Uint8Array ? new CID(cid) : cid}`,
1616
dataEncoding: 'base64',
1717
...options
1818
}),
@@ -21,7 +21,7 @@ module.exports = configure(api => {
2121
const data = await res.json()
2222

2323
return new DAGNode(
24-
Buffer.from(data.Data, 'base64'),
24+
uint8ArrayFromString(data.Data, 'base64pad'),
2525
(data.Links || []).map(l => new DAGLink(l.Name, l.Size, l.Hash))
2626
)
2727
}

‎src/object/links.js

+1-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
'use strict'
22

3-
const { Buffer } = require('buffer')
43
const CID = require('cids')
54
const { DAGLink } = require('ipld-dag-pb')
65
const configure = require('../lib/configure')
@@ -12,7 +11,7 @@ module.exports = configure(api => {
1211
timeout: options.timeout,
1312
signal: options.signal,
1413
searchParams: toUrlSearchParams({
15-
arg: `${Buffer.isBuffer(cid) ? new CID(cid) : cid}`,
14+
arg: `${cid instanceof Uint8Array ? new CID(cid) : cid}`,
1615
...options
1716
}),
1817
headers: options.headers

‎src/object/patch/add-link.js

+1-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
'use strict'
22

3-
const { Buffer } = require('buffer')
43
const CID = require('cids')
54
const configure = require('../../lib/configure')
65
const toUrlSearchParams = require('../../lib/to-url-search-params')
@@ -12,7 +11,7 @@ module.exports = configure(api => {
1211
signal: options.signal,
1312
searchParams: toUrlSearchParams({
1413
arg: [
15-
`${Buffer.isBuffer(cid) ? new CID(cid) : cid}`,
14+
`${cid instanceof Uint8Array ? new CID(cid) : cid}`,
1615
dLink.Name || dLink.name || '',
1716
(dLink.Hash || dLink.cid || '').toString() || null
1817
],

‎src/object/patch/append-data.js

+1-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
'use strict'
22

3-
const { Buffer } = require('buffer')
43
const CID = require('cids')
54
const multipartRequest = require('../../lib/multipart-request')
65
const configure = require('../../lib/configure')
@@ -18,7 +17,7 @@ module.exports = configure(api => {
1817
timeout: options.timeout,
1918
signal,
2019
searchParams: toUrlSearchParams({
21-
arg: `${Buffer.isBuffer(cid) ? new CID(cid) : cid}`,
20+
arg: `${cid instanceof Uint8Array ? new CID(cid) : cid}`,
2221
...options
2322
}),
2423
...(

‎src/object/patch/rm-link.js

+1-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
'use strict'
22

3-
const { Buffer } = require('buffer')
43
const CID = require('cids')
54
const configure = require('../../lib/configure')
65
const toUrlSearchParams = require('../../lib/to-url-search-params')
@@ -12,7 +11,7 @@ module.exports = configure(api => {
1211
signal: options.signal,
1312
searchParams: toUrlSearchParams({
1413
arg: [
15-
`${Buffer.isBuffer(cid) ? new CID(cid) : cid}`,
14+
`${cid instanceof Uint8Array ? new CID(cid) : cid}`,
1615
dLink.Name || dLink.name || null
1716
],
1817
...options

‎src/object/patch/set-data.js

+1-2
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,5 @@
11
'use strict'
22

3-
const { Buffer } = require('buffer')
43
const CID = require('cids')
54
const multipartRequest = require('../../lib/multipart-request')
65
const configure = require('../../lib/configure')
@@ -19,7 +18,7 @@ module.exports = configure(api => {
1918
signal,
2019
searchParams: toUrlSearchParams({
2120
arg: [
22-
`${Buffer.isBuffer(cid) ? new CID(cid) : cid}`
21+
`${cid instanceof Uint8Array ? new CID(cid) : cid}`
2322
],
2423
...options
2524
}),

0 commit comments

Comments
 (0)