Skip to content
This repository was archived by the owner on Feb 12, 2024. It is now read-only.

Commit 44045b0

Browse files
dirkmcAlan Shaw
authored and
Alan Shaw
committed
feat: garbage collection (#2022)
resolves #2012 Depends on - [x] #2004 - [x] ipfs-inactive/js-ipfs-http-client#992 - [x] ipfs-inactive/interface-js-ipfs-core#462 - [x] achingbrain/mortice#1 TODO: - [x] Core (mark and sweep) - [x] CLI - [x] http interface - [x] interface-js-ipfs-core tests ipfs-inactive/interface-js-ipfs-core#462 - [x] nodejs-specific tests - [x] Locking - [x] Tests for locking
1 parent 02e521e commit 44045b0

File tree

24 files changed

+1599
-435
lines changed

24 files changed

+1599
-435
lines changed

package.json

+5-2
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@
6565
"array-shuffle": "^1.0.1",
6666
"async": "^2.6.1",
6767
"async-iterator-all": "^1.0.0",
68-
"async-iterator-to-pull-stream": "^1.1.0",
68+
"async-iterator-to-pull-stream": "^1.3.0",
6969
"async-iterator-to-stream": "^1.1.0",
7070
"base32.js": "~0.1.0",
7171
"bignumber.js": "^9.0.0",
@@ -83,6 +83,7 @@
8383
"debug": "^4.1.0",
8484
"dlv": "^1.1.3",
8585
"err-code": "^2.0.0",
86+
"explain-error": "^1.0.4",
8687
"file-type": "^12.0.1",
8788
"fnv1a": "^1.0.1",
8889
"fsm-event": "^2.1.0",
@@ -95,7 +96,7 @@
9596
"ipfs-bitswap": "~0.25.1",
9697
"ipfs-block": "~0.8.1",
9798
"ipfs-block-service": "~0.15.2",
98-
"ipfs-http-client": "^33.1.0",
99+
"ipfs-http-client": "^33.1.1",
99100
"ipfs-http-response": "~0.3.1",
100101
"ipfs-mfs": "~0.12.0",
101102
"ipfs-multipart": "~0.1.1",
@@ -139,6 +140,7 @@
139140
"merge-options": "^1.0.1",
140141
"mime-types": "^2.1.21",
141142
"mkdirp": "~0.5.1",
143+
"mortice": "^1.2.2",
142144
"multiaddr": "^6.1.0",
143145
"multiaddr-to-uri": "^5.0.0",
144146
"multibase": "~0.6.0",
@@ -193,6 +195,7 @@
193195
"ipfsd-ctl": "^0.43.0",
194196
"libp2p-websocket-star": "~0.10.2",
195197
"ncp": "^2.0.0",
198+
"p-event": "^4.1.0",
196199
"qs": "^6.5.2",
197200
"rimraf": "^3.0.0",
198201
"sinon": "^7.4.0",

src/cli/commands/repo/gc.js

+24-5
Original file line numberDiff line numberDiff line change
@@ -5,12 +5,31 @@ module.exports = {
55

66
describe: 'Perform a garbage collection sweep on the repo.',
77

8-
builder: {},
8+
builder: {
9+
quiet: {
10+
alias: 'q',
11+
desc: 'Write minimal output',
12+
type: 'boolean',
13+
default: false
14+
},
15+
'stream-errors': {
16+
desc: 'Output individual errors thrown when deleting blocks.',
17+
type: 'boolean',
18+
default: true
19+
}
20+
},
921

10-
handler (argv) {
11-
argv.resolve((async () => {
12-
const ipfs = await argv.getIpfs()
13-
await ipfs.repo.gc()
22+
handler ({ getIpfs, print, quiet, streamErrors, resolve }) {
23+
resolve((async () => {
24+
const ipfs = await getIpfs()
25+
const res = await ipfs.repo.gc()
26+
for (const r of res) {
27+
if (r.err) {
28+
streamErrors && print(r.err.message, true, true)
29+
} else {
30+
print((quiet ? '' : 'removed ') + r.cid)
31+
}
32+
}
1433
})())
1534
}
1635
}

src/core/components/block.js

+15-10
Original file line numberDiff line numberDiff line change
@@ -81,17 +81,19 @@ module.exports = function block (self) {
8181
cb(null, new Block(block, cid))
8282
})
8383
},
84-
(block, cb) => self._blockService.put(block, (err) => {
85-
if (err) {
86-
return cb(err)
87-
}
84+
(block, cb) => self._gcLock.readLock((_cb) => {
85+
self._blockService.put(block, (err) => {
86+
if (err) {
87+
return _cb(err)
88+
}
8889

89-
if (options.preload !== false) {
90-
self._preload(block.cid)
91-
}
90+
if (options.preload !== false) {
91+
self._preload(block.cid)
92+
}
9293

93-
cb(null, block)
94-
})
94+
_cb(null, block)
95+
})
96+
}, cb)
9597
], callback)
9698
}),
9799
rm: promisify((cid, callback) => {
@@ -100,7 +102,10 @@ module.exports = function block (self) {
100102
} catch (err) {
101103
return setImmediate(() => callback(errCode(err, 'ERR_INVALID_CID')))
102104
}
103-
self._blockService.delete(cid, callback)
105+
106+
// We need to take a write lock here to ensure that adding and removing
107+
// blocks are exclusive operations
108+
self._gcLock.writeLock((cb) => self._blockService.delete(cid, cb), callback)
104109
}),
105110
stat: promisify((cid, options, callback) => {
106111
if (typeof options === 'function') {

src/core/components/files-regular/add-pull-stream.js

+5-3
Original file line numberDiff line numberDiff line change
@@ -116,7 +116,9 @@ function pinFile (file, self, opts, cb) {
116116
const isRootDir = !file.path.includes('/')
117117
const shouldPin = pin && isRootDir && !opts.onlyHash && !opts.hashAlg
118118
if (shouldPin) {
119-
return self.pin.add(file.hash, { preload: false }, err => cb(err, file))
119+
// Note: addPullStream() has already taken a GC lock, so tell
120+
// pin.add() not to take a (second) GC lock
121+
return self.pin.add(file.hash, { preload: false, lock: false }, err => cb(err, file))
120122
} else {
121123
cb(null, file)
122124
}
@@ -156,7 +158,7 @@ module.exports = function (self) {
156158
}
157159

158160
opts.progress = progress
159-
return pull(
161+
return self._gcLock.pullReadLock(() => pull(
160162
pullMap(content => normalizeContent(content, opts)),
161163
pullFlatten(),
162164
pullMap(file => ({
@@ -167,6 +169,6 @@ module.exports = function (self) {
167169
pullAsyncMap((file, cb) => prepareFile(file, self, opts, cb)),
168170
pullMap(file => preloadFile(file, self, opts)),
169171
pullAsyncMap((file, cb) => pinFile(file, self, opts, cb))
170-
)
172+
))
171173
}
172174
}

src/core/components/object.js

+15-13
Original file line numberDiff line numberDiff line change
@@ -242,19 +242,21 @@ module.exports = function object (self) {
242242
}
243243

244244
function next () {
245-
self._ipld.put(node, multicodec.DAG_PB, {
246-
cidVersion: 0,
247-
hashAlg: multicodec.SHA2_256
248-
}).then(
249-
(cid) => {
250-
if (options.preload !== false) {
251-
self._preload(cid)
252-
}
253-
254-
callback(null, cid)
255-
},
256-
(error) => callback(error)
257-
)
245+
self._gcLock.readLock((cb) => {
246+
self._ipld.put(node, multicodec.DAG_PB, {
247+
cidVersion: 0,
248+
hashAlg: multicodec.SHA2_256
249+
}).then(
250+
(cid) => {
251+
if (options.preload !== false) {
252+
self._preload(cid)
253+
}
254+
255+
cb(null, cid)
256+
},
257+
cb
258+
)
259+
}, callback)
258260
}
259261
}),
260262

0 commit comments

Comments
 (0)