From 2230eb8adbc09bb0073ec2503168c34162735a75 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 14 Jan 2020 16:09:36 +0000 Subject: [PATCH 1/3] wip: store pins in datastore --- src/default-options-browser.js | 8 +++++++- src/default-options.js | 3 ++- src/index.js | 5 +++-- test/repo-test.js | 5 +++-- 4 files changed, 15 insertions(+), 6 deletions(-) diff --git a/src/default-options-browser.js b/src/default-options-browser.js index 07e59d1a..dd7d6c10 100644 --- a/src/default-options-browser.js +++ b/src/default-options-browser.js @@ -7,7 +7,8 @@ module.exports = { root: require('datastore-level'), blocks: require('datastore-level'), keys: require('datastore-level'), - datastore: require('datastore-level') + datastore: require('datastore-level'), + pins: require('datastore-level') }, storageBackendOptions: { root: { @@ -29,6 +30,11 @@ module.exports = { sharding: false, prefix: '', version: 2 + }, + pins: { + sharding: false, + prefix: '', + version: 2 } } } diff --git a/src/default-options.js b/src/default-options.js index e9322d67..510a2609 100644 --- a/src/default-options.js +++ b/src/default-options.js @@ -7,7 +7,8 @@ module.exports = { root: require('datastore-fs'), blocks: require('datastore-fs'), keys: require('datastore-fs'), - datastore: require('datastore-level') + datastore: require('datastore-level'), + pins: require('datastore-level') }, storageBackendOptions: { root: { diff --git a/src/index.js b/src/index.js index cfe4119d..230a2216 100644 --- a/src/index.js +++ b/src/index.js @@ -32,7 +32,6 @@ const lockers = { /** * IpfsRepo implements all required functionality to read and write to an ipfs repo. - * */ class IpfsRepo { /** @@ -118,6 +117,8 @@ class IpfsRepo { this.blocks = await blockstore(blocksBaseStore, this.options.storageBackendOptions.blocks) log('creating keystore') this.keys = backends.create('keys', path.join(this.path, 'keys'), this.options) + log('creating pins') + this.pins = backends.create('pins', path.join(this.path, 'pins'), this.options) const isCompatible = await this.version.check(constants.repoVersion) if (!isCompatible) { @@ -251,7 +252,7 @@ class IpfsRepo { } } - await Promise.all([this.root, this.blocks, this.keys, this.datastore].map((store) => store.close())) + await Promise.all([this.root, this.blocks, this.keys, this.datastore, this.pins].map((store) => store.close())) log('unlocking') this.closed = true await this._closeLock() diff --git a/test/repo-test.js b/test/repo-test.js index c4e53ab0..f2e54944 100644 --- a/test/repo-test.js +++ b/test/repo-test.js @@ -165,13 +165,14 @@ module.exports = (repo) => { root: FakeDatastore, blocks: FakeDatastore, keys: FakeDatastore, - datastore: FakeDatastore + datastore: FakeDatastore, + pins: FakeDatastore } }) await repo.init({}) await repo.open() await repo.close() - expect(count).to.be.eq(4) + expect(count).to.be.eq(5) }) it('open twice throws error', async () => { From c1b36d18eefd51fbf59627fd14f5b0ce5e2f4d9c Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 21 Jul 2020 11:02:56 +0100 Subject: [PATCH 2/3] chore: add tests, upgrade deps --- .aegir.js | 10 ++++++ package.json | 4 +-- test/browser.js | 1 + test/node.js | 1 + test/pins-test.js | 79 +++++++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 93 insertions(+), 2 deletions(-) create mode 100644 .aegir.js create mode 100644 test/pins-test.js diff --git a/.aegir.js b/.aegir.js new file mode 100644 index 00000000..f1766146 --- /dev/null +++ b/.aegir.js @@ -0,0 +1,10 @@ +'use strict' + +module.exports = { + webpack: { + node: { + // this is needed until level stops using node buffers in browser code + Buffer: true + } + } +} diff --git a/package.json b/package.json index 871b39e3..ec760887 100644 --- a/package.json +++ b/package.json @@ -43,7 +43,7 @@ "npm": ">=3.0.0" }, "devDependencies": { - "aegir": "^23.0.0", + "aegir": "^25.0.0", "chai": "^4.2.0", "chai-as-promised": "^7.1.1", "dirty-chai": "^2.0.1", @@ -68,7 +68,7 @@ "debug": "^4.1.0", "err-code": "^2.0.0", "interface-datastore": "^1.0.2", - "ipfs-repo-migrations": "^1.0.0", + "ipfs-repo-migrations": "^2.0.0", "ipfs-utils": "^2.2.0", "ipld-block": "^0.9.1", "it-map": "^1.0.2", diff --git a/test/browser.js b/test/browser.js index f955a7f5..82e2c8f7 100644 --- a/test/browser.js +++ b/test/browser.js @@ -38,5 +38,6 @@ describe('IPFS Repo Tests on the Browser', () => { require('./config-test')(repo) require('./api-addr-test')(repo) require('./lock-test')(repo) + require('./pins-test')(repo) require('./is-initialized') }) diff --git a/test/node.js b/test/node.js index 3b515f60..0ca879f1 100644 --- a/test/node.js +++ b/test/node.js @@ -114,6 +114,7 @@ describe('IPFS Repo Tests onNode.js', () => { if (!r.init) { require('./interop-test')(repo) } + require('./pins-test')(repo) require('./is-initialized') })) diff --git a/test/pins-test.js b/test/pins-test.js new file mode 100644 index 00000000..aeca9f7e --- /dev/null +++ b/test/pins-test.js @@ -0,0 +1,79 @@ +/* eslint max-nested-callbacks: ["error", 8] */ +/* eslint-env mocha */ +'use strict' + +const { Buffer } = require('buffer') +const { expect } = require('./utils/chai') +const range = require('just-range') +const Key = require('interface-datastore').Key + +module.exports = (repo) => { + describe('pins', () => { + const dataList = range(100).map((i) => Buffer.from(`hello-${i}-${Math.random()}`)) + const data = Buffer.from('hello world') + const b = new Key('hello') + + it('exists', () => { + expect(repo).to.have.property('pins') + }) + + it('implements interface-datastore', () => { + const pins = repo.pins + expect(pins.batch).to.exist() + expect(pins.query).to.exist() + }) + + describe('.put', () => { + it('simple', async () => { + await repo.pins.put(b, data) + }) + + it('multi write (locks)', async () => { + await Promise.all([repo.pins.put(b, data), repo.pins.put(b, data)]) + }) + + it('massive multiwrite', async function () { + this.timeout(15000) // add time for ci + await Promise.all(range(100).map((i) => { + return repo.pins.put(new Key('hello' + i), dataList[i]) + })) + }) + }) + + describe('.get', () => { + it('simple', async () => { + const val = await repo.pins.get(b) + expect(val).to.be.eql(data) + }) + + it('massive read', async function () { + this.timeout(15000) // add time for ci + await Promise.all(range(20 * 100).map(async (i) => { + const j = i % dataList.length + const val = await repo.pins.get(new Key('hello' + j)) + expect(val).to.be.eql(dataList[j]) + })) + }).timeout(10 * 1000) + }) + + describe('.has', () => { + it('existing pin', async () => { + const exists = await repo.pins.has(b) + expect(exists).to.eql(true) + }) + + it('non existent pin', async () => { + const exists = await repo.pins.has(new Key('world')) + expect(exists).to.eql(false) + }) + }) + + describe('.delete', () => { + it('simple', async () => { + await repo.pins.delete(b) + const exists = await repo.pins.has(b) + expect(exists).to.equal(false) + }) + }) + }) +} From 437a0a7b2107ad59b859a3564e9a127a8dcf4f47 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 21 Jul 2020 13:46:37 +0100 Subject: [PATCH 3/3] chore: remove redundant pin test --- test/pins-test.js | 6 ------ 1 file changed, 6 deletions(-) diff --git a/test/pins-test.js b/test/pins-test.js index aeca9f7e..d0c12c04 100644 --- a/test/pins-test.js +++ b/test/pins-test.js @@ -17,12 +17,6 @@ module.exports = (repo) => { expect(repo).to.have.property('pins') }) - it('implements interface-datastore', () => { - const pins = repo.pins - expect(pins.batch).to.exist() - expect(pins.query).to.exist() - }) - describe('.put', () => { it('simple', async () => { await repo.pins.put(b, data)