test: adds tests for multi-bucket pin migration (#29)

Once the number of pinned blocks is over 8192, a second layer of pins is created in the pinset data structure.

This PR refactors the tests to use car files of CIDs/blocks to recreate this pinset structure and ensure we can migrate backwards and forwards between them.

It also adds tests for directly pinned and also non-dag-pb blocks to ensure those pins are migrated sucessfully.

Author: achingbrain

.aegir.js

@@ -1,6 +1,10 @@
 'use strict'
 
 module.exports = {
+  karma: {
+    // multi-bucket pinset migrations are slow
+    browserNoActivityTimeout: 240 * 1000
+  },
   webpack: {
     node: {
       // this is needed until level stops using node buffers in browser code

.gitignore

@@ -28,5 +28,5 @@ node_modules
 package-lock.json
 yarn.lock
 # Tests
-test/test-repo-for*
+test-repo-for*
 test/sharness/tmp

.travis.yml

@@ -54,13 +54,13 @@ jobs:
       name: electron-main
       os: osx
       script:
-        - npx aegir test -t electron-main --bail --timeout 10000
+        - npx aegir test -t electron-main --bail --timeout 10000 -f test/node.js
 
     - stage: test
       name: electron-renderer
       os: osx
       script:
-        - npx aegir test -t electron-renderer --bail --timeout 10000
+        - npx aegir test -t electron-renderer --bail --timeout 10000 -f test/node.js
 
 notifications:
   email: false

migrations/migration-9/index.js

@@ -21,7 +21,7 @@ async function pinsToDatastore (blockstore, datastore, pinstore) {
     }
 
     if (cid.version !== 0) {
-      pin.version = version
+      pin.version = cid.version
     }
 
     if (cid.codec !== 'dag-pb') {
@@ -37,7 +37,7 @@ async function pinsToDatastore (blockstore, datastore, pinstore) {
     }
 
     if (cid.version !== 0) {
-      pin.version = version
+      pin.version = cid.version
     }
 
     if (cid.codec !== 'dag-pb') {

migrations/migration-9/pin-set.js

@@ -54,11 +54,11 @@ function readHeader (rootNode) {
 }
 
 function hash (seed, key) {
-  const buffer = new ArrayBuffer(4)
-  const dataView = new DataView(buffer)
+  const buffer = new Uint8Array(4)
+  const dataView = new DataView(buffer.buffer)
   dataView.setUint32(0, seed, true)
   const encodedKey = uint8ArrayFromString(toB58String(key))
-  const data = uint8ArrayConcat([buf, encodedKey], buf.length + encodedKey.length)
+  const data = uint8ArrayConcat([buffer, encodedKey], buffer.byteLength + encodedKey.byteLength)
 
   return fnv1a(uint8ArrayToString(data))
 }
@@ -168,7 +168,7 @@ function storeItems (blockstore, items) {
 
     async function storeChild (child, binIdx) {
       const buf = dagpb.util.serialize(child)
-      const cid = dagpb.util.cid(buf, {
+      const cid = await dagpb.util.cid(buf, {
         cidVersion: 0,
         hashAlg: multicodec.SHA2_256,
       })

package.json

@@ -68,6 +68,7 @@
     "aegir": "^25.0.0",
     "chai": "^4.2.0",
     "chai-as-promised": "^7.1.1",
+    "datastore-car": "^1.2.0",
     "dirty-chai": "^2.0.1",
     "it-all": "^1.0.2",
     "just-safe-set": "^2.1.0",

test/fixtures/generate-car-files.js

@@ -0,0 +1,144 @@
+'use strict'
+
+/* eslint-disable no-console */
+
+// nb. must be [email protected] or below
+const IPFS = require('ipfs')
+const {
+  CID
+} = IPFS
+const { Key } = require('interface-datastore')
+const PIN_DS_KEY = new Key('/local/pins')
+const fs = require('fs')
+const CarDatastore = require('datastore-car')
+const path = require('path')
+
+const TO_PIN = 9000
+
+const main = async () => {
+  const ipfs = await IPFS.create({
+    profile: 'lowpower'
+  })
+
+  const directPins = []
+
+  for (let i = TO_PIN; i < TO_PIN + 10; i++) {
+    const data = `hello-${i}`
+    const { cid } = await ipfs.add(data, { pin: false })
+
+    await ipfs.pin.add(cid, {
+      recursive: false
+    })
+
+    directPins.push(cid)
+  }
+
+  console.info('const directPins = [')
+  console.info(' ', directPins.map(cid => `'${cid}'`).join(',\n  '))
+  console.info(']')
+
+  const nonDagPbRecursivePins = []
+
+  for (let i = TO_PIN + 10; i < TO_PIN + 20; i++) {
+    const data = { derp: `hello-${i}` }
+    const cid = await ipfs.dag.put(data)
+
+    await ipfs.pin.add(cid, {
+      recursive: true
+    })
+
+    nonDagPbRecursivePins.push(`${cid}`)
+  }
+
+  console.info('const nonDagPbRecursivePins = [')
+  console.info(' ', nonDagPbRecursivePins.join(',\n  '))
+  console.info(']')
+
+  const nonDagPbDirectPins = []
+
+  for (let i = TO_PIN + 20; i < TO_PIN + 30; i++) {
+    const data = { derp: `hello-${i}` }
+    const cid = await ipfs.dag.put(data)
+
+    await ipfs.pin.add(cid, {
+      recursive: false
+    })
+
+    nonDagPbDirectPins.push(`${cid}`)
+  }
+
+  console.info('const nonDagPbDirectPins = [')
+  console.info(' ', nonDagPbDirectPins.join(',\n  '))
+  console.info(']')
+
+  console.info('const pinsets = {')
+
+  await writeCar('basic pinset', true)
+
+  for (let i = 0; i < TO_PIN; i++) {
+    const data = `hello-${i}`
+    await ipfs.add(data)
+  }
+
+  await writeCar('multiple buckets pinset')
+
+  console.info('}')
+
+  await ipfs.stop()
+
+  async function writeCar (pinsetName, more) {
+    const fileName = `pinset-${pinsetName.replace(/\s/g, '-').replace('-pinset', '')}.car`
+
+    console.info(`  '${pinsetName}': {`)
+    console.info(`    car: loadFixture('test/fixtures/${fileName}'),`)
+
+    const buf = await ipfs.libp2p.datastore.get(PIN_DS_KEY)
+    const cid = new CID(buf)
+
+    console.info(`    root: new CID('${cid}'),`)
+
+    const outStream = fs.createWriteStream(path.join(__dirname, fileName))
+    const car = await CarDatastore.writeStream(outStream)
+
+    await car.setRoots(cid)
+
+    await walk(cid, car)
+
+    let pins = 0
+
+    for await (const _ of ipfs.pin.ls()) { // eslint-disable-line no-unused-vars
+      pins++
+    }
+
+    console.info(`    pins: ${pins}`)
+    console.info(`  }${more ? ',' : ''}`)
+
+    await car.close()
+  }
+
+  async function walk (cid, car, cids = {}) {
+    if (cids[cid.toString()]) {
+      return
+    }
+
+    cids[cid.toString()] = true
+
+    const block = await ipfs.block.get(cid)
+
+    car.put(cid, block.data)
+
+    const { value: node } = await ipfs.dag.get(cid)
+
+    if (node.Links) {
+      for (const link of node.Links) {
+        await walk(link.Hash, car, cids)
+      }
+    }
+  }
+}
+
+main()
+  .catch(err => {
+    console.error(err)
+    process.exit(1)
+  })

test/fixtures/pinset-basic.car

@@ -1,9 +1,14 @@
 /* eslint-env mocha */
 'use strict'
 
+const DatastoreLevel = require('datastore-level')
+
 const CONFIGURATIONS = [{
   name: 'with sharding',
   options: {
+    storageBackends: {
+      pins: DatastoreLevel
+    },
     storageBackendOptions: {
       root: {
         sharding: true,
@@ -21,7 +26,7 @@ const CONFIGURATIONS = [{
         sharding: true,
         extension: '.data'
       },
-      pinstore: {
+      pins: {
         sharding: true,
         extension: '.data'
       }
@@ -30,6 +35,9 @@ const CONFIGURATIONS = [{
 }, {
   name: 'without sharding',
   options: {
+    storageBackends: {
+      pins: DatastoreLevel
+    },
     storageBackendOptions: {
       root: {
         sharding: false,
@@ -47,7 +55,7 @@ const CONFIGURATIONS = [{
         sharding: false,
         extension: '.data'
       },
-      pinstore: {
+      pins: {
         sharding: false,
         extension: '.data'
       }

test/fixtures/pinset-multiple-buckets.car

@@ -1,11 +1,7 @@
 /* eslint-env mocha */
 'use strict'
 
-const chai = require('chai')
-chai.use(require('dirty-chai'))
-const chaiAsPromised = require('chai-as-promised')
-chai.use(chaiAsPromised)
-const expect = chai.expect
+const { expect } = require('aegir/utils/chai')
 
 const migration = require('../../migrations/migration-8')
 const Key = require('interface-datastore').Key