Skip to content

chore: update types #302

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 3 commits into from
Apr 15, 2021
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 8 additions & 9 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -51,34 +51,33 @@
"@types/ncp": "^2.0.4",
"@types/proper-lockfile": "^4.1.1",
"@types/rimraf": "^3.0.0",
"aegir": "^31.0.1",
"aegir": "^33.1.0",
"assert": "^2.0.0",
"events": "^3.3.0",
"it-all": "^1.0.2",
"it-drain": "^1.0.1",
"it-first": "^1.0.2",
"just-range": "^2.1.0",
"memdown": "^5.1.0",
"memdown": "^6.0.0",
"multihashing-async": "^2.1.0",
"ncp": "^2.0.0",
"process": "^0.11.10",
"readable-stream": "^3.6.0",
"rimraf": "^3.0.0",
"sinon": "^9.0.2",
"sinon": "^10.0.0",
"url": "^0.11.0",
"util": "^0.12.3"
},
"dependencies": {
"bignumber.js": "^9.0.0",
"bytes": "^3.1.0",
"cids": "^1.1.6",
"datastore-core": "^3.0.0",
"datastore-fs": "^3.0.0",
"datastore-level": "^4.0.0",
"datastore-core": "^4.0.0",
"datastore-fs": "^4.0.0",
"datastore-level": "^5.0.0",
"debug": "^4.1.0",
"err-code": "^3.0.1",
"interface-datastore": "^3.0.3",
"ipfs-repo-migrations": "^7.0.1",
"interface-datastore": "^4.0.0",
"ipfs-repo-migrations": "^8.0.0",
"ipfs-utils": "^6.0.0",
"ipld-block": "^0.11.0",
"it-filter": "^1.0.2",
Expand Down
2 changes: 1 addition & 1 deletion src/api-addr.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ const apiFile = new Key('api')

/**
*
* @param {import("interface-datastore").Datastore} store
* @param {import('interface-datastore').Datastore} store
*/
module.exports = (store) => {
return {
Expand Down
6 changes: 3 additions & 3 deletions src/backends.js
Original file line number Diff line number Diff line change
@@ -1,9 +1,9 @@
'use strict'

/**
* @typedef {import("interface-datastore").Datastore} Datastore
* @typedef {import("./types").Backends} Backends
* @typedef {Required<import("./types").Options>} Options
* @typedef {import('interface-datastore').Datastore} Datastore
* @typedef {import('./types').Backends} Backends
* @typedef {Required<import('./types').Options>} Options
*/

/**
Expand Down
9 changes: 5 additions & 4 deletions src/blockstore.js
Original file line number Diff line number Diff line change
Expand Up @@ -7,10 +7,10 @@ const map = require('it-map')
const drain = require('it-drain')
const pushable = require('it-pushable')
/**
* @typedef {import("interface-datastore").Query} Query
* @typedef {import("interface-datastore").Datastore} Datastore
* @typedef {import("interface-datastore").Options} DatastoreOptions
* @typedef {import("cids")} CID
* @typedef {import('interface-datastore').Query} Query
* @typedef {import('interface-datastore').Datastore} Datastore
* @typedef {import('interface-datastore').Options} DatastoreOptions
* @typedef {import('cids')} CID
* @typedef {import('./types').Blockstore} Blockstore
*/

Expand Down Expand Up @@ -45,6 +45,7 @@ function createBaseStore (store) {
return store.open()
},

// @ts-ignore TODO: ts does not think we will yield only CIDs or only Blocks
async * query (query, options) {
for await (const { key, value } of store.query(query, options)) {
// TODO: we should make this a different method
Expand Down
57 changes: 27 additions & 30 deletions src/config.js
Original file line number Diff line number Diff line change
Expand Up @@ -2,9 +2,7 @@

const { Key } = require('interface-datastore')
const { default: Queue } = require('p-queue')
// @ts-ignore
const _get = require('just-safe-get')
// @ts-ignore
const _set = require('just-safe-set')
const errcode = require('err-code')
const errors = require('./errors')
Expand All @@ -19,8 +17,11 @@ const {
const configKey = new Key('config')

/**
*
* @param {import("interface-datastore").Datastore} store
* @typedef {import('./types').Config} Config
*/

/**
* @param {import('interface-datastore').Datastore} store
*/
module.exports = (store) => {
const setQueue = new Queue({ concurrency: 1 })
Expand All @@ -31,56 +32,52 @@ module.exports = (store) => {
*
* @param {Object} [options] - options
* @param {AbortSignal} [options.signal] - abort this config read
* @returns {Promise<unknown>}
* @returns {Promise<Config>}
*/
getAll (options = {}) { // eslint-disable-line require-await
return configStore.get(undefined, options)
async getAll (options = {}) { // eslint-disable-line require-await
// [email protected] cannot read keys from [email protected] dbs so fall back to
Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Is there an issue to track this? If not, can you create one?

Copy link
Member Author

@achingbrain achingbrain Apr 15, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I can do, but it's not something that's ever going to be fixed so the issue will hang around forever.

The level-js 4 -> 5 version bump was because of this breaking change.

Copy link
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok makes sense, so maybe not worth

// using IndexedDB API with string keys - only necessary until we do
// the migratiion to v10 or above
const encodedValue = await getWithFallback(configKey, store.get.bind(store), store.has.bind(store), store, {
signal: options.signal
})

return JSON.parse(uint8ArrayToString(encodedValue))
},

/**
* Get the value for the passed configuration key from the repo.
*
* @param {string} [key] - the config key to get
* @param {string} key - the config key to get
* @param {Object} [options] - options
* @param {AbortSignal} [options.signal] - abort this config read
* @returns {Promise<unknown>}
*/
async get (key, options = {}) {
if (!key) {
key = undefined
if (key == null) {
throw new errors.NotFoundError(`Key ${key} does not exist in config`)
}

// [email protected] cannot read keys from [email protected] dbs so fall back to
// using IndexedDB API with string keys - only necessary until we do
// the migratiion to v10 or above
const encodedValue = await getWithFallback(configKey, store.get.bind(store), store.has.bind(store), store)

if (options.signal && options.signal.aborted) {
return
}
const config = await this.getAll(options)
const value = _get(config, key)

const config = JSON.parse(uint8ArrayToString(encodedValue))
if (key !== undefined && _get(config, key) === undefined) {
if (value === undefined) {
throw new errors.NotFoundError(`Key ${key} does not exist in config`)
}

const value = key !== undefined ? _get(config, key) : config
return value
},

/**
* Set the current configuration for this repo.
*
* @param {string | unknown} [key] - the config key to be written
* @param {unknown} [value] - the config value to be written
* @param {string} key - the config key to be written
* @param {any} [value] - the config value to be written
* @param {Object} [options] - options
* @param {AbortSignal} [options.signal] - abort this config write
*/
set (key, value, options = {}) {
if (arguments.length === 1) {
value = key
key = undefined
} else if (!key || typeof key !== 'string') {
// @ts-ignore ts thinks key will only be a string, but it may not be
if (typeof key !== 'string' && !(key instanceof String)) {
throw errcode(new Error('Invalid key type: ' + typeof key), 'ERR_INVALID_KEY')
}

Expand All @@ -97,7 +94,7 @@ module.exports = (store) => {
/**
* Set the current configuration for this repo.
*
* @param {Object} [value] - the config value to be written
* @param {Config} [value] - the config value to be written
* @param {Object} [options] - options
* @param {AbortSignal} [options.signal] - abort this config write
*/
Expand Down Expand Up @@ -138,7 +135,7 @@ module.exports = (store) => {
const key = m.key
const value = m.value
if (key) {
const config = await configStore.get()
const config = await configStore.getAll()
if (typeof config === 'object' && config !== null) {
_set(config, key, value)
}
Expand Down
2 changes: 1 addition & 1 deletion src/default-options.js
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
// Default configuration for a repo in node.js

/**
* @type {Required<import("./types").Options>}
* @type {Required<import('./types').Options>}
*/
module.exports = {
autoMigrate: true,
Expand Down
8 changes: 5 additions & 3 deletions src/idstore.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,9 @@ const CID = require('cids')
const errcode = require('err-code')

/**
* @typedef {import("interface-datastore").Query} Query
* @typedef {import("interface-datastore").Datastore} Datastore
* @typedef {import("interface-datastore").Options} DatastoreOptions
* @typedef {import('interface-datastore').Query} Query
* @typedef {import('interface-datastore').Datastore} Datastore
* @typedef {import('interface-datastore').Options} DatastoreOptions
* @typedef {import('./types').Blockstore} Blockstore
*/

Expand All @@ -30,6 +30,8 @@ function createIdStore (store) {
open () {
return store.open()
},

// @ts-ignore TODO: ts does not think we will yield only CIDs or only Blocks
query (query, options) {
return store.query(query, options)
},
Expand Down
54 changes: 25 additions & 29 deletions src/index.js
Original file line number Diff line number Diff line change
@@ -1,9 +1,7 @@
'use strict'

// @ts-ignore
const _get = require('just-safe-get')
const debug = require('debug')
const Big = require('bignumber.js').BigNumber
const errcode = require('err-code')
const migrator = require('ipfs-repo-migrations')
const bytes = require('bytes')
Expand Down Expand Up @@ -33,12 +31,14 @@ const lockers = {
}

/**
* @typedef {import("./types").Options} Options
* @typedef {import("./types").Lock} Lock
* @typedef {import("./types").LockCloser} LockCloser
* @typedef {import("./types").Stat} Stat
* @typedef {import("ipld-block")} Block
* @typedef {import("interface-datastore").Datastore} Datastore
* @typedef {import('./types').Options} Options
* @typedef {import('./types').Lock} Lock
* @typedef {import('./types').LockCloser} LockCloser
* @typedef {import('./types').Stat} Stat
* @typedef {import('./types').Blockstore} Blockstore
* @typedef {import('./types').Config} Config
* @typedef {import('ipld-block')} Block
* @typedef {import('interface-datastore').Datastore} Datastore
*/

/**
Expand Down Expand Up @@ -78,13 +78,13 @@ class IpfsRepo {
/**
* Initialize a new repo.
*
* @param {any} config - config to write into `config`.
* @param {import('./types').Config} config - config to write into `config`.
* @returns {Promise<void>}
*/
async init (config) {
log('initializing at: %s', this.path)
await this._openRoot()
await this.config.set(buildConfig(config))
await this.config.replace(buildConfig(config))
await this.spec.set(buildDatastoreSpec(config))
await this.version.set(constants.repoVersion)
}
Expand Down Expand Up @@ -319,9 +319,7 @@ class IpfsRepo {
getSize(this.datastore),
getSize(this.keys)
])
const size = blocks.size
.plus(datastore)
.plus(keys)
const size = blocks.size + datastore + keys

return {
repoPath: this.path,
Expand Down Expand Up @@ -389,26 +387,25 @@ class IpfsRepo {
async _storageMaxStat () {
try {
const max = /** @type {number} */(await this.config.get('Datastore.StorageMax'))
return new Big(bytes(max))
return BigInt(bytes(max))
} catch (err) {
return new Big(noLimit)
return BigInt(noLimit)
}
}

/**
* @private
*/
async _blockStat () {
let count = new Big(0)
let size = new Big(0)
let count = BigInt(0)
let size = BigInt(0)

if (this.blocks) {
for await (const blockOrCid of this.blocks.query({})) {
const block = /** @type {Block} */(blockOrCid)
count = count.plus(1)
size = size
.plus(block.data.byteLength)
.plus(block.cid.bytes.byteLength)
count += BigInt(1)
size += BigInt(block.data.byteLength)
size += BigInt(block.cid.bytes.byteLength)
}
}

Expand All @@ -420,10 +417,10 @@ class IpfsRepo {
* @param {Datastore} datastore
*/
async function getSize (datastore) {
const sum = new Big(0)
let sum = BigInt(0)
for await (const block of datastore.query({})) {
sum.plus(block.value.byteLength)
.plus(block.key.uint8Array().byteLength)
sum += BigInt(block.value.byteLength)
sum += BigInt(block.key.uint8Array().byteLength)
}
return sum
}
Expand All @@ -433,24 +430,23 @@ module.exports.utils = { blockstore: require('./blockstore-utils') }
module.exports.repoVersion = constants.repoVersion
module.exports.errors = ERRORS

// TODO this should come from js-ipfs instead
/**
* @param {any} _config
* @param {import('./types').Config} _config
*/
function buildConfig (_config) {
_config.datastore = Object.assign({}, defaultDatastore, _get(_config, 'datastore'))
_config.Datastore = Object.assign({}, defaultDatastore, _get(_config, 'datastore'))

return _config
}

/**
* @param {any} _config
* @param {import('./types').Config} _config
*/
function buildDatastoreSpec (_config) {
/** @type { {type: string, mounts: Array<{mountpoint: string, type: string, prefix: string, child: {type: string, path: 'string', sync: boolean, shardFunc: string}}>}} */
const spec = {
...defaultDatastore.Spec,
..._get(_config, 'datastore.Spec')
..._get(_config, 'Datastore.Spec')
}

return {
Expand Down
2 changes: 1 addition & 1 deletion src/lock-memory.js
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@ const lockFile = 'repo.lock'
const LOCKS = {}

/**
* @typedef {import("./types").LockCloser} LockCloser
* @typedef {import('./types').LockCloser} LockCloser
*/

/**
Expand Down
2 changes: 1 addition & 1 deletion src/lock.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ const log = debug('ipfs:repo:lock')
const lockFile = 'repo.lock'

/**
* @typedef {import("./types").LockCloser} LockCloser
* @typedef {import('./types').LockCloser} LockCloser
*/

/**
Expand Down
2 changes: 1 addition & 1 deletion src/spec.js
Original file line number Diff line number Diff line change
Expand Up @@ -9,7 +9,7 @@ const specKey = new Key('datastore_spec')

/**
*
* @param {import("interface-datastore").Datastore} store
* @param {import('interface-datastore').Datastore} store
*/
module.exports = (store) => {
return {
Expand Down
Loading