From 59f5e71d312c07497365a6b6df98f758878ffbb7 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 3 Sep 2019 15:32:11 +0100 Subject: [PATCH 1/6] feat: add glob-source from js-ipfs The business of turning a path and pattern into an iterator of files is duplicated between js-ipfs and js-ipfs-http-client so moving it here to aid deduplication. --- package.json | 9 ++- src/files/glob-source.js | 89 +++++++++++++++++++++++++++ test/files/glob-source.spec.js | 108 +++++++++++++++++++++++++++++++++ test/fixtures/dir/.hidden.txt | 0 test/fixtures/dir/file-1.txt | 0 test/fixtures/dir/file-2.js | 0 test/fixtures/dir/file-3.css | 0 test/fixtures/file-0.html | 0 8 files changed, 205 insertions(+), 1 deletion(-) create mode 100644 src/files/glob-source.js create mode 100644 test/files/glob-source.spec.js create mode 100644 test/fixtures/dir/.hidden.txt create mode 100644 test/fixtures/dir/file-1.txt create mode 100644 test/fixtures/dir/file-2.js create mode 100644 test/fixtures/dir/file-3.css create mode 100644 test/fixtures/file-0.html diff --git a/package.json b/package.json index d9c0ac3..a80f1b4 100644 --- a/package.json +++ b/package.json @@ -28,10 +28,12 @@ "dependencies": { "buffer": "^5.2.1", "err-code": "^2.0.0", + "fs-extra": "^8.1.0", "is-buffer": "^2.0.3", "is-electron": "^2.2.0", "is-pull-stream": "0.0.0", "is-stream": "^2.0.0", + "it-glob": "0.0.4", "kind-of": "^6.0.2", "pull-stream-to-async-iterator": "^1.0.2", "readable-stream": "^3.4.0" @@ -40,12 +42,17 @@ "aegir": "^20.0.0", "async-iterator-all": "^1.0.0", "chai": "^4.2.0", + "chai-as-promised": "^7.1.1", "dirty-chai": "^2.0.1", "electron": "^6.0.6", "electron-mocha": "^8.0.3", + "is-node": "^1.0.2", "pull-stream": "^3.6.13" }, "contributors": [ "Hugo Dias " - ] + ], + "browser": { + "fs-extra": false + } } diff --git a/src/files/glob-source.js b/src/files/glob-source.js new file mode 100644 index 0000000..56a67de --- /dev/null +++ b/src/files/glob-source.js @@ -0,0 +1,89 @@ +'use strict' + +const fs = require('fs-extra') +const glob = require('it-glob') +const Path = require('path') +const errCode = require('err-code') + +/** +* Create an async iterator that yields paths that match requested file paths. +* +* @param {String} ...paths File system path(s) to glob from +* @param {Object} [options] Optional options +* @param {Boolean} [options.recursive] Recursively glob all paths in directories +* @param {Boolean} [options.hidden] Include .dot files in matched paths +* @param {Array} [options.ignore] Glob paths to ignore +* @param {Boolean} [options.followSymlinks] follow symlinks +* @yields {Object} File objects in the form `{ path: String, content: AsyncIterator }` +*/ +module.exports = async function * globSource (...args) { + const options = typeof args[args.length - 1] === 'string' ? {} : args.pop() + const paths = args + + const globSourceOptions = { + recursive: options.recursive, + glob: { + dot: Boolean(options.hidden), + ignore: Array.isArray(options.ignore) ? options.ignore : [], + follow: options.followSymlinks != null ? options.followSymlinks : true + } + } + + // Check the input paths comply with options.recursive and convert to glob sources + for (const path of paths) { + if (typeof path !== 'string') { + throw errCode( + new Error(`Path must be a string`), + 'ERR_INVALID_PATH', + { path } + ) + } + + const absolutePath = Path.resolve(process.cwd(), path) + const stat = await fs.stat(absolutePath) + const prefix = Path.dirname(absolutePath) + + for await (const entry of toGlobSource({ path, type: stat.isDirectory() ? 'dir' : 'file', prefix }, globSourceOptions)) { + yield entry + } + } +} + +async function * toGlobSource ({ path, type, prefix }, options) { + options = options || {} + + const baseName = Path.basename(path) + + if (type === 'file') { + yield { + path: baseName.replace(prefix, ''), + content: fs.createReadStream(Path.isAbsolute(path) ? path : Path.join(process.cwd(), path)) + } + + return + } + + if (type === 'dir' && !options.recursive) { + throw errCode( + new Error(`'${path}' is a directory and recursive option not set`), + 'ERR_DIR_NON_RECURSIVE', + { path } + ) + } + + const globOptions = Object.assign({}, options.glob, { + cwd: path, + nodir: true, + realpath: false, + absolute: true + }) + + for await (const p of glob(path, '**/*', globOptions)) { + yield { + path: toPosix(p.replace(prefix, '')), + content: fs.createReadStream(p) + } + } +} + +const toPosix = path => path.replace(/\\/g, '/') diff --git a/test/files/glob-source.spec.js b/test/files/glob-source.spec.js new file mode 100644 index 0000000..3f7efa9 --- /dev/null +++ b/test/files/glob-source.spec.js @@ -0,0 +1,108 @@ +'use strict' + +/* eslint-env mocha */ +const chai = require('chai') +const dirtyChai = require('dirty-chai') +const chaiAsPromised = require('chai-as-promised') +const globSource = require('../../src/files/glob-source') +const all = require('async-iterator-all') +const path = require('path') +const isNode = require('is-node') + +chai.use(dirtyChai) +chai.use(chaiAsPromised) +const expect = chai.expect + +describe('glob-source', () => { + it('single file, relative path', async function () { + if (!isNode) { + return this.skip() + } + + const result = await all(globSource(path.relative(process.cwd(), path.join(__dirname, '..', 'fixtures', 'file-0.html')))) + + expect(result.length).to.equal(1) + expect(result[0].path).to.equal('file-0.html') + }) + + it('directory, relative path', async function () { + if (!isNode) { + return this.skip() + } + + const result = await all(globSource(path.relative(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir')), { + recursive: true + })) + + expect(result.length).to.equal(3) + expect(result[0].path).to.equal('/dir/file-1.txt') + expect(result[1].path).to.equal('/dir/file-2.js') + expect(result[2].path).to.equal('/dir/file-3.css') + }) + + it('single file, absolute path', async function () { + if (!isNode) { + return this.skip() + } + + const result = await all(globSource(path.resolve(process.cwd(), path.join(__dirname, '..', 'fixtures', 'file-0.html')))) + + expect(result.length).to.equal(1) + expect(result[0].path).to.equal('file-0.html') + }) + + it('directory, relative path', async function () { + if (!isNode) { + return this.skip() + } + + const result = await all(globSource(path.resolve(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir')), { + recursive: true + })) + + expect(result.length).to.equal(3) + expect(result[0].path).to.equal('/dir/file-1.txt') + expect(result[1].path).to.equal('/dir/file-2.js') + expect(result[2].path).to.equal('/dir/file-3.css') + }) + + it('directory, hidden files', async function () { + if (!isNode) { + return this.skip() + } + + const result = await all(globSource(path.resolve(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir')), { + recursive: true, + hidden: true + })) + + expect(result.length).to.equal(4) + expect(result[0].path).to.equal('/dir/.hidden.txt') + expect(result[1].path).to.equal('/dir/file-1.txt') + expect(result[2].path).to.equal('/dir/file-2.js') + expect(result[3].path).to.equal('/dir/file-3.css') + }) + + it('directory, ignore files', async function () { + if (!isNode) { + return this.skip() + } + + const result = await all(globSource(path.resolve(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir')), { + recursive: true, + ignore: ['**/file-1.txt'] + })) + + expect(result.length).to.equal(2) + expect(result[0].path).to.equal('/dir/file-2.js') + expect(result[1].path).to.equal('/dir/file-3.css') + }) + + it('require recusive flag for directory', async function () { + if (!isNode) { + return this.skip() + } + + await expect(all(globSource(path.resolve(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir'))))).to.be.rejectedWith(/recursive option not set/) + }) +}) diff --git a/test/fixtures/dir/.hidden.txt b/test/fixtures/dir/.hidden.txt new file mode 100644 index 0000000..e69de29 diff --git a/test/fixtures/dir/file-1.txt b/test/fixtures/dir/file-1.txt new file mode 100644 index 0000000..e69de29 diff --git a/test/fixtures/dir/file-2.js b/test/fixtures/dir/file-2.js new file mode 100644 index 0000000..e69de29 diff --git a/test/fixtures/dir/file-3.css b/test/fixtures/dir/file-3.css new file mode 100644 index 0000000..e69de29 diff --git a/test/fixtures/file-0.html b/test/fixtures/file-0.html new file mode 100644 index 0000000..e69de29 From 4b50299a6550aff68d948050c83d6d227265f919 Mon Sep 17 00:00:00 2001 From: Alex Potsides Date: Tue, 3 Sep 2019 17:57:41 +0100 Subject: [PATCH 2/6] chore: update test/files/glob-source.spec.js Fix typo Co-Authored-By: Hugo Dias --- test/files/glob-source.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/files/glob-source.spec.js b/test/files/glob-source.spec.js index 3f7efa9..b367cb7 100644 --- a/test/files/glob-source.spec.js +++ b/test/files/glob-source.spec.js @@ -98,7 +98,7 @@ describe('glob-source', () => { expect(result[1].path).to.equal('/dir/file-3.css') }) - it('require recusive flag for directory', async function () { + it('require recursive flag for directory', async function () { if (!isNode) { return this.skip() } From 70c74a3ec5b207cd1516872b79d6af929a711443 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 3 Sep 2019 18:00:49 +0100 Subject: [PATCH 3/6] chore: use env file instead of dep --- package.json | 1 - test/files/glob-source.spec.js | 4 +++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index a80f1b4..50cda09 100644 --- a/package.json +++ b/package.json @@ -46,7 +46,6 @@ "dirty-chai": "^2.0.1", "electron": "^6.0.6", "electron-mocha": "^8.0.3", - "is-node": "^1.0.2", "pull-stream": "^3.6.13" }, "contributors": [ diff --git a/test/files/glob-source.spec.js b/test/files/glob-source.spec.js index b367cb7..9d9ab7e 100644 --- a/test/files/glob-source.spec.js +++ b/test/files/glob-source.spec.js @@ -7,7 +7,9 @@ const chaiAsPromised = require('chai-as-promised') const globSource = require('../../src/files/glob-source') const all = require('async-iterator-all') const path = require('path') -const isNode = require('is-node') +const { + isNode +} = require('../../src/env') chai.use(dirtyChai) chai.use(chaiAsPromised) From f792f8c7f381a626b8e46aaa598bd7f77a3e8e6c Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 3 Sep 2019 18:01:25 +0100 Subject: [PATCH 4/6] chore: fix another typo --- test/files/glob-source.spec.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/test/files/glob-source.spec.js b/test/files/glob-source.spec.js index 9d9ab7e..e4041a2 100644 --- a/test/files/glob-source.spec.js +++ b/test/files/glob-source.spec.js @@ -100,7 +100,7 @@ describe('glob-source', () => { expect(result[1].path).to.equal('/dir/file-3.css') }) - it('require recursive flag for directory', async function () { + it('requires recursive flag for directory', async function () { if (!isNode) { return this.skip() } From e56484a951fa299398c7966722a73149973efc04 Mon Sep 17 00:00:00 2001 From: achingbrain Date: Tue, 3 Sep 2019 18:03:28 +0100 Subject: [PATCH 5/6] test: add test for multiple paths --- test/files/glob-source.spec.js | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/test/files/glob-source.spec.js b/test/files/glob-source.spec.js index e4041a2..7676311 100644 --- a/test/files/glob-source.spec.js +++ b/test/files/glob-source.spec.js @@ -100,6 +100,21 @@ describe('glob-source', () => { expect(result[1].path).to.equal('/dir/file-3.css') }) + it('multiple paths', async function () { + if (!isNode) { + return this.skip() + } + + const result = await all(globSource( + path.relative(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir', 'file-1.txt')), + path.relative(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir', 'file-2.js')) + )) + + expect(result.length).to.equal(2) + expect(result[0].path).to.equal('file-1.txt') + expect(result[1].path).to.equal('file-2.js') + }) + it('requires recursive flag for directory', async function () { if (!isNode) { return this.skip() From 6d976fa8d4a3905b454bd6552672a5deee9bba7e Mon Sep 17 00:00:00 2001 From: achingbrain Date: Wed, 4 Sep 2019 09:55:11 +0100 Subject: [PATCH 6/6] feat: make paths (async)iterable or string --- src/files/glob-source.js | 14 +++++++++----- test/files/glob-source.spec.js | 4 ++-- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/src/files/glob-source.js b/src/files/glob-source.js index 56a67de..6d33056 100644 --- a/src/files/glob-source.js +++ b/src/files/glob-source.js @@ -4,11 +4,12 @@ const fs = require('fs-extra') const glob = require('it-glob') const Path = require('path') const errCode = require('err-code') +const kindOf = require('kind-of') /** * Create an async iterator that yields paths that match requested file paths. * -* @param {String} ...paths File system path(s) to glob from +* @param {Iterable|AsyncIterable|String} paths File system path(s) to glob from * @param {Object} [options] Optional options * @param {Boolean} [options.recursive] Recursively glob all paths in directories * @param {Boolean} [options.hidden] Include .dot files in matched paths @@ -16,9 +17,12 @@ const errCode = require('err-code') * @param {Boolean} [options.followSymlinks] follow symlinks * @yields {Object} File objects in the form `{ path: String, content: AsyncIterator }` */ -module.exports = async function * globSource (...args) { - const options = typeof args[args.length - 1] === 'string' ? {} : args.pop() - const paths = args +module.exports = async function * globSource (paths, options) { + options = options || {} + + if (kindOf(paths) === 'string') { + paths = [paths] + } const globSourceOptions = { recursive: options.recursive, @@ -30,7 +34,7 @@ module.exports = async function * globSource (...args) { } // Check the input paths comply with options.recursive and convert to glob sources - for (const path of paths) { + for await (const path of paths) { if (typeof path !== 'string') { throw errCode( new Error(`Path must be a string`), diff --git a/test/files/glob-source.spec.js b/test/files/glob-source.spec.js index 7676311..214645c 100644 --- a/test/files/glob-source.spec.js +++ b/test/files/glob-source.spec.js @@ -105,10 +105,10 @@ describe('glob-source', () => { return this.skip() } - const result = await all(globSource( + const result = await all(globSource([ path.relative(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir', 'file-1.txt')), path.relative(process.cwd(), path.join(__dirname, '..', 'fixtures', 'dir', 'file-2.js')) - )) + ])) expect(result.length).to.equal(2) expect(result[0].path).to.equal('file-1.txt')