From f42e7eb8cb410fd047f9b9334103a8c8b2ef3aea Mon Sep 17 00:00:00 2001 From: nginnever Date: Thu, 12 May 2016 18:21:19 -0700 Subject: [PATCH 1/5] get command --- src/api/get.js | 24 +++++++++ src/load-commands.js | 1 + test/api/get.spec.js | 119 +++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 144 insertions(+) create mode 100644 src/api/get.js create mode 100644 test/api/get.spec.js diff --git a/src/api/get.js b/src/api/get.js new file mode 100644 index 000000000..f2d122652 --- /dev/null +++ b/src/api/get.js @@ -0,0 +1,24 @@ +'use strict' + +module.exports = (send) => { + return function get (path, archive, compress, compressionLevel, cb) { + if (archive === true && typeof compress === 'function') { + cb = compress + compressionLevel = null + compress = null + } + if (archive === true && typeof compress === 'number') { + archive = null + cb = compressionLevel + compressionLevel = compress + compress = true + } + if (typeof archive === 'function') { + cb = archive + archive = null + compressionLevel = null + compress = null + } + return send('get', path, [archive, compress, compressionLevel], null, cb) + } +} diff --git a/src/load-commands.js b/src/load-commands.js index 74bd33540..95c73d6df 100644 --- a/src/load-commands.js +++ b/src/load-commands.js @@ -13,6 +13,7 @@ function requireCommands () { dht: require('./api/dht'), diag: require('./api/diag'), id: require('./api/id'), + get: require('./api/get'), log: require('./api/log'), ls: require('./api/ls'), mount: require('./api/mount'), diff --git a/test/api/get.spec.js b/test/api/get.spec.js new file mode 100644 index 000000000..9674f6476 --- /dev/null +++ b/test/api/get.spec.js @@ -0,0 +1,119 @@ +/* eslint-env mocha */ +/* globals apiClients */ +'use strict' + +const expect = require('chai').expect +const isNode = require('detect-node') +const fs = require('fs') + +const path = require('path') +const streamEqual = require('stream-equal') + +let testfile +let testfileBig + +if (isNode) { + testfile = fs.readFileSync(path.join(__dirname, '/../testfile.txt')) + testfileBig = fs.createReadStream(path.join(__dirname, '/../15mb.random'), { bufferSize: 128 }) +} else { + testfile = require('raw!../testfile.txt') +} + +describe('.get', () => { + it('get with no compression args', (done) => { + apiClients.a + .get('Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP', (err, res) => { + expect(err).to.not.exist + + let buf = '' + res + .on('error', (err) => { + expect(err).to.not.exist + }) + .on('data', (data) => { + buf += data + }) + .on('end', () => { + expect(buf).to.contain(testfile.toString()) + done() + }) + }) + }) + + it('get with archive true', (done) => { + apiClients.a + .get('Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP', true, (err, res) => { + expect(err).to.not.exist + + let buf = '' + res + .on('error', (err) => { + expect(err).to.not.exist + }) + .on('data', (data) => { + buf += data + }) + .on('end', () => { + expect(buf).to.contain(testfile.toString()) + done() + }) + }) + }) + + it('get with compression level', (done) => { + apiClients.a + .get('Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP', true, 1, (err, res) => { + expect(err).to.not.exist + + let buf = '' + res + .on('error', (err) => { + expect(err).to.not.exist + }) + .on('data', (data) => { + buf += data + }) + .on('end', () => { + expect(buf).to.contain(testfile.toString()) + done() + }) + }) + }) + + it.skip('get BIG file', (done) => { + if (!isNode) { + return done() + } + + apiClients.a.get('Qme79tX2bViL26vNjPsF3DP1R9rMKMvnPYJiKTTKPrXJjq', (err, res) => { + expect(err).to.not.exist + + // Do not blow out the memory of nodejs :) + streamEqual(res, testfileBig, (err, equal) => { + expect(err).to.not.exist + expect(equal).to.be.true + done() + }) + }) + }) + + describe('promise', () => { + it.skip('get', (done) => { + return apiClients.a.get('Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP') + .then((res) => { + let buf = '' + res + .on('error', (err) => { + throw err + }) + .on('data', (data) => { + buf += data + }) + .on('end', () => { + expect(buf).to.contain(testfile.toString()) + done() + }) + }) + }) + }) +}) From c238f43b15084c75c241a7e0422234a43cfd486b Mon Sep 17 00:00:00 2001 From: nginnever Date: Fri, 13 May 2016 09:23:01 -0700 Subject: [PATCH 2/5] cr --- src/api/get.js | 23 ++++------------ test/api/get.spec.js | 62 ++++++++++++++++---------------------------- 2 files changed, 27 insertions(+), 58 deletions(-) diff --git a/src/api/get.js b/src/api/get.js index f2d122652..aabaabba0 100644 --- a/src/api/get.js +++ b/src/api/get.js @@ -1,24 +1,11 @@ 'use strict' module.exports = (send) => { - return function get (path, archive, compress, compressionLevel, cb) { - if (archive === true && typeof compress === 'function') { - cb = compress - compressionLevel = null - compress = null + return function get (path, opts, cb) { + if (typeof opts === 'function' && !cb) { + cb = opts + opts = {} } - if (archive === true && typeof compress === 'number') { - archive = null - cb = compressionLevel - compressionLevel = compress - compress = true - } - if (typeof archive === 'function') { - cb = archive - archive = null - compressionLevel = null - compress = null - } - return send('get', path, [archive, compress, compressionLevel], null, cb) + return send('get', path, opts, null, cb) } } diff --git a/test/api/get.spec.js b/test/api/get.spec.js index 9674f6476..82278616a 100644 --- a/test/api/get.spec.js +++ b/test/api/get.spec.js @@ -5,6 +5,7 @@ const expect = require('chai').expect const isNode = require('detect-node') const fs = require('fs') +const bl = require('bl') const path = require('path') const streamEqual = require('stream-equal') @@ -24,59 +25,40 @@ describe('.get', () => { apiClients.a .get('Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP', (err, res) => { expect(err).to.not.exist - - let buf = '' - res - .on('error', (err) => { - expect(err).to.not.exist - }) - .on('data', (data) => { - buf += data - }) - .on('end', () => { - expect(buf).to.contain(testfile.toString()) - done() - }) + res.pipe(bl((err, bldata) => { + expect(err).to.not.exist + expect(bldata.toString()).to.contain(testfile.toString()) + done() + })) }) }) it('get with archive true', (done) => { apiClients.a - .get('Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP', true, (err, res) => { + .get('Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP', {archive: true}, (err, res) => { expect(err).to.not.exist + res.pipe(bl((err, bldata) => { + expect(err).to.not.exist + expect(bldata.toString()).to.contain(testfile.toString()) + done() + })) + }) + }) - let buf = '' - res - .on('error', (err) => { - expect(err).to.not.exist - }) - .on('data', (data) => { - buf += data - }) - .on('end', () => { - expect(buf).to.contain(testfile.toString()) - done() - }) + it('get err with out of range compression level', (done) => { + apiClients.a + .get('Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP', {compress: true, 'compression-level': 10}, (err, res) => { + expect(err).to.exist + expect(err.toString()).to.equal('Error: Compression level must be between 1 and 9') + done() }) }) it('get with compression level', (done) => { apiClients.a - .get('Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP', true, 1, (err, res) => { + .get('Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP', {compress: true, 'compression-level': 1}, (err, res) => { expect(err).to.not.exist - - let buf = '' - res - .on('error', (err) => { - expect(err).to.not.exist - }) - .on('data', (data) => { - buf += data - }) - .on('end', () => { - expect(buf).to.contain(testfile.toString()) - done() - }) + done() }) }) From 4611a77c98d2dba792366b266e29c0c170c49fb9 Mon Sep 17 00:00:00 2001 From: Stephen Whitmore Date: Tue, 14 Jun 2016 13:18:39 -0700 Subject: [PATCH 3/5] Move cleanMultihash into a module. --- src/api/cat.js | 13 +------------ src/clean-multihash.js | 15 +++++++++++++++ 2 files changed, 16 insertions(+), 12 deletions(-) create mode 100644 src/clean-multihash.js diff --git a/src/api/cat.js b/src/api/cat.js index 8b61f66a1..06e9bac90 100644 --- a/src/api/cat.js +++ b/src/api/cat.js @@ -1,8 +1,7 @@ 'use strict' -const bs58 = require('bs58') -const isIPFS = require('is-ipfs') const promisify = require('promisify-es6') +const cleanMultihash = require('../clean-multihash') module.exports = (send) => { const cat = promisify((multihash, callback) => { @@ -15,13 +14,3 @@ module.exports = (send) => { }) return cat } - -function cleanMultihash (multihash) { - if (!isIPFS.multihash(multihash)) { - throw new Error('not valid multihash') - } - if (Buffer.isBuffer(multihash)) { - return bs58.encode(multihash) - } - return multihash -} diff --git a/src/clean-multihash.js b/src/clean-multihash.js new file mode 100644 index 000000000..bbf3f9a39 --- /dev/null +++ b/src/clean-multihash.js @@ -0,0 +1,15 @@ +'use strict' + +const bs58 = require('bs58') +const isIPFS = require('is-ipfs') + +module.exports = function (multihash) { + if (!isIPFS.multihash(multihash)) { + throw new Error('not valid multihash') + } + if (Buffer.isBuffer(multihash)) { + return bs58.encode(multihash) + } + return multihash +} + From aebb37e4e640ac830b46443e7a9f3bd973499296 Mon Sep 17 00:00:00 2001 From: Stephen Whitmore Date: Tue, 14 Jun 2016 19:13:34 -0700 Subject: [PATCH 4/5] Add files.get command and tests. --- src/api/get.js | 25 +++++++++++++++++++--- src/load-commands.js | 6 ++++++ src/tar-stream-to-objects.js | 32 +++++++++++++++++++++++++++++ test/api/get.spec.js | 40 ++++++++++++++++++++++++++++-------- 4 files changed, 92 insertions(+), 11 deletions(-) create mode 100644 src/tar-stream-to-objects.js diff --git a/src/api/get.js b/src/api/get.js index aabaabba0..b1b833d9d 100644 --- a/src/api/get.js +++ b/src/api/get.js @@ -1,11 +1,30 @@ 'use strict' +const tarStreamToObjects = require('../tar-stream-to-objects') +const cleanMultihash = require('../clean-multihash') +const promisify = require('promisify-es6') + module.exports = (send) => { - return function get (path, opts, cb) { + return promisify(function get (path, opts, cb) { if (typeof opts === 'function' && !cb) { cb = opts opts = {} } - return send('get', path, opts, null, cb) - } + + // opts is the real callback -- 'cb' is being injected by promisify + if (typeof opts === 'function' && typeof cb === 'function') { + cb = opts + opts = {} + } + + try { + path = cleanMultihash(path) + } catch (err) { + return cb(err) + } + + var sendWithTransform = send.withTransform(tarStreamToObjects) + + return sendWithTransform('get', path, opts, null, cb) + }) } diff --git a/src/load-commands.js b/src/load-commands.js index 95c73d6df..e9b6072cd 100644 --- a/src/load-commands.js +++ b/src/load-commands.js @@ -33,6 +33,12 @@ function requireCommands () { const files = require('./api/files')(send) files.add = require('./api/add')(send) files.createAddStream = require('./api/add-stream.js')(send) + files.get = require('./api/get')(send) + + // aliases + cmds.add = files.add + cmds.createAddStream = files.createAddStream + cmds.get = files.get return files } diff --git a/src/tar-stream-to-objects.js b/src/tar-stream-to-objects.js new file mode 100644 index 000000000..9e8ba0c51 --- /dev/null +++ b/src/tar-stream-to-objects.js @@ -0,0 +1,32 @@ +'use strict' + +const tar = require('tar-stream') +const Readable = require('readable-stream') +const through = require('through2') + +// transform tar stream into readable stream of +// { path: 'string', content: Readable } +module.exports = function (err, res, send, done) { + + if (err) return done(err) + + var ex = tar.extract() + res.pipe(ex) + + var objStream = new Readable({ objectMode: true }) + objStream._read = function noop () {} + + ex.on('entry', function (header, stream, next) { + objStream.push({ + path: header.name, + content: stream + }) + next() + }) + ex.on('finish', () => { + objStream.push(null) + }) + + done(null, objStream) +} + diff --git a/test/api/get.spec.js b/test/api/get.spec.js index 82278616a..30f9a8479 100644 --- a/test/api/get.spec.js +++ b/test/api/get.spec.js @@ -6,10 +6,14 @@ const expect = require('chai').expect const isNode = require('detect-node') const fs = require('fs') const bl = require('bl') +const concat = require('concat-stream') +const through = require('through2') const path = require('path') const streamEqual = require('stream-equal') +const extract = require('tar-stream').extract + let testfile let testfileBig @@ -24,10 +28,20 @@ describe('.get', () => { it('get with no compression args', (done) => { apiClients.a .get('Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP', (err, res) => { - expect(err).to.not.exist - res.pipe(bl((err, bldata) => { - expect(err).to.not.exist - expect(bldata.toString()).to.contain(testfile.toString()) + + // accumulate the files and their content + var files = [] + res.pipe(through.obj((file, enc, next) => { + file.content.pipe(concat((content) => { + files.push({ + path: file.path, + content: content + }) + next() + })) + }, () => { + expect(files).to.be.length(1) + expect(files[0].content.toString()).to.contain(testfile.toString()) done() })) }) @@ -36,10 +50,20 @@ describe('.get', () => { it('get with archive true', (done) => { apiClients.a .get('Qma4hjFTnCasJ8PVp3mZbZK5g2vGDT4LByLJ7m8ciyRFZP', {archive: true}, (err, res) => { - expect(err).to.not.exist - res.pipe(bl((err, bldata) => { - expect(err).to.not.exist - expect(bldata.toString()).to.contain(testfile.toString()) + + // accumulate the files and their content + var files = [] + res.pipe(through.obj((file, enc, next) => { + file.content.pipe(concat((content) => { + files.push({ + path: file.path, + content: content + }) + next() + })) + }, () => { + expect(files).to.be.length(1) + expect(files[0].content.toString()).to.contain(testfile.toString()) done() })) }) From 5dcc4a0cd6681a57f078f94172b4e7cd00cb25cf Mon Sep 17 00:00:00 2001 From: Stephen Whitmore Date: Thu, 23 Jun 2016 12:19:11 -0700 Subject: [PATCH 5/5] Set content to null for directories. --- src/tar-stream-to-objects.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/tar-stream-to-objects.js b/src/tar-stream-to-objects.js index 9e8ba0c51..888aca797 100644 --- a/src/tar-stream-to-objects.js +++ b/src/tar-stream-to-objects.js @@ -19,7 +19,7 @@ module.exports = function (err, res, send, done) { ex.on('entry', function (header, stream, next) { objStream.push({ path: header.name, - content: stream + content: header.type !== 'directory' ? stream : null }) next() })