Skip to content
This repository was archived by the owner on Feb 12, 2024. It is now read-only.

Commit e0b69a8

Browse files
daviddiasdaviddias
authored
Merge pull request #469 from ipfs/pull-add
Upgrade the files branch (#323) to work with pull-streams
2 parents 1c2ca57 + 4050195 commit e0b69a8

File tree

12 files changed

+203
-71
lines changed

12 files changed

+203
-71
lines changed

‎package.json

+1
Original file line numberDiff line numberDiff line change
@@ -96,6 +96,7 @@
9696
"promisify-es6": "^1.0.1",
9797
"pull-file": "^1.0.0",
9898
"pull-paramap": "^1.1.6",
99+
"pull-pushable": "^2.0.1",
99100
"pull-sort": "^1.0.0",
100101
"pull-stream": "^3.4.5",
101102
"pull-stream-to-stream": "^1.3.3",

‎src/core/ipfs/files.js

+24-17
Original file line numberDiff line numberDiff line change
@@ -50,23 +50,26 @@ module.exports = function files (self) {
5050
return callback(new Error('You must supply a multihash'))
5151
}
5252

53-
pull(
54-
pull.values([hash]),
55-
pull.asyncMap(self._dagS.get.bind(self._dagS)),
56-
pull.map((node) => {
57-
const data = UnixFS.unmarshal(node.data)
58-
if (data.type === 'directory') {
59-
return pull.error(new Error('This dag node is a directory'))
60-
}
61-
62-
return exporter(hash, self._dagS)
63-
}),
64-
pull.flatten(),
65-
pull.collect((err, files) => {
66-
if (err) return callback(err)
67-
callback(null, toStream.source(files[0].content))
68-
})
69-
)
53+
self._dagS.get(hash, (err, node) => {
54+
if (err) {
55+
return callback(err)
56+
}
57+
58+
const data = UnixFS.unmarshal(node.data)
59+
if (data.type === 'directory') {
60+
return callback(
61+
new Error('This dag node is a directory')
62+
)
63+
}
64+
65+
pull(
66+
exporter(hash, self._dagS),
67+
pull.collect((err, files) => {
68+
if (err) return callback(err)
69+
callback(null, toStream.source(files[0].content))
70+
})
71+
)
72+
})
7073
}),
7174

7275
get: promisify((hash, callback) => {
@@ -81,6 +84,10 @@ module.exports = function files (self) {
8184
return file
8285
})
8386
)))
87+
}),
88+
89+
getPull: promisify((hash, callback) => {
90+
callback(null, exporter(hash, self._dagS))
8491
})
8592
}
8693
}

‎src/core/ipfs/object.js

-1
Original file line numberDiff line numberDiff line change
@@ -155,7 +155,6 @@ module.exports = function object (self) {
155155
if (err) {
156156
return cb(err)
157157
}
158-
159158
cb(null, node.data)
160159
})
161160
}),

‎src/http-api/resources/files.js

+144-3
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,16 @@
11
'use strict'
22

33
const bs58 = require('bs58')
4+
const multipart = require('ipfs-multipart')
45
const debug = require('debug')
6+
const tar = require('tar-stream')
57
const log = debug('http-api:files')
68
log.error = debug('http-api:files:error')
9+
const pull = require('pull-stream')
10+
const toStream = require('pull-stream-to-stream')
11+
const toPull = require('stream-to-pull-stream')
12+
const pushable = require('pull-pushable')
13+
const EOL = require('os').EOL
714

815
exports = module.exports
916

@@ -33,18 +40,152 @@ exports.cat = {
3340
// main route handler which is called after the above `parseArgs`, but only if the args were valid
3441
handler: (request, reply) => {
3542
const key = request.pre.args.key
43+
const ipfs = request.server.app.ipfs
3644

37-
request.server.app.ipfs.files.cat(key, (err, stream) => {
45+
ipfs.files.cat(key, (err, stream) => {
3846
if (err) {
3947
log.error(err)
4048
return reply({
4149
Message: 'Failed to cat file: ' + err,
4250
Code: 0
4351
}).code(500)
4452
}
45-
stream.on('data', (data) => {
46-
return reply(data)
53+
54+
// hapi is not very clever and throws if no
55+
// - _read method
56+
// - _readableState object
57+
// are there :(
58+
stream._read = () => {}
59+
stream._readableState = {}
60+
return reply(stream).header('X-Stream-Output', '1')
61+
})
62+
}
63+
}
64+
65+
exports.get = {
66+
// uses common parseKey method that returns a `key`
67+
parseArgs: exports.parseKey,
68+
69+
// main route handler which is called after the above `parseArgs`, but only if the args were valid
70+
handler: (request, reply) => {
71+
const key = request.pre.args.key
72+
const ipfs = request.server.app.ipfs
73+
const pack = tar.pack()
74+
75+
ipfs.files.getPull(key, (err, stream) => {
76+
if (err) {
77+
log.error(err)
78+
79+
reply({
80+
Message: 'Failed to get file: ' + err,
81+
Code: 0
82+
}).code(500)
83+
return
84+
}
85+
86+
pull(
87+
stream,
88+
pull.asyncMap((file, cb) => {
89+
const header = {name: file.path}
90+
91+
if (!file.content) {
92+
header.type = 'directory'
93+
pack.entry(header)
94+
cb()
95+
} else {
96+
header.size = file.size
97+
toStream.source(file.content)
98+
.pipe(pack.entry(header, cb))
99+
}
100+
}),
101+
pull.onEnd((err) => {
102+
if (err) {
103+
log.error(err)
104+
105+
reply({
106+
Message: 'Failed to get file: ' + err,
107+
Code: 0
108+
}).code(500)
109+
return
110+
}
111+
112+
pack.finalize()
113+
})
114+
)
115+
116+
// the reply must read the tar stream,
117+
// to pull values through
118+
reply(pack).header('X-Stream-Output', '1')
119+
})
120+
}
121+
}
122+
123+
exports.add = {
124+
handler: (request, reply) => {
125+
if (!request.payload) {
126+
return reply('Array, Buffer, or String is required.').code(400).takeover()
127+
}
128+
129+
const ipfs = request.server.app.ipfs
130+
// TODO: make pull-multipart
131+
const parser = multipart.reqParser(request.payload)
132+
let filesParsed = false
133+
134+
const fileAdder = pushable()
135+
136+
parser.on('file', (fileName, fileStream) => {
137+
const filePair = {
138+
path: fileName,
139+
content: toPull(fileStream)
140+
}
141+
filesParsed = true
142+
fileAdder.push(filePair)
143+
})
144+
145+
parser.on('directory', (directory) => {
146+
fileAdder.push({
147+
path: directory,
148+
content: ''
47149
})
48150
})
151+
152+
parser.on('end', () => {
153+
if (!filesParsed) {
154+
return reply("File argument 'data' is required.")
155+
.code(400).takeover()
156+
}
157+
fileAdder.end()
158+
})
159+
160+
pull(
161+
fileAdder,
162+
ipfs.files.createAddPullStream(),
163+
pull.map((file) => {
164+
return {
165+
Name: file.path ? file.path : file.hash,
166+
Hash: file.hash
167+
}
168+
}),
169+
pull.map((file) => JSON.stringify(file) + EOL),
170+
pull.collect((err, files) => {
171+
if (err) {
172+
return reply({
173+
Message: err,
174+
Code: 0
175+
}).code(500)
176+
}
177+
178+
if (files.length === 0 && filesParsed) {
179+
return reply({
180+
Message: 'Failed to add files.',
181+
Code: 0
182+
}).code(500)
183+
}
184+
185+
reply(files.join(''))
186+
.header('x-chunked-output', '1')
187+
.header('content-type', 'application/json')
188+
})
189+
)
49190
}
50191
}

‎src/http-api/routes/files.js

+26
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ module.exports = (server) => {
66
const api = server.select('API')
77

88
api.route({
9+
// TODO fix method
910
method: '*',
1011
path: '/api/v0/cat',
1112
config: {
@@ -15,4 +16,29 @@ module.exports = (server) => {
1516
handler: resources.files.cat.handler
1617
}
1718
})
19+
20+
api.route({
21+
// TODO fix method
22+
method: '*',
23+
path: '/api/v0/get',
24+
config: {
25+
pre: [
26+
{ method: resources.files.get.parseArgs, assign: 'args' }
27+
],
28+
handler: resources.files.get.handler
29+
}
30+
})
31+
32+
api.route({
33+
// TODO fix method
34+
method: '*',
35+
path: '/api/v0/add',
36+
config: {
37+
payload: {
38+
parse: false,
39+
output: 'stream'
40+
},
41+
handler: resources.files.add.handler
42+
}
43+
})
1844
}

‎test/core/both/test-init.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -13,7 +13,7 @@ describe('init', function () {
1313
const repo = createTempRepo()
1414
const ipfs = new IPFS(repo)
1515

16-
ipfs.init({ emptyRepo: true }, (err) => {
16+
ipfs.init({ emptyRepo: true, bits: 128 }, (err) => {
1717
expect(err).to.not.exist
1818

1919
repo.exists((err, res) => {

‎test/http-api/interface-ipfs-core-over-ipfs-api/test-files.js

+1-5
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@
22

33
'use strict'
44

5-
/*
65
const test = require('interface-ipfs-core')
76
const FactoryClient = require('./../../utils/factory-http')
87

@@ -17,8 +16,5 @@ const common = {
1716
fc.dismantle(callback)
1817
}
1918
}
20-
*/
2119

22-
// TODO
23-
// needs: https://github.com/ipfs/js-ipfs/pull/323
24-
// test.files(common)
20+
test.files(common)

‎test/http-api/ipfs-api/test-files.js

-38
This file was deleted.

‎test/utils/factory-core/index.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -36,7 +36,7 @@ function Factory () {
3636

3737
if (!config) {
3838
config = JSON.parse(JSON.stringify(defaultConfig))
39-
const pId = PeerId.create({ bits: 32 }).toJSON()
39+
const pId = PeerId.create({ bits: 512 }).toJSON()
4040
config.Identity.PeerID = pId.id
4141
config.Identity.PrivKey = pId.privKey
4242
}
@@ -69,7 +69,7 @@ function Factory () {
6969

7070
// create the IPFS node
7171
const ipfs = new IPFS(repo)
72-
ipfs.init({ emptyRepo: true }, (err) => {
72+
ipfs.init({ emptyRepo: true, bits: 512 }, (err) => {
7373
if (err) {
7474
return callback(err)
7575
}

‎test/utils/factory-http/index.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,7 @@ function Factory () {
3737

3838
if (!config) {
3939
config = JSON.parse(JSON.stringify(defaultConfig))
40-
const pId = PeerId.create({ bits: 32 }).toJSON()
40+
const pId = PeerId.create({ bits: 512 }).toJSON()
4141
config.Identity.PeerID = pId.id
4242
config.Identity.PrivKey = pId.privKey
4343
}
@@ -53,7 +53,7 @@ function Factory () {
5353

5454
// create the IPFS node
5555
const ipfs = new IPFS(repo)
56-
ipfs.init({ emptyRepo: true }, (err) => {
56+
ipfs.init({ emptyRepo: true, bits: 512 }, (err) => {
5757
if (err) {
5858
return callback(err)
5959
}

‎test/utils/temp-node.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ function createTempNode (num, callback) {
3131
num = leftPad(num, 3, 0)
3232

3333
series([
34-
(cb) => ipfs.init({ emptyRepo: true }, cb),
34+
(cb) => ipfs.init({ emptyRepo: true, bits: 512 }, cb),
3535
(cb) => setAddresses(repo, num, cb),
3636
(cb) => ipfs.load(cb)
3737
], (err) => {

0 commit comments

Comments
 (0)