Skip to content

Commit 74b2c5c

Browse files
dcodeIOdcodeIO
committed
Process huge strings in chunks, see #555; Run bench in tests for a while; Raw alloc benchmark
1 parent ed86f3a commit 74b2c5c

21 files changed

+477
-109
lines changed

‎.travis.yml

+4
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,10 @@ node_js:
66
- 6
77
- 7
88

9+
script:
10+
- npm test
11+
- npm run bench
12+
913
branches:
1014
only:
1115
- master

‎README.md

+4-4
Original file line numberDiff line numberDiff line change
@@ -81,7 +81,7 @@ Production:
8181

8282
Or [download](https://github.com/dcodeIO/protobuf.js/tree/master/dist) the library.
8383

84-
The `protobuf` namespace will always be available globally but also supports AMD.
84+
The `protobuf` namespace will always be available globally / also supports AMD loaders.
8585

8686
Examples
8787
--------
@@ -354,11 +354,11 @@ usage: pbts [options] file1.js file2.js ...
354354

355355
### Descriptors vs. static modules
356356

357-
While .proto and JSON files require the full library (about 18kb gzipped), pretty much all code but the relatively short descriptors is shared.
357+
While .proto and JSON files require the full library (about 18kb gzipped), pretty much all code but the relatively short descriptors is shared. Hence, it usually doesn't become much larger than that.
358358

359-
Static code, on the other hand, requires just the minimal runtime (about 5.5kb gzipped), but generates relatively large code bases without any reflection features.
359+
Static code, on the other hand, requires just the minimal runtime (about 5.5kb gzipped, no reflection features), but generates relatively large codebases that you can edit or strip down by hand.
360360

361-
When `new Function` is supported (and it usually is), there is no difference performance-wise as the code generated statically is the same generated at runtime.
361+
When `new Function` is supported (and it usually is), there is no difference performance-wise as the code generated statically is the same as generated at runtime.
362362

363363
Building
364364
--------

‎bench/alloc.js

+45
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
var protobuf = require(".."),
2+
newSuite = require("./suite");
3+
4+
var pool = require("../src/util/pool"),
5+
poolAlloc = pool(function(size) {
6+
return new Uint8Array(size);
7+
}, Uint8Array.prototype.subarray);
8+
9+
[
10+
64,
11+
256,
12+
1024
13+
]
14+
.forEach(function(size) {
15+
16+
newSuite("buffer[" + size + "]")
17+
.add("new Uint8Array", function() {
18+
var buf = new Uint8Array(size);
19+
})
20+
.add("Buffer.alloc", function() {
21+
var buf = Buffer.alloc(size);
22+
})
23+
.add("poolAlloc<Uint8Array>", function() {
24+
var buf = poolAlloc(size);
25+
})
26+
.add("Buffer.allocUnsafe", function() {
27+
var buf = Buffer.allocUnsafe(size);
28+
})
29+
.add("new Buffer", function() {
30+
var buf = new Buffer(size);
31+
})
32+
.run();
33+
34+
var ab = new ArrayBuffer(size);
35+
36+
newSuite("wrap[" + size + "]")
37+
.add("new Uint8Array(ArrayBuffer)", function() {
38+
var buf = new Uint8Array(ab);
39+
})
40+
.add("Buffer.from(ArrayBuffer)", function() {
41+
var buf = Buffer.from(ab);
42+
})
43+
.run();
44+
45+
});

‎bench/index.js

+8-3
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
var protobuf = require("../src/index"),
1+
var protobuf = require(".."),
22
newSuite = require("./suite"),
33
data = require("./bench.json");
44

@@ -35,6 +35,11 @@ for (var i = 0; i < 500000; ++i)
3535
Test.verify(data);
3636
console.log("");
3737

38+
if (!Buffer.from)
39+
Buffer.from = function from(str, enc) {
40+
return new Buffer(str, enc);
41+
};
42+
3843
// give the optimizer some time to do its job
3944
setTimeout(function() {
4045
var str = JSON.stringify(data),
@@ -48,7 +53,7 @@ setTimeout(function() {
4853
JSON.stringify(data);
4954
})
5055
.add("JSON.stringify to buffer", function() {
51-
new Buffer(JSON.stringify(data), "utf8");
56+
Buffer.from(JSON.stringify(data), "utf8");
5257
})
5358
.run();
5459

@@ -72,7 +77,7 @@ setTimeout(function() {
7277
JSON.parse(JSON.stringify(data));
7378
})
7479
.add("JSON to/from buffer", function() {
75-
JSON.parse(new Buffer(JSON.stringify(data), "utf8").toString("utf8"));
80+
JSON.parse(Buffer.from(JSON.stringify(data), "utf8").toString("utf8"));
7681
})
7782
.run();
7883

‎bench/read.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
var protobuf = require("../src/index"),
1+
var protobuf = require(".."),
22
newSuite = require("./suite"),
33
ieee754 = require("../lib/ieee754");
44

‎bench/write.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
var protobuf = require("../src/index"),
1+
var protobuf = require(".."),
22
newSuite = require("./suite"),
33
ieee754 = require("../lib/ieee754");
44

‎cli/pbjs.js

+7
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,13 @@ exports.main = function(args) {
5454
"",
5555
" -r, --root Specifies an alternative protobuf.roots name for *-module targets.",
5656
"",
57+
" Static code generation only:",
58+
"",
59+
" --no-encode Does not generate encode functions.",
60+
" --no-decode Does not generate decode functions.",
61+
" --no-verify Does not generate verify functions.",
62+
" --no-delimited Does not generate delimited encode/decode functions.",
63+
"",
5764
"usage: " + chalk.bold.green(path.basename(process.argv[1])) + " [options] file1.proto file2.json ..."
5865
].join("\n"));
5966
return 1;

‎cli/targets/static.js

+84-62
Original file line numberDiff line numberDiff line change
@@ -13,10 +13,12 @@ var Type = protobuf.Type,
1313

1414
var out = [];
1515
var indent = 0;
16+
var config = {};
1617

1718
static_target.description = "Static code without reflection";
1819

1920
function static_target(root, options, callback) {
21+
config = options;
2022
try {
2123
push("// Lazily resolved type references");
2224
push("var $lazyTypes = [];");
@@ -51,6 +53,7 @@ function static_target(root, options, callback) {
5153
} finally {
5254
out = [];
5355
indent = 0;
56+
config = {};
5457
}
5558
}
5659

@@ -266,72 +269,91 @@ function buildType(ref, type) {
266269
push("});");
267270
});
268271

269-
// #encode
270-
push("");
271-
pushComment([
272-
"Encodes the specified " + type.name + ".",
273-
"@function",
274-
"@param {" + fullName + "|Object} message " + type.name + " or plain object to encode",
275-
"@param {Writer} [writer] Writer to encode to",
276-
"@returns {Writer} Writer"
277-
]);
278-
buildFunction(type, "encode", protobuf.encode.generate(type), {
279-
Writer : "$protobuf.Writer",
280-
util : "$protobuf.util"
281-
});
272+
if (config.encode !== false) {
282273

283-
// #encodeDelimited
284-
push("");
285-
pushComment([
286-
"Encodes the specified " + type.name + ", length delimited.",
287-
"@param {" + fullName + "|Object} message " + type.name + " or plain object to encode",
288-
"@param {Writer} [writer] Writer to encode to",
289-
"@returns {Writer} Writer"
290-
]);
291-
push(name(type.name) + ".encodeDelimited = function encodeDelimited(message, writer) {");
292-
++indent;
293-
push("return this.encode(message, writer).ldelim();");
294-
--indent;
295-
push("};");
274+
// #encode
275+
push("");
276+
pushComment([
277+
"Encodes the specified " + type.name + ".",
278+
"@function",
279+
"@param {" + fullName + "|Object} message " + type.name + " or plain object to encode",
280+
"@param {Writer} [writer] Writer to encode to",
281+
"@returns {Writer} Writer"
282+
]);
283+
buildFunction(type, "encode", protobuf.encode.generate(type), {
284+
Writer : "$protobuf.Writer",
285+
util : "$protobuf.util"
286+
});
296287

297-
// #decode
298-
push("");
299-
pushComment([
300-
"Decodes a " + type.name + " from the specified reader or buffer.",
301-
"@function",
302-
"@param {Reader|Uint8Array} readerOrBuffer Reader or buffer to decode from",
303-
"@param {number} [length] Message length if known beforehand",
304-
"@returns {" + fullName + "} " + type.name
305-
]);
306-
buildFunction(type, "decode", protobuf.decode.generate(type), {
307-
Reader : "$protobuf.Reader",
308-
util : "$protobuf.util"
309-
});
288+
if (config.delimited !== false) {
310289

311-
// #decodeDelimited
312-
push("");
313-
pushComment([
314-
"Decodes a " + type.name + " from the specified reader or buffer, length delimited.",
315-
"@param {Reader|Uint8Array} readerOrBuffer Reader or buffer to decode from",
316-
"@returns {" + fullName + "} " + type.name
317-
]);
318-
push(name(type.name) + ".decodeDelimited = function decodeDelimited(readerOrBuffer) {");
319-
++indent;
320-
push("readerOrBuffer = readerOrBuffer instanceof $protobuf.Reader ? readerOrBuffer : $protobuf.Reader(readerOrBuffer);");
321-
push("return this.decode(readerOrBuffer, readerOrBuffer.uint32());");
322-
--indent;
323-
push("};");
290+
// #encodeDelimited
291+
push("");
292+
pushComment([
293+
"Encodes the specified " + type.name + ", length delimited.",
294+
"@param {" + fullName + "|Object} message " + type.name + " or plain object to encode",
295+
"@param {Writer} [writer] Writer to encode to",
296+
"@returns {Writer} Writer"
297+
]);
298+
push(name(type.name) + ".encodeDelimited = function encodeDelimited(message, writer) {");
299+
++indent;
300+
push("return this.encode(message, writer).ldelim();");
301+
--indent;
302+
push("};");
324303

325-
// #verify
326-
push("");
327-
pushComment([
328-
"Verifies a " + type.name + ".",
329-
"@param {" + fullName + "|Object} message " + type.name + " or plain object to verify",
330-
"@returns {?string} `null` if valid, otherwise the reason why it is not"
331-
]);
332-
buildFunction(type, "verify", protobuf.verify.generate(type), {
333-
util : "$protobuf.util"
334-
});
304+
}
305+
306+
}
307+
308+
if (config.decode !== false) {
309+
310+
// #decode
311+
push("");
312+
pushComment([
313+
"Decodes a " + type.name + " from the specified reader or buffer.",
314+
"@function",
315+
"@param {Reader|Uint8Array} readerOrBuffer Reader or buffer to decode from",
316+
"@param {number} [length] Message length if known beforehand",
317+
"@returns {" + fullName + "} " + type.name
318+
]);
319+
buildFunction(type, "decode", protobuf.decode.generate(type), {
320+
Reader : "$protobuf.Reader",
321+
util : "$protobuf.util"
322+
});
323+
324+
if (config.delimited !== false) {
325+
326+
// #decodeDelimited
327+
push("");
328+
pushComment([
329+
"Decodes a " + type.name + " from the specified reader or buffer, length delimited.",
330+
"@param {Reader|Uint8Array} readerOrBuffer Reader or buffer to decode from",
331+
"@returns {" + fullName + "} " + type.name
332+
]);
333+
push(name(type.name) + ".decodeDelimited = function decodeDelimited(readerOrBuffer) {");
334+
++indent;
335+
push("readerOrBuffer = readerOrBuffer instanceof $protobuf.Reader ? readerOrBuffer : $protobuf.Reader(readerOrBuffer);");
336+
push("return this.decode(readerOrBuffer, readerOrBuffer.uint32());");
337+
--indent;
338+
push("};");
339+
340+
}
341+
}
342+
343+
if (config.verify !== false) {
344+
345+
// #verify
346+
push("");
347+
pushComment([
348+
"Verifies a " + type.name + ".",
349+
"@param {" + fullName + "|Object} message " + type.name + " or plain object to verify",
350+
"@returns {?string} `null` if valid, otherwise the reason why it is not"
351+
]);
352+
buildFunction(type, "verify", protobuf.verify.generate(type), {
353+
util : "$protobuf.util"
354+
});
355+
356+
}
335357
}
336358

337359
function buildService(ref, service) {

0 commit comments

Comments
 (0)