Skip to content

Commit 2ddb76b

Browse files
dcodeIOdcodeIO
committed
CLI: Added an experimental --sparse option to limit pbjs output to actually referenced types within main files; Other: Added a few more common google types from google/api, see #433 
1 parent d246024 commit 2ddb76b

15 files changed

+561
-29
lines changed

Diff for: ‎.travis.yml

+7-5
Original file line numberDiff line numberDiff line change
@@ -5,17 +5,19 @@ node_js:
55
- 4.3.2
66
- 6
77
- 7
8+
branches:
9+
only: master
10+
script: npm test && npm run bench
11+
812
env: CXX=g++-4.8
913
addons:
1014
apt:
1115
sources: ubuntu-toolchain-r-test
1216
packages: g++-4.8
13-
script: npm test && npm run bench
14-
branches:
15-
only: master
17+
1618
matrix:
1719
include:
1820
- node_js: 6
19-
script: set -e; if [ -n "$SAUCE_USERNAME" ]; then npm install [email protected] [email protected]; travis_wait npm run zuul; sleep 3; fi
21+
script: set -e; if [ -n "$SAUCE_USERNAME" ]; then npm install zuul@^3.11.1 zuul-ngrok@^4.0.0; travis_wait npm run zuul; sleep 3; fi
2022
- node_js: 6
21-
script: npm run coverage-ci
23+
script: npm install coveralls@^2.11.15; npm run coverage-ci

Diff for: ‎cli/pbjs.js

+101-15
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,7 @@ exports.main = function(args, callback) {
3131
lint : "l"
3232
},
3333
string: [ "target", "out", "path", "wrap", "root", "lint" ],
34-
boolean: [ "keep-case", "create", "encode", "decode", "verify", "convert", "delimited", "beautify", "comments", "es6" ],
34+
boolean: [ "keep-case", "create", "encode", "decode", "verify", "convert", "delimited", "beautify", "comments", "es6", "sparse" ],
3535
default: {
3636
target : "json",
3737
create : true,
@@ -51,6 +51,9 @@ exports.main = function(args, callback) {
5151
files = argv._,
5252
paths = typeof argv.path === "string" ? [ argv.path ] : argv.path || [];
5353

54+
// protobuf.js package directory contains additional, otherwise non-bundled google types
55+
paths.push(path.relative(process.cwd(), path.join(__dirname, "..")) || ".");
56+
5457
if (!files.length) {
5558
var descs = Object.keys(targets).filter(function(key) { return !targets[key].private; }).map(function(key) {
5659
return " " + util.pad(key, 14, true) + targets[key].description;
@@ -71,6 +74,8 @@ exports.main = function(args, callback) {
7174
"",
7275
" -o, --out Saves to a file instead of writing to stdout.",
7376
"",
77+
" --sparse Exports only those types referenced from a main file (experimental).",
78+
"",
7479
chalk.bold.gray(" Module targets only:"),
7580
"",
7681
" -w, --wrap Specifies the wrapper to use. Also accepts a path to require a custom wrapper.",
@@ -124,17 +129,33 @@ exports.main = function(args, callback) {
124129

125130
var root = new protobuf.Root();
126131

132+
var mainFiles = [];
133+
127134
// Search include paths when resolving imports
128135
root.resolvePath = function pbjsResolvePath(origin, target) {
129-
var filepath = protobuf.util.path.resolve(origin, target);
130-
if (fs.existsSync(filepath))
131-
return filepath;
136+
var normOrigin = protobuf.util.path.normalize(origin),
137+
normTarget = protobuf.util.path.normalize(target);
138+
if (!normOrigin)
139+
mainFiles.push(normTarget);
140+
141+
var resolved = protobuf.util.path.resolve(normOrigin, normTarget, true);
142+
var idx = resolved.lastIndexOf("google/protobuf/");
143+
if (idx > -1) {
144+
var altname = resolved.substring(idx);
145+
if (altname in protobuf.common)
146+
resolved = altname;
147+
}
148+
149+
if (fs.existsSync(resolved))
150+
return resolved;
151+
132152
for (var i = 0; i < paths.length; ++i) {
133-
var ifilepath = protobuf.util.path.resolve(paths[i] + "/", target);
134-
if (fs.existsSync(ifilepath))
135-
return ifilepath;
153+
var iresolved = protobuf.util.path.resolve(paths[i] + "/", target);
154+
if (fs.existsSync(iresolved))
155+
return iresolved;
136156
}
137-
return filepath;
157+
158+
return resolved;
138159
};
139160

140161
// Use es6 syntax if not explicitly specified on the command line and the es6 wrapper is used
@@ -153,19 +174,28 @@ exports.main = function(args, callback) {
153174
});
154175
process.stdin.on("end", function() {
155176
var source = Buffer.concat(data).toString("utf8");
156-
if (source.charAt(0) !== "{") {
157-
protobuf.parse(source, root, parseOptions);
158-
} else {
159-
var json = JSON.parse(source);
160-
root.setOptions(json.options).addJSON(json);
177+
try {
178+
if (source.charAt(0) !== "{") {
179+
protobuf.parse.filename = "-";
180+
protobuf.parse(source, root, parseOptions);
181+
} else {
182+
var json = JSON.parse(source);
183+
root.setOptions(json.options).addJSON(json);
184+
}
185+
callTarget();
186+
} catch (err) {
187+
if (callback)
188+
return callback(err);
189+
throw err;
161190
}
162-
callTarget();
163191
});
164192

165193
// Load from disk
166194
} else {
167195
try {
168-
root.loadSync(files, parseOptions); // sync is deterministic while async is not
196+
root.loadSync(files, parseOptions).resolveAll(); // sync is deterministic while async is not
197+
if (argv.sparse)
198+
sparsify(root);
169199
callTarget();
170200
} catch (err) {
171201
if (callback) {
@@ -176,6 +206,62 @@ exports.main = function(args, callback) {
176206
}
177207
}
178208

209+
function markReferenced(tobj) {
210+
tobj.referenced = true;
211+
// also mark a type's fields and oneofs
212+
if (tobj.fieldsArray)
213+
tobj.fieldsArray.forEach(function(fobj) {
214+
fobj.referenced = true;
215+
});
216+
if (tobj.oneofsArray)
217+
tobj.oneofsArray.forEach(function(oobj) {
218+
oobj.referenced = true;
219+
});
220+
// also mark an extension field's extended type, but not its (other) fields
221+
if (tobj.extensionField)
222+
tobj.extensionField.parent.referenced = true;
223+
}
224+
225+
function sparsify(root) {
226+
227+
// 1. mark directly or indirectly referenced objects
228+
util.traverse(root, function(obj) {
229+
if (!obj.filename)
230+
return;
231+
if (mainFiles.indexOf(obj.filename) > -1)
232+
util.traverseResolved(obj, markReferenced);
233+
});
234+
235+
// 2. empty unreferenced objects
236+
util.traverse(root, function(obj) {
237+
var parent = obj.parent;
238+
if (!parent || obj.referenced) // root or referenced
239+
return;
240+
// remove unreferenced namespaces
241+
if (obj instanceof protobuf.Namespace) {
242+
var hasReferenced = false;
243+
util.traverse(obj, function(iobj) {
244+
if (iobj.referenced)
245+
hasReferenced = true;
246+
});
247+
if (hasReferenced) { // replace with plain namespace if a namespace subclass
248+
if (obj instanceof protobuf.Type || obj instanceof protobuf.Service) {
249+
var robj = new protobuf.Namespace(obj.name, obj.options);
250+
robj.nested = obj.nested;
251+
parent.add(robj);
252+
}
253+
} else // remove completely if nothing inside is referenced
254+
parent.remove(obj);
255+
256+
// remove everything else unreferenced
257+
} else if (!(obj instanceof protobuf.Namespace))
258+
parent.remove(obj);
259+
});
260+
261+
// 3. validate that everything is fine
262+
root.resolveAll();
263+
}
264+
179265
function callTarget() {
180266
target(root, argv, function targetCallback(err, output) {
181267
if (err) {

Diff for: ‎cli/util.js

+32
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,38 @@ exports.requireAll = function requireAll(dirname) {
3333
return all;
3434
};
3535

36+
exports.traverse = function traverse(current, fn) {
37+
fn(current);
38+
if (current.fieldsArray)
39+
current.fieldsArray.forEach(function(field) {
40+
traverse(field, fn);
41+
});
42+
if (current.oneofsArray)
43+
current.oneofsArray.forEach(function(oneof) {
44+
traverse(oneof, fn);
45+
});
46+
if (current.methodsArray)
47+
current.methodsArray.forEach(function(method) {
48+
traverse(method, fn);
49+
});
50+
if (current.nestedArray)
51+
current.nestedArray.forEach(function(nested) {
52+
traverse(nested, fn);
53+
});
54+
};
55+
56+
exports.traverseResolved = function traverseResolved(current, fn) {
57+
fn(current);
58+
if (current.resolvedType)
59+
traverseResolved(current.resolvedType, fn);
60+
if (current.resolvedKeyType)
61+
traverseResolved(current.resolvedKeyType, fn);
62+
if (current.resolvedRequestType)
63+
traverseResolved(current.resolvedRequestType, fn);
64+
if (current.resolvedResponseType)
65+
traverseResolved(current.resolvedResponseType, fn);
66+
};
67+
3668
exports.inspect = function inspect(object, indent) {
3769
if (!object)
3870
return "";

Diff for: ‎google/api/annotations.json

+83
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,83 @@
1+
{
2+
"nested": {
3+
"google": {
4+
"nested": {
5+
"api": {
6+
"nested": {
7+
"http": {
8+
"type": "HttpRule",
9+
"id": 72295728,
10+
"extend": "google.protobuf.MethodOptions"
11+
},
12+
"HttpRule": {
13+
"oneofs": {
14+
"pattern": {
15+
"oneof": [
16+
"get",
17+
"put",
18+
"post",
19+
"delete",
20+
"patch",
21+
"custom"
22+
]
23+
}
24+
},
25+
"fields": {
26+
"get": {
27+
"type": "string",
28+
"id": 2
29+
},
30+
"put": {
31+
"type": "string",
32+
"id": 3
33+
},
34+
"post": {
35+
"type": "string",
36+
"id": 4
37+
},
38+
"delete": {
39+
"type": "string",
40+
"id": 5
41+
},
42+
"patch": {
43+
"type": "string",
44+
"id": 6
45+
},
46+
"custom": {
47+
"type": "CustomHttpPattern",
48+
"id": 8
49+
},
50+
"selector": {
51+
"type": "string",
52+
"id": 1
53+
},
54+
"body": {
55+
"type": "string",
56+
"id": 7
57+
},
58+
"additionalBindings": {
59+
"rule": "repeated",
60+
"type": "HttpRule",
61+
"id": 11
62+
}
63+
}
64+
}
65+
}
66+
},
67+
"protobuf": {
68+
"nested": {
69+
"MethodOptions": {
70+
"fields": {},
71+
"extensions": [
72+
[
73+
1000,
74+
536870911
75+
]
76+
]
77+
}
78+
}
79+
}
80+
}
81+
}
82+
}
83+
}

Diff for: ‎google/api/annotations.proto

+11
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
syntax = "proto3";
2+
3+
package google.api;
4+
5+
import "google/api/http.proto";
6+
import "google/protobuf/descriptor.proto";
7+
8+
extend google.protobuf.MethodOptions {
9+
10+
HttpRule http = 72295728;
11+
}

0 commit comments

Comments
 (0)