Skip to content

Commit d63626f

Browse files
authored
feat(broccoli): Add naive caching strategy for Broccoli. (#190)
- Aggregator Broc plugin now lives in Broccoli package. - Use fs-tree-diff to better keep input and output dirs in sync with minimal changes. - Don't do any work if input directory has not changed. - Add more robust tests for Analyze and Aggregate plugins with broccoli-test-helper. - Use typed versions of walk-sync and fs-tree-diff. - TODO: If input directory has changed, we still re-build the world. This can be improved.
1 parent 3179998 commit d63626f

File tree

19 files changed

+597
-342
lines changed

19 files changed

+597
-342
lines changed

package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -44,7 +44,7 @@
4444
"tslint": "^5.9.1",
4545
"typedoc": "^0.11.0",
4646
"typedoc-plugin-monorepo": "^0.1.0",
47-
"typescript": "~2.8.0",
47+
"typescript": "2.8",
4848
"watch": "^1.0.2"
4949
},
5050
"workspaces": [

packages/@css-blocks/broccoli/package.json

+6-3
Original file line numberDiff line numberDiff line change
@@ -29,23 +29,26 @@
2929
"devDependencies": {
3030
"@css-blocks/code-style": "^0.18.0",
3131
"@css-blocks/glimmer": "^0.19.0",
32+
"@types/glob": "^5.0.35",
33+
"broccoli-test-helper": "^1.4.0",
3234
"watch": "^1.0.2"
3335
},
3436
"dependencies": {
3537
"@css-blocks/core": "^0.19.0",
3638
"@glimmer/compiler": "^0.33.0",
3739
"@glimmer/syntax": "^0.33.0",
3840
"@opticss/template-api": "^0.3.0",
39-
"@types/recursive-readdir": "^2.2.0",
4041
"broccoli-funnel": "^2.0.1",
4142
"broccoli-merge-trees": "^3.0.0",
4243
"broccoli-plugin": "^1.3.0",
4344
"broccoli-test-helper": "^1.2.0",
4445
"colors": "^1.2.1",
4546
"debug": "^3.1.0",
4647
"fs-extra": "^5.0.0",
48+
"fs-tree-diff": "^0.5.9",
49+
"glob": "^7.1.2",
4750
"opticss": "^0.3.0",
48-
"recursive-readdir": "^2.2.2",
49-
"walk-sync": "^0.3.2"
51+
"symlink-or-copy": "^1.2.0",
52+
"walk-sync": "^0.3.3"
5053
}
5154
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
2+
import * as path from "path";
3+
4+
import * as fs from "fs-extra";
5+
import * as FSTree from "fs-tree-diff";
6+
import * as walkSync from "walk-sync";
7+
8+
import { Transport } from "./Transport";
9+
import { BroccoliPlugin } from "./utils";
10+
11+
// Common CSS preprocessor file endings to auto-discover
12+
const COMMON_FILE_ENDINGS = [".scss", ".sass", ".less", ".stylus"];
13+
14+
/**
15+
* Process-global dumping zone for CSS output as it comes through the pipeline 🤮
16+
* This will disappear once we have a functional language server and replaced
17+
* with a post-build step.
18+
*/
19+
export class CSSBlocksAggregate extends BroccoliPlugin {
20+
21+
private transport: Transport;
22+
private out: string;
23+
private _out = "";
24+
private previousCSS = "";
25+
private previous = new FSTree();
26+
27+
/**
28+
* Initialize this new instance with the app tree, transport, and analysis options.
29+
* @param inputNodes Broccoli trees who's output we depend on. First node must be the tree where stylesheets are placed.
30+
* @param transport Magical shared-memory Transport object shared with the aggregator and Template transformer.
31+
* @param out Output file name.
32+
*/
33+
// tslint:disable-next-line:prefer-whatever-to-any
34+
constructor(inputNodes: any[], transport: Transport, out: string) {
35+
super(inputNodes, {
36+
name: "broccoli-css-blocks-aggregate",
37+
persistentOutput: true,
38+
});
39+
this.transport = transport;
40+
this.out = out;
41+
}
42+
43+
/**
44+
* Re-run the broccoli build over supplied inputs.
45+
*/
46+
build() {
47+
let output = this.outputPath;
48+
let input = this.inputPaths[0];
49+
let { id, css } = this.transport;
50+
51+
// Test if anything has changed since last time. If not, skip trying to update tree.
52+
let newFsTree = FSTree.fromEntries(walkSync.entries(input));
53+
let diff = this.previous.calculatePatch(newFsTree);
54+
if (diff.length) {
55+
this.previous = newFsTree;
56+
FSTree.applyPatch(input, output, diff);
57+
}
58+
59+
// Auto-discover common preprocessor extensions.
60+
if (!this._out) {
61+
let prev = path.parse(path.join(input, this.out));
62+
let origExt = prev.ext;
63+
prev.base = ""; // Needed for path.format to register ext change
64+
for (let ending of COMMON_FILE_ENDINGS) {
65+
prev.ext = ending;
66+
if (fs.existsSync(path.format(prev))) { break; }
67+
prev.ext = origExt;
68+
}
69+
let out = path.parse(this.out);
70+
out.base = ""; // Needed for path.format to register ext change
71+
out.ext = prev.ext;
72+
this._out = path.format(out);
73+
}
74+
75+
let outHasChanged = !!diff.find((o) => o[1] === this._out);
76+
if (outHasChanged || this.previousCSS !== css) {
77+
let prev = path.join(input, this._out);
78+
let out = path.join(output, this._out);
79+
prev = fs.existsSync(prev) ? fs.readFileSync(prev).toString() : "";
80+
if (fs.existsSync(out)) { fs.unlinkSync(out); }
81+
fs.writeFileSync(out, `${prev}\n/* CSS Blocks Start: "${id}" */\n${css}\n/* CSS Blocks End: "${id}" */\n`);
82+
this.previousCSS = css;
83+
}
84+
}
85+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,142 @@
1+
import * as path from "path";
2+
3+
import { Analyzer, BlockCompiler, StyleMapping } from "@css-blocks/core";
4+
import { TemplateTypes } from "@opticss/template-api";
5+
6+
import * as debugGenerator from "debug";
7+
import * as fs from "fs-extra";
8+
import * as FSTree from "fs-tree-diff";
9+
import * as glob from "glob";
10+
import { OptiCSSOptions, Optimizer, postcss } from "opticss";
11+
import * as walkSync from "walk-sync";
12+
13+
import { Transport } from "./Transport";
14+
import { BroccoliPlugin, symlinkOrCopy } from "./utils";
15+
16+
const debug = debugGenerator("css-blocks:broccoli");
17+
18+
export interface BroccoliOptions {
19+
entry: string[];
20+
output: string;
21+
root: string;
22+
analyzer: Analyzer<keyof TemplateTypes>;
23+
optimization?: Partial<OptiCSSOptions>;
24+
}
25+
26+
/**
27+
* Runs analysis on an `inputNode` that represents the entire
28+
* application. `options.transport` will be populated with
29+
* analysis results. Output is the same application tree
30+
* with all Block files removed.
31+
*/
32+
export class CSSBlocksAnalyze extends BroccoliPlugin {
33+
34+
private analyzer: Analyzer<keyof TemplateTypes>;
35+
private entries: string[];
36+
private output: string;
37+
private root: string;
38+
private transport: Transport;
39+
private optimizationOptions: Partial<OptiCSSOptions>;
40+
private previous: FSTree = new FSTree();
41+
42+
/**
43+
* Initialize this new instance with the app tree, transport, and analysis options.
44+
* @param inputNode Single Broccoli tree node containing *entire* app.
45+
* @param transport Magical shared-memory Transport object shared with the aggregator and Template transformer.
46+
* @param options Analysis options.
47+
*/
48+
// tslint:disable-next-line:prefer-whatever-to-any
49+
constructor(inputNode: any, transport: Transport, options: BroccoliOptions) {
50+
super([inputNode], {
51+
name: "broccoli-css-blocks-analyze",
52+
persistentOutput: true,
53+
});
54+
this.transport = transport;
55+
this.entries = options.entry.slice(0);
56+
this.output = options.output || "css-blocks.css";
57+
this.optimizationOptions = options.optimization || {};
58+
this.analyzer = options.analyzer;
59+
this.root = options.root || process.cwd();
60+
this.transport.css = this.transport.css ? this.transport.css : "";
61+
}
62+
63+
/**
64+
* Re-run the broccoli build over supplied inputs.
65+
*/
66+
async build() {
67+
let input = this.inputPaths[0];
68+
let output = this.outputPath;
69+
let options = this.analyzer.cssBlocksOptions;
70+
let blockCompiler = new BlockCompiler(postcss, options);
71+
let optimizer = new Optimizer(this.optimizationOptions, this.analyzer.optimizationOptions);
72+
73+
// Test if anything has changed since last time. If not, skip all analysis work.
74+
let newFsTree = FSTree.fromEntries(walkSync.entries(input));
75+
let diff = this.previous.calculatePatch(newFsTree);
76+
if (!diff.length) { return; }
77+
this.previous = newFsTree;
78+
FSTree.applyPatch(input, output, diff);
79+
80+
// When no entry points are passed, we treat *every* template as an entry point.
81+
this.entries = this.entries.length ? this.entries : glob.sync("**/*.hbs", { cwd: input });
82+
83+
// The glimmer-analyzer package tries to require() package.json
84+
// in the root of the directory it is passed. We pass it our broccoli
85+
// tree, so it needs to contain package.json too.
86+
// TODO: Ideally this is configurable in glimmer-analyzer. We can
87+
// contribute that option back to the project. However,
88+
// other template integrations may want this available too...
89+
let pjsonLink = path.join(input, "package.json");
90+
if (!fs.existsSync(pjsonLink)) {
91+
symlinkOrCopy(path.join(this.root, "package.json"), pjsonLink);
92+
}
93+
94+
// Oh hey look, we're analyzing.
95+
this.analyzer.reset();
96+
this.transport.reset();
97+
await this.analyzer.analyze(input, this.entries);
98+
99+
// Compile all Blocks and add them as sources to the Optimizer.
100+
// TODO: handle a sourcemap from compiling the block file via a preprocessor.
101+
let blocks = this.analyzer.transitiveBlockDependencies();
102+
for (let block of blocks) {
103+
if (block.stylesheet) {
104+
let root = blockCompiler.compile(block, block.stylesheet, this.analyzer);
105+
let result = root.toResult({ to: this.output, map: { inline: false, annotation: false } });
106+
let filesystemPath = options.importer.filesystemPath(block.identifier, options);
107+
let filename = filesystemPath || options.importer.debugIdentifier(block.identifier, options);
108+
109+
// If this Block has a representation on disk, remove it from our output tree.
110+
if (filesystemPath) {
111+
let outputStylesheet = path.join(output, path.relative(input, filesystemPath));
112+
debug(`Removing block file ${outputStylesheet} from output.`);
113+
if (fs.existsSync(outputStylesheet)) { fs.removeSync(outputStylesheet); }
114+
}
115+
116+
// Add the compiled Block file to the optimizer.
117+
optimizer.addSource({
118+
content: result.css,
119+
filename,
120+
sourceMap: result.map.toJSON(),
121+
});
122+
}
123+
}
124+
125+
// Add each Analysis to the Optimizer.
126+
this.analyzer.eachAnalysis((a) => optimizer.addAnalysis(a.forOptimizer(options)));
127+
128+
// Run optimization and compute StyleMapping.
129+
let optimized = await optimizer.optimize(this.output);
130+
let styleMapping = new StyleMapping<keyof TemplateTypes>(optimized.styleMapping, blocks, options, this.analyzer.analyses());
131+
132+
// Attach all computed data to our magic shared memory transport object...
133+
this.transport.mapping = styleMapping;
134+
this.transport.blocks = blocks;
135+
this.transport.analyzer = this.analyzer;
136+
this.transport.css += optimized.output.content.toString();
137+
138+
debug(`Compilation Finished: ${this.transport.id}`);
139+
140+
}
141+
142+
}
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
import { Analyzer, Block, StyleMapping } from "@css-blocks/core";
2+
import { TemplateTypes } from "@opticss/template-api";
3+
4+
// Magic shared memory transport object 🤮
5+
// This will disappear once we have a functional language server.
6+
export class Transport {
7+
id: string;
8+
css = "";
9+
blocks: Set<Block> = new Set();
10+
mapping?: StyleMapping<keyof TemplateTypes>;
11+
analyzer?: Analyzer<keyof TemplateTypes>;
12+
13+
constructor(id: string) {
14+
this.id = id;
15+
this.reset();
16+
}
17+
18+
reset() {
19+
this.css = "";
20+
this.blocks = new Set();
21+
this.mapping = undefined;
22+
this.analyzer = undefined;
23+
}
24+
}

0 commit comments

Comments
 (0)