Skip to content

Commit ac532d6

Browse files
cgeweckecgewecke
authored
Generate mocha JSON output with --matrix (#601)
1 parent 308f32d commit ac532d6

File tree

12 files changed

+243
-30
lines changed

12 files changed

+243
-30
lines changed

‎README.md

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -126,7 +126,8 @@ module.exports = {
126126
| measureFunctionCoverage | *boolean* | `true` | Computes function coverage. [More...][34] |
127127
| measureModifierCoverage | *boolean* | `true` | Computes each modifier invocation as a code branch. [More...][34] |
128128
| modifierWhitelist | *String[]* | `[]` | List of modifier names (ex: "onlyOwner") to exclude from branch measurement. (Useful for modifiers which prepare something instead of acting as a gate.)) |
129-
| matrixOutputPath | *String* | `./testMatrix.json` | Relative path to write test matrix JSON object to. [More...][38] |
129+
| matrixOutputPath | *String* | `./testMatrix.json` | Relative path to write test matrix JSON object to. [More...][38]|
130+
| mochaJsonOutputPath | *String* | `./mochaOutput.json` | Relative path to write mocha JSON reporter object to. [More...][38]|
130131
| abiOutputPath | *String* | `./humanReadableAbis.json` | Relative path to write diff-able ABI data to. [More...][38] |
131132
| istanbulFolder | *String* | `./coverage` | Folder location for Istanbul coverage reports. |
132133
| istanbulReporter | *Array* | `['html', 'lcov', 'text', 'json']` | [Istanbul coverage reporters][2] |

‎docs/advanced.md

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -109,6 +109,15 @@ to guess where bugs might exist in a given codebase.
109109
Running the coverage command with `--matrix` will write [a JSON test matrix][25] which maps greppable
110110
test names to each line of code to a file named `testMatrix.json` in your project's root.
111111

112+
It also generates a `mochaOutput.json` file which contains test run data similar to that
113+
generated by mocha's built-in [JSON reporter][27].
114+
115+
In combination these data sets can be passed to Joram's Honig's [tarantula][29] tool which uses
116+
a fault localization algorithm to generate 'suspiciousness' ratings for each line of
117+
Solidity code in your project.
118+
112119
[22]: https://github.com/JoranHonig/vertigo#vertigo
113120
[23]: http://spideruci.org/papers/jones05.pdf
114121
[25]: https://github.com/sc-forks/solidity-coverage/blob/master/docs/matrix.md
122+
[27]: https://mochajs.org/api/reporters_json.js.html
123+
[29]: https://github.com/JoranHonig/tarantula

‎lib/api.js

Lines changed: 21 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ class API {
3535
this.cwd = config.cwd || process.cwd();
3636
this.abiOutputPath = config.abiOutputPath || "humanReadableAbis.json";
3737
this.matrixOutputPath = config.matrixOutputPath || "testMatrix.json";
38+
this.mochaJsonOutputPath = config.mochaJsonOutputPath || "mochaOutput.json";
3839
this.matrixReporterPath = config.matrixReporterPath || "solidity-coverage/plugins/resources/matrix.js"
3940

4041
this.defaultHook = () => {};
@@ -318,29 +319,31 @@ class API {
318319
id
319320
} = this.instrumenter.instrumentationData[hash];
320321

321-
if (type === 'line' && hits > 0){
322+
if (type === 'line'){
322323
if (!this.testMatrix[contractPath]){
323324
this.testMatrix[contractPath] = {};
324325
}
325326
if (!this.testMatrix[contractPath][id]){
326327
this.testMatrix[contractPath][id] = [];
327328
}
328329

329-
// Search for and exclude duplicate entries
330-
let duplicate = false;
331-
for (const item of this.testMatrix[contractPath][id]){
332-
if (item.title === title && item.file === file){
333-
duplicate = true;
334-
break;
330+
if (hits > 0){
331+
// Search for and exclude duplicate entries
332+
let duplicate = false;
333+
for (const item of this.testMatrix[contractPath][id]){
334+
if (item.title === title && item.file === file){
335+
duplicate = true;
336+
break;
337+
}
335338
}
336-
}
337339

338-
if (!duplicate) {
339-
this.testMatrix[contractPath][id].push({title, file});
340-
}
340+
if (!duplicate) {
341+
this.testMatrix[contractPath][id].push({title, file});
342+
}
341343

342-
// Reset line data
343-
this.instrumenter.instrumentationData[hash].hits = 0;
344+
// Reset line data
345+
this.instrumenter.instrumentationData[hash].hits = 0;
346+
}
344347
}
345348
}
346349
}
@@ -360,6 +363,11 @@ class API {
360363
fs.writeFileSync(matrixPath, JSON.stringify(mapping, null, ' '));
361364
}
362365

366+
saveMochaJsonOutput(data){
367+
const outputPath = path.join(this.cwd, this.mochaJsonOutputPath);
368+
fs.writeFileSync(outputPath, JSON.stringify(data, null, ' '));
369+
}
370+
363371
saveHumanReadableAbis(data){
364372
const abiPath = path.join(this.cwd, this.abiOutputPath);
365373
fs.writeFileSync(abiPath, JSON.stringify(data, null, ' '));

‎plugins/resources/matrix.js

Lines changed: 78 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,7 @@
11
const mocha = require("mocha");
22
const inherits = require("util").inherits;
33
const Spec = mocha.reporters.Spec;
4-
4+
const path = require('path');
55

66
/**
77
* This file adapted from mocha's stats-collector
@@ -40,10 +40,13 @@ function mochaStats(runner) {
4040
}
4141

4242
/**
43-
* Based on the Mocha 'Spec' reporter. Watches an Ethereum test suite run
44-
* and collects data about which tests hit which lines of code.
45-
* This "test matrix" can be used as an input to
43+
* Based on the Mocha 'Spec' reporter.
44+
*
45+
* Watches an Ethereum test suite run and collects data about which tests hit
46+
* which lines of code. This "test matrix" can be used as an input to fault localization tools
47+
* like: https://github.com/JoranHonig/tarantula
4648
*
49+
* Mocha's JSON reporter output is also generated and saved to a separate file
4750
*
4851
* @param {Object} runner mocha's runner
4952
* @param {Object} options reporter.options (see README example usage)
@@ -52,6 +55,11 @@ function Matrix(runner, options) {
5255
// Spec reporter
5356
Spec.call(this, runner, options);
5457

58+
const self = this;
59+
const tests = [];
60+
const failures = [];
61+
const passes = [];
62+
5563
// Initialize stats for Mocha 6+ epilogue
5664
if (!runner.stats) {
5765
mochaStats(runner);
@@ -60,7 +68,73 @@ function Matrix(runner, options) {
6068

6169
runner.on("test end", (info) => {
6270
options.reporterOptions.collectTestMatrixData(info);
71+
tests.push(info);
72+
});
73+
74+
runner.on('pass', function(info) {
75+
passes.push(info)
76+
})
77+
runner.on('fail', function(info) {
78+
failures.push(info)
79+
});
80+
81+
runner.once('end', function() {
82+
delete self.stats.start;
83+
delete self.stats.end;
84+
delete self.stats.duration;
85+
86+
var obj = {
87+
stats: self.stats,
88+
tests: tests.map(clean),
89+
failures: failures.map(clean),
90+
passes: passes.map(clean)
91+
};
92+
runner.testResults = obj;
93+
options.reporterOptions.saveMochaJsonOutput(obj)
6394
});
95+
96+
// >>>>>>>>>>>>>>>>>>>>>>>>>
97+
// Mocha JSON Reporter Utils
98+
// Code taken from:
99+
// https://mochajs.org/api/reporters_json.js.html
100+
// >>>>>>>>>>>>>>>>>>>>>>>>>
101+
function clean(info) {
102+
var err = info.err || {};
103+
if (err instanceof Error) {
104+
err = errorJSON(err);
105+
}
106+
return {
107+
title: info.title,
108+
fullTitle: info.fullTitle(),
109+
file: path.relative(options.reporterOptions.cwd, info.file),
110+
currentRetry: info.currentRetry(),
111+
err: cleanCycles(err)
112+
};
113+
}
114+
115+
function cleanCycles(obj) {
116+
var cache = [];
117+
return JSON.parse(
118+
JSON.stringify(obj, function(key, value) {
119+
if (typeof value === 'object' && value !== null) {
120+
if (cache.indexOf(value) !== -1) {
121+
// Instead of going in a circle, we'll print [object Object]
122+
return '' + value;
123+
}
124+
cache.push(value);
125+
}
126+
return value;
127+
})
128+
);
129+
}
130+
131+
function errorJSON(err) {
132+
var res = {};
133+
Object.getOwnPropertyNames(err).forEach(function(key) {
134+
res[key] = err[key];
135+
}, err);
136+
return res;
137+
}
64138
}
65139

66140
/**

‎plugins/resources/nomiclabs.utils.js

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -174,7 +174,9 @@ function collectTestMatrixData(args, env, api){
174174
mochaConfig = env.config.mocha || {};
175175
mochaConfig.reporter = api.matrixReporterPath;
176176
mochaConfig.reporterOptions = {
177-
collectTestMatrixData: api.collectTestMatrixData.bind(api)
177+
collectTestMatrixData: api.collectTestMatrixData.bind(api),
178+
saveMochaJsonOutput: api.saveMochaJsonOutput.bind(api),
179+
cwd: api.cwd
178180
}
179181
env.config.mocha = mochaConfig;
180182
}

‎plugins/resources/truffle.utils.js

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -240,7 +240,9 @@ function collectTestMatrixData(config, api){
240240
config.mocha = config.mocha || {};
241241
config.mocha.reporter = api.matrixReporterPath;
242242
config.mocha.reporterOptions = {
243-
collectTestMatrixData: api.collectTestMatrixData.bind(api)
243+
collectTestMatrixData: api.collectTestMatrixData.bind(api),
244+
saveMochaJsonOutput: api.saveMochaJsonOutput.bind(api),
245+
cwd: api.cwd
244246
}
245247
}
246248
}

‎test/integration/projects/matrix/.solcover.js

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ module.exports = {
99
// "solidity-coverage/plugins/resources/matrix.js"
1010
matrixReporterPath: reporterPath,
1111
matrixOutputPath: "alternateTestMatrix.json",
12+
mochaJsonOutputPath: "alternateMochaOutput.json",
1213

1314
skipFiles: ['Migrations.sol'],
1415
silent: process.env.SILENT ? true : false,

‎test/integration/projects/matrix/contracts/MatrixA.sol

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,4 +14,8 @@ contract MatrixA {
1414
uint y = 5;
1515
return y;
1616
}
17+
18+
function unhit() public {
19+
uint z = 7;
20+
}
1721
}

‎test/integration/projects/matrix/expectedMochaOutput.json

Lines changed: 99 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,99 @@
1+
{
2+
"stats": {
3+
"suites": 2,
4+
"tests": 6,
5+
"passes": 6,
6+
"pending": 0,
7+
"failures": 0
8+
},
9+
"tests": [
10+
{
11+
"title": "sends to A",
12+
"fullTitle": "Contract: Matrix A and B sends to A",
13+
"file": "test/matrix_a_b.js",
14+
"currentRetry": 0,
15+
"err": {}
16+
},
17+
{
18+
"title": "sends to A",
19+
"fullTitle": "Contract: Matrix A and B sends to A",
20+
"file": "test/matrix_a_b.js",
21+
"currentRetry": 0,
22+
"err": {}
23+
},
24+
{
25+
"title": "calls B",
26+
"fullTitle": "Contract: Matrix A and B calls B",
27+
"file": "test/matrix_a_b.js",
28+
"currentRetry": 0,
29+
"err": {}
30+
},
31+
{
32+
"title": "sends to B",
33+
"fullTitle": "Contract: Matrix A and B sends to B",
34+
"file": "test/matrix_a_b.js",
35+
"currentRetry": 0,
36+
"err": {}
37+
},
38+
{
39+
"title": "sends",
40+
"fullTitle": "Contract: MatrixA sends",
41+
"file": "test/matrix_a.js",
42+
"currentRetry": 0,
43+
"err": {}
44+
},
45+
{
46+
"title": "calls",
47+
"fullTitle": "Contract: MatrixA calls",
48+
"file": "test/matrix_a.js",
49+
"currentRetry": 0,
50+
"err": {}
51+
}
52+
],
53+
"failures": [],
54+
"passes": [
55+
{
56+
"title": "sends to A",
57+
"fullTitle": "Contract: Matrix A and B sends to A",
58+
"file": "test/matrix_a_b.js",
59+
"currentRetry": 0,
60+
"err": {}
61+
},
62+
{
63+
"title": "sends to A",
64+
"fullTitle": "Contract: Matrix A and B sends to A",
65+
"file": "test/matrix_a_b.js",
66+
"currentRetry": 0,
67+
"err": {}
68+
},
69+
{
70+
"title": "calls B",
71+
"fullTitle": "Contract: Matrix A and B calls B",
72+
"file": "test/matrix_a_b.js",
73+
"currentRetry": 0,
74+
"err": {}
75+
},
76+
{
77+
"title": "sends to B",
78+
"fullTitle": "Contract: Matrix A and B sends to B",
79+
"file": "test/matrix_a_b.js",
80+
"currentRetry": 0,
81+
"err": {}
82+
},
83+
{
84+
"title": "sends",
85+
"fullTitle": "Contract: MatrixA sends",
86+
"file": "test/matrix_a.js",
87+
"currentRetry": 0,
88+
"err": {}
89+
},
90+
{
91+
"title": "calls",
92+
"fullTitle": "Contract: MatrixA calls",
93+
"file": "test/matrix_a.js",
94+
"currentRetry": 0,
95+
"err": {}
96+
}
97+
]
98+
}
99+

‎test/integration/projects/matrix/expectedTestMatrixHardhat.json

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,8 @@
2121
"title": "calls",
2222
"file": "test/matrix_a.js"
2323
}
24-
]
24+
],
25+
"19": []
2526
},
2627
"contracts/MatrixB.sol": {
2728
"10": [
@@ -43,4 +44,5 @@
4344
}
4445
]
4546
}
46-
}
47+
}
48+

‎test/units/hardhat/flags.js

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -184,12 +184,18 @@ describe('Hardhat Plugin: command line options', function() {
184184
await this.env.run("coverage", taskArgs);
185185

186186
// Integration test checks output path configurabililty
187-
const altPath = path.join(process.cwd(), './alternateTestMatrix.json');
188-
const expPath = path.join(process.cwd(), './expectedTestMatrixHardhat.json');
189-
const producedMatrix = require(altPath)
190-
const expectedMatrix = require(expPath);
187+
const altMatrixPath = path.join(process.cwd(), './alternateTestMatrix.json');
188+
const expMatrixPath = path.join(process.cwd(), './expectedTestMatrixHardhat.json');
189+
const altMochaPath = path.join(process.cwd(), './alternateMochaOutput.json');
190+
const expMochaPath = path.join(process.cwd(), './expectedMochaOutput.json');
191+
192+
const producedMatrix = require(altMatrixPath)
193+
const expectedMatrix = require(expMatrixPath);
194+
const producedMochaOutput = require(altMochaPath);
195+
const expectedMochaOutput = require(expMochaPath);
191196

192197
assert.deepEqual(producedMatrix, expectedMatrix);
198+
assert.deepEqual(producedMochaOutput, expectedMochaOutput);
193199
});
194200

195201
it('--abi', async function(){

0 commit comments

Comments
 (0)