Skip to content

Commit 53cba24

Browse files
authored
Allow A/B tests to run as just aggregator of local engine benchmarks (flutter#146479)
Add `--ab-local-engine-only` flag to change A/B benchmark aggregate behavior to instead run the number of tasks specified with the local engine only, not the default engine. This is useful when you are trying to get average and noise results for a particular engine branch, without needing the default engine every time you change the engine. Fixes flutter#143770
1 parent a34086a commit 53cba24

File tree

1 file changed

+26
-15
lines changed

1 file changed

+26
-15
lines changed

dev/devicelab/bin/run.dart

+26-15
Original file line numberDiff line numberDiff line change
@@ -114,6 +114,7 @@ Future<void> main(List<String> rawArgs) async {
114114
deviceId: deviceId,
115115
resultsFile: resultsFile,
116116
taskName: taskNames.single,
117+
onlyLocalEngine: (args['ab-local-engine-only'] as bool?) ?? false,
117118
);
118119
} else {
119120
await runTasks(taskNames,
@@ -142,6 +143,7 @@ Future<void> _runABTest({
142143
required String? deviceId,
143144
required String resultsFile,
144145
required String taskName,
146+
bool onlyLocalEngine = false,
145147
}) async {
146148
print('$taskName A/B test. Will run $runsPerTest times.');
147149

@@ -155,23 +157,27 @@ Future<void> _runABTest({
155157
for (int i = 1; i <= runsPerTest; i++) {
156158
section('Run #$i');
157159

158-
print('Running with the default engine (A)');
159-
final TaskResult defaultEngineResult = await runTask(
160-
taskName,
161-
silent: silent,
162-
deviceId: deviceId,
163-
);
164-
165-
print('Default engine result:');
166-
print(const JsonEncoder.withIndent(' ').convert(defaultEngineResult));
167-
168-
if (!defaultEngineResult.succeeded) {
169-
stderr.writeln('Task failed on the default engine.');
170-
exit(1);
160+
if (onlyLocalEngine) {
161+
print('Skipping default engine (A)');
162+
} else {
163+
print('Running with the default engine (A)');
164+
final TaskResult defaultEngineResult = await runTask(
165+
taskName,
166+
silent: silent,
167+
deviceId: deviceId,
168+
);
169+
170+
print('Default engine result:');
171+
print(const JsonEncoder.withIndent(' ').convert(defaultEngineResult));
172+
173+
if (!defaultEngineResult.succeeded) {
174+
stderr.writeln('Task failed on the default engine.');
175+
exit(1);
176+
}
177+
178+
abTest.addAResult(defaultEngineResult);
171179
}
172180

173-
abTest.addAResult(defaultEngineResult);
174-
175181
print('Running with the local engine (B)');
176182
final TaskResult localEngineResult = await runTask(
177183
taskName,
@@ -273,6 +279,11 @@ ArgParser createArgParser(List<String> taskNames) {
273279
'The filename may contain a single # character to be replaced by a sequence\n'
274280
'number if the name already exists.',
275281
)
282+
..addFlag(
283+
'ab-local-engine-only',
284+
help: 'When running the A/B aggregator, do not run benchmarks with the default engine (A), only the local engine (B).\n'
285+
'Shows the averages and noise report for the local engine without comparison to anything else.',
286+
)
276287
..addFlag(
277288
'exit',
278289
help: 'Exit on the first test failure. Currently flakes are intentionally (though '

0 commit comments

Comments
 (0)