github/flutter
7
0
Fork 0
mirror of https://github.com/flutter/flutter.git synced 2025-06-03 00:51:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Allow A/B tests to run as just aggregator of local engine benchmarks (#146479)

Browse Source 
Add `--ab-local-engine-only` flag to change A/B benchmark aggregate behavior to instead run the number of tasks specified with the local engine only, not the default engine.  This is useful when you are trying to get average and noise results for a particular engine branch, without needing the default engine every time you change the engine.

Fixes https://github.com/flutter/flutter/issues/143770
This commit is contained in:
Jenn Magder 2024-04-11 21:07:19 -07:00 committed by GitHub
parent a34086a573
commit 53cba24de3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 24 additions and 13 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

37
dev/devicelab/bin/run.dart
View File

@ -114,6 +114,7 @@ Future<void> main(List<String> rawArgs) async {
deviceId: deviceId,
resultsFile: resultsFile,
taskName: taskNames.single,
onlyLocalEngine: (args['ab-local-engine-only'] as bool?) ?? false,
);
} else {
await runTasks(taskNames,
@ -142,6 +143,7 @@ Future<void> _runABTest({
required String? deviceId,
required String resultsFile,
required String taskName,
bool onlyLocalEngine = false,
}) async {
print('$taskName A/B test. Will run $runsPerTest times.');
@ -155,23 +157,27 @@ Future<void> _runABTest({
for (int i = 1; i <= runsPerTest; i++) {
section('Run #$i');
print('Running with the default engine (A)');
final TaskResult defaultEngineResult = await runTask(
taskName,
silent: silent,
deviceId: deviceId,
);
if (onlyLocalEngine) {
print('Skipping default engine (A)');
} else {
print('Running with the default engine (A)');
final TaskResult defaultEngineResult = await runTask(
taskName,
silent: silent,
deviceId: deviceId,
);
print('Default engine result:');
print(const JsonEncoder.withIndent(' ').convert(defaultEngineResult));
print('Default engine result:');
print(const JsonEncoder.withIndent(' ').convert(defaultEngineResult));
if (!defaultEngineResult.succeeded) {
stderr.writeln('Task failed on the default engine.');
exit(1);
if (!defaultEngineResult.succeeded) {
stderr.writeln('Task failed on the default engine.');
exit(1);
}
abTest.addAResult(defaultEngineResult);
}
abTest.addAResult(defaultEngineResult);
print('Running with the local engine (B)');
final TaskResult localEngineResult = await runTask(
taskName,
@ -273,6 +279,11 @@ ArgParser createArgParser(List<String> taskNames) {
'The filename may contain a single # character to be replaced by a sequence\n'
'number if the name already exists.',
)
..addFlag(
'ab-local-engine-only',
help: 'When running the A/B aggregator, do not run benchmarks with the default engine (A), only the local engine (B).\n'
'Shows the averages and noise report for the local engine without comparison to anything else.',
)
..addFlag(
'exit',
help: 'Exit on the first test failure. Currently flakes are intentionally (though '