Run all microbenchmarks (#154374)

Two things:

**Re-land**: Uninstall microbenchmarks before running them.
Flakes in #153828 stem from adb saying the app isn't installed, but then failing to install wtih `-r`. Several other tests uninstall the app before trying to run it. Previous fix called uninstall between tests, but iOS takes 12 to 13 seconds to perform uninstall / install, which timed out the test.  Just uninstall the one time since we only care about any lingering apps with different keys.
Potential solution #153828

**Make things go fast**
Instead of installing 21 different compilations of the same app to get results; compile and run them together. Locally on Mac+iOS, this should takes ~3 minutes instead of ~15 minutes.
This commit is contained in:
John McDole 2024-08-30 11:22:06 -07:00 committed by GitHub
parent 0d8247eb7d
commit 9fc160b11c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
26 changed files with 201 additions and 105 deletions

View File

@ -1,22 +1,17 @@
# microbenchmarks
To run these benchmarks on a device, first run `flutter logs' in one
window to see the device logs, then, in a different window, run any of
these:
To run these benchmarks on a device, first run `flutter logs` in one
window to see the device logs, then, in a different window, run:
```sh
flutter run --release lib/gestures/velocity_tracker_bench.dart
flutter run --release lib/gestures/gesture_detector_bench.dart
flutter run --release lib/stocks/animation_bench.dart
flutter run --release lib/stocks/build_bench.dart
flutter run --release lib/stocks/layout_bench.dart
flutter run -d $DEVICE_ID --profile lib/benchmark_collection.dart
```
The results should be in the device logs.
### Avoid changing names of the benchmarks
## Avoid changing names of the benchmarks
Each microbenchmark is identified by a name, for example,
"catmullrom_transform_iteration". Changing the name of an existing
microbenchmarks will effectively remove the old benchmark and create a new one,
"catmullrom_transform_iteration". Changing the name passed to `BenchmarkResultPrinter.addResult`
will effectively remove the old benchmark and create a new one,
losing the historical data associated with the old benchmark in the process.

View File

@ -0,0 +1,23 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:flutter_test/flutter_test.dart';
class BenchmarkingBinding extends LiveTestWidgetsFlutterBinding {
BenchmarkingBinding();
final Stopwatch drawFrameWatch = Stopwatch();
@override
void handleBeginFrame(Duration? rawTimeStamp) {
drawFrameWatch.start();
super.handleBeginFrame(rawTimeStamp);
}
@override
void handleDrawFrame() {
super.handleDrawFrame();
drawFrameWatch.stop();
}
}

View File

@ -0,0 +1,107 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:io';
import 'package:flutter/material.dart';
import 'package:flutter/scheduler.dart';
import 'package:flutter_test/flutter_test.dart';
import 'benchmark_binding.dart';
import 'foundation/all_elements_bench.dart' as all_elements_bench;
import 'foundation/change_notifier_bench.dart' as change_notifier_bench;
import 'foundation/clamp.dart' as clamp;
import 'foundation/decode_and_parse_asset_manifest.dart'
as decode_and_parse_asset_manifest;
import 'foundation/platform_asset_bundle.dart' as platform_asset_bundle;
import 'foundation/standard_message_codec_bench.dart'
as standard_message_codec_bench;
import 'foundation/standard_method_codec_bench.dart'
as standard_method_codec_bench;
import 'foundation/timeline_bench.dart' as timeline_bench;
import 'geometry/matrix_utils_transform_bench.dart'
as matrix_utils_transform_bench;
import 'geometry/rrect_contains_bench.dart' as rrect_contains_bench;
import 'gestures/gesture_detector_bench.dart' as gesture_detector_bench;
import 'gestures/velocity_tracker_bench.dart' as velocity_tracker_bench;
import 'language/compute_bench.dart' as compute_bench;
import 'language/sync_star_bench.dart' as sync_star_bench;
import 'language/sync_star_semantics_bench.dart' as sync_star_semantics_bench;
import 'layout/text_intrinsic_bench.dart' as text_intrinsic_bench;
import 'stocks/animation_bench.dart' as animation_bench;
import 'stocks/build_bench.dart' as build_bench;
import 'stocks/build_bench_profiled.dart' as build_bench_profiled;
import 'stocks/layout_bench.dart' as layout_bench;
import 'ui/image_bench.dart' as image_bench;
typedef Benchmark = (String name, Future<void> Function() value);
Future<void> main() async {
assert(false,
"Don't run benchmarks in debug mode! Use 'flutter run --release'.");
// BenchmarkingBinding is used by animation_bench, providing a simple
// stopwatch interface over rendering. Lifting it here makes all
// benchmarks run together.
final BenchmarkingBinding binding = BenchmarkingBinding();
final List<Benchmark> benchmarks = <Benchmark>[
('foundation/change_notifier_bench.dart', change_notifier_bench.execute),
('foundation/clamp.dart', clamp.execute),
('foundation/platform_asset_bundle.dart', platform_asset_bundle.execute),
(
'foundation/standard_message_codec_bench.dart',
standard_message_codec_bench.execute
),
(
'foundation/standard_method_codec_bench.dart',
standard_method_codec_bench.execute
),
('foundation/timeline_bench.dart', timeline_bench.execute),
(
'foundation/decode_and_parse_asset_manifest.dart',
decode_and_parse_asset_manifest.execute
),
(
'geometry/matrix_utils_transform_bench.dart',
matrix_utils_transform_bench.execute
),
('geometry/rrect_contains_bench.dart', rrect_contains_bench.execute),
('gestures/gesture_detector_bench.dart', gesture_detector_bench.execute),
('gestures/velocity_tracker_bench.dart', velocity_tracker_bench.execute),
('language/compute_bench.dart', compute_bench.execute),
('language/sync_star_bench.dart', sync_star_bench.execute),
(
'language/sync_star_semantics_bench.dart',
sync_star_semantics_bench.execute
),
('stocks/animation_bench.dart', () => animation_bench.execute(binding)),
('stocks/build_bench.dart', build_bench.execute),
('stocks/build_bench_profiled.dart', build_bench_profiled.execute),
('stocks/layout_bench.dart', layout_bench.execute),
('ui/image_bench.dart', image_bench.execute),
('layout/text_intrinsic_bench.dart', text_intrinsic_bench.execute),
(
'foundation/all_elements_bench.dart',
() async {
binding.framePolicy =
LiveTestWidgetsFlutterBindingFramePolicy.fullyLive;
runApp(const SizedBox.shrink()); // ensure dispose
await SchedulerBinding.instance.endOfFrame;
all_elements_bench.execute();
}
),
];
print('╡ ••• Running microbenchmarks ••• ╞');
for (final Benchmark mark in benchmarks) {
// Reset the frame policy to default - each test can set it on their own.
binding.framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.fadePointers;
print('╡ ••• Running ${mark.$1} ••• ╞');
await mark.$2();
}
print('\n\n╡ ••• Done ••• ╞\n\n');
exit(0);
}

View File

@ -11,8 +11,7 @@ import '../common.dart';
const int _kNumIters = 10000;
Future<void> main() async {
assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
Future<void> execute() async {
runApp(MaterialApp(
home: Scaffold(
body: GridView.count(

View File

@ -10,7 +10,7 @@ const int _kNumIterations = 65536;
const int _kNumWarmUp = 100;
const int _kScale = 1000;
void main() {
Future<void> execute() async {
assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
// In the following benchmarks, we won't remove the listeners when we don't

View File

@ -9,7 +9,7 @@ import '../common.dart';
const int _kBatchSize = 100000;
const int _kNumIterations = 1000;
void main() {
Future<void> execute() async {
assert(false,
"Don't run benchmarks in debug mode! Use 'flutter run --release'.");
final BenchmarkResultPrinter printer = BenchmarkResultPrinter();

View File

@ -9,7 +9,7 @@ import '../common.dart';
const int _kNumIterations = 1000;
void main() async {
Future<void> execute() async {
assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
final BenchmarkResultPrinter printer = BenchmarkResultPrinter();

View File

@ -10,7 +10,7 @@ import '../common.dart';
const int _kBatchSize = 100;
const int _kNumIterations = 100;
void main() async {
Future<void> execute() async {
assert(false,
"Don't run benchmarks in debug mode! Use 'flutter run --release'.");
final BenchmarkResultPrinter printer = BenchmarkResultPrinter();

View File

@ -8,7 +8,7 @@ import '../common.dart';
const int _kNumIterations = 100000;
void main() {
Future<void> execute() async {
assert(false,
"Don't run benchmarks in debug mode! Use 'flutter run --release'.");
final BenchmarkResultPrinter printer = BenchmarkResultPrinter();

View File

@ -8,7 +8,7 @@ import '../common.dart';
const int _kNumIterations = 100000;
void main() {
Future<void> execute() async {
assert(false,
"Don't run benchmarks in debug mode! Use 'flutter run --release'.");
final BenchmarkResultPrinter printer = BenchmarkResultPrinter();

View File

@ -8,7 +8,7 @@ import '../common.dart';
const int _kNumIterations = 10000;
void main() {
Future<void> execute() async {
assert(false,
"Don't run benchmarks in debug mode! Use 'flutter run --release'.");
final BenchmarkResultPrinter printer = BenchmarkResultPrinter();

View File

@ -11,7 +11,7 @@ import '../common.dart';
const int _kNumIterations = 10000000;
const int _kNumWarmUp = 100000;
void main() {
Future<void> execute() async {
assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
print('MatrixUtils.transformRect and .transformPoint benchmark...');

View File

@ -8,7 +8,7 @@ import '../common.dart';
const int _kNumIters = 10000;
void main() {
Future<void> execute() async {
assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
final Stopwatch watch = Stopwatch();
print('RRect contains benchmark...');

View File

@ -10,7 +10,7 @@ import 'apps/button_matrix_app.dart' as button_matrix;
const int _kNumWarmUpIters = 20;
const int _kNumIters = 300;
Future<void> main() async {
Future<void> execute() async {
assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
final Stopwatch watch = Stopwatch();
print('GestureDetector semantics benchmark...');

View File

@ -17,7 +17,7 @@ class TrackerBenchmark {
final String name;
}
Future<void> main() async {
Future<void> execute() async {
assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
final BenchmarkResultPrinter printer = BenchmarkResultPrinter();
final List<TrackerBenchmark> benchmarks = <TrackerBenchmark>[

View File

@ -23,7 +23,7 @@ List<Data> test(int length) {
(int index) => Data(index * index));
}
Future<void> main() async {
Future<void> execute() async {
assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
// Warm up lap

View File

@ -7,7 +7,7 @@ import '../common.dart';
const int _kNumIterations = 1000;
const int _kNumWarmUp = 100;
void main() {
Future<void> execute() async {
assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
// Warm up lap

View File

@ -9,7 +9,7 @@ import '../common.dart';
const int _kNumIterations = 1000;
const int _kNumWarmUp = 100;
void main() {
Future<void> execute() async {
final List<String> words = 'Lorem Ipsum is simply dummy text of the printing and'
" typesetting industry. Lorem Ipsum has been the industry's"
' standard dummy text ever since the 1500s, when an unknown'

View File

@ -20,7 +20,7 @@ final Widget intrinsicTextHeight = Directionality(
),
);
Future<void> main() async {
Future<void> execute() async {
assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
// We control the framePolicy below to prevent us from scheduling frames in

View File

@ -7,35 +7,16 @@ import 'package:flutter_test/flutter_test.dart';
import 'package:stocks/main.dart' as stocks;
import 'package:stocks/stock_data.dart' as stock_data;
import '../benchmark_binding.dart';
import '../common.dart';
const Duration kBenchmarkTime = Duration(seconds: 15);
class BenchmarkingBinding extends LiveTestWidgetsFlutterBinding {
BenchmarkingBinding(this.stopwatch);
final Stopwatch stopwatch;
@override
void handleBeginFrame(Duration? rawTimeStamp) {
stopwatch.start();
super.handleBeginFrame(rawTimeStamp);
}
@override
void handleDrawFrame() {
super.handleDrawFrame();
stopwatch.stop();
}
}
Future<void> main() async {
Future<void> execute(BenchmarkingBinding binding) async {
assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
stock_data.StockData.actuallyFetchData = false;
final Stopwatch wallClockWatch = Stopwatch();
final Stopwatch cpuWatch = Stopwatch();
BenchmarkingBinding(cpuWatch);
int totalOpenFrameElapsedMicroseconds = 0;
int totalOpenIterationCount = 0;
@ -52,27 +33,27 @@ Future<void> main() async {
bool drawerIsOpen = false;
wallClockWatch.start();
while (wallClockWatch.elapsed < kBenchmarkTime) {
cpuWatch.reset();
binding.drawFrameWatch.reset();
if (drawerIsOpen) {
await tester.tapAt(const Offset(780.0, 250.0)); // Close drawer
await tester.pump();
totalCloseIterationCount += 1;
totalCloseFrameElapsedMicroseconds += cpuWatch.elapsedMicroseconds;
totalCloseFrameElapsedMicroseconds += binding.drawFrameWatch.elapsedMicroseconds;
} else {
await tester.tapAt(const Offset(20.0, 50.0)); // Open drawer
await tester.pump();
totalOpenIterationCount += 1;
totalOpenFrameElapsedMicroseconds += cpuWatch.elapsedMicroseconds;
totalOpenFrameElapsedMicroseconds += binding.drawFrameWatch.elapsedMicroseconds;
}
drawerIsOpen = !drawerIsOpen;
// Time how long each frame takes
cpuWatch.reset();
binding.drawFrameWatch.reset();
while (SchedulerBinding.instance.hasScheduledFrame) {
await tester.pump();
totalSubsequentFramesIterationCount += 1;
}
totalSubsequentFramesElapsedMicroseconds += cpuWatch.elapsedMicroseconds;
totalSubsequentFramesElapsedMicroseconds += binding.drawFrameWatch.elapsedMicroseconds;
}
});

View File

@ -55,7 +55,7 @@ Future<List<double>> runBuildBenchmark() async {
return values;
}
Future<void> main() async {
Future<void> execute() async {
final BenchmarkResultPrinter printer = BenchmarkResultPrinter();
printer.addResultStatistics(
description: 'Stock build',

View File

@ -8,7 +8,7 @@ import 'package:flutter_test/flutter_test.dart';
import '../common.dart';
import 'build_bench.dart';
Future<void> main() async {
Future<void> execute() async {
debugProfileBuildsEnabledUserWidgets = true;
final BenchmarkResultPrinter printer = BenchmarkResultPrinter();
printer.addResultStatistics(

View File

@ -12,7 +12,7 @@ import '../common.dart';
const Duration kBenchmarkTime = Duration(seconds: 15);
Future<void> main() async {
Future<void> execute() async {
assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
stock_data.StockData.actuallyFetchData = false;

View File

@ -78,7 +78,7 @@ const List<String> assets = <String>[
// Measures the time it takes to load a fixed number of assets into an
// immutable buffer to later be decoded by skia.
Future<void> main() async {
Future<void> execute() async {
assert(false, "Don't run benchmarks in debug mode! Use 'flutter run --release'.");
final Stopwatch watch = Stopwatch();

View File

@ -14,6 +14,8 @@ Future<Map<String, double>> readJsonResults(Process process) {
const String jsonStart = '================ RESULTS ================';
const String jsonEnd = '================ FORMATTED ==============';
const String jsonPrefix = ':::JSON:::';
const String testComplete = '╡ ••• Done ••• ╞';
bool jsonStarted = false;
final StringBuffer jsonBuf = StringBuffer();
final Completer<Map<String, double>> completer = Completer<Map<String, double>>();
@ -25,8 +27,9 @@ Future<Map<String, double>> readJsonResults(Process process) {
stderr.writeln('[STDERR] $line');
});
final List<String> collectedJson = <String>[];
bool processWasKilledIntentionally = false;
bool resultsHaveBeenParsed = false;
final StreamSubscription<String> stdoutSub = process.stdout
.transform<String>(const Utf8Decoder())
.transform<String>(const LineSplitter())
@ -38,48 +41,40 @@ Future<Map<String, double>> readJsonResults(Process process) {
return;
}
if (jsonStarted && line.contains(jsonEnd)) {
final String jsonOutput = jsonBuf.toString();
// If we end up here and have already parsed the results, it suggests that
// we have received output from another test because our `flutter run`
// process did not terminate correctly.
// https://github.com/flutter/flutter/issues/19096#issuecomment-402756549
if (resultsHaveBeenParsed) {
throw 'Additional JSON was received after results has already been '
'processed. This suggests the `flutter run` process may have lived '
'past the end of our test and collected additional output from the '
'next test.\n\n'
'The JSON below contains all collected output, including both from '
'the original test and what followed.\n\n'
'$jsonOutput';
}
jsonStarted = false;
if (line.contains(testComplete)) {
processWasKilledIntentionally = true;
resultsHaveBeenParsed = true;
// Sending a SIGINT/SIGTERM to the process here isn't reliable because [process] is
// the shell (flutter is a shell script) and doesn't pass the signal on.
// Sending a `q` is an instruction to quit using the console runner.
// See https://github.com/flutter/flutter/issues/19208
process.stdin.write('q');
await process.stdin.flush();
// Give the process a couple of seconds to exit and run shutdown hooks
// before sending kill signal.
// TODO(fujino): https://github.com/flutter/flutter/issues/134566
await Future<void>.delayed(const Duration(seconds: 2));
// Also send a kill signal in case the `q` above didn't work.
process.kill(ProcessSignal.sigint);
try {
completer.complete(Map<String, double>.from(json.decode(jsonOutput) as Map<String, dynamic>));
final Map<String, double> results =
Map<String, double>.from(<String, dynamic>{
for (final String data in collectedJson)
...json.decode(data) as Map<String, dynamic>
});
completer.complete(results);
} catch (ex) {
completer.completeError('Decoding JSON failed ($ex). JSON string was: $jsonOutput');
completer.completeError(
'Decoding JSON failed ($ex). JSON strings where: $collectedJson');
}
return;
}
if (jsonStarted && line.contains(jsonEnd)) {
collectedJson.add(jsonBuf.toString().trim());
jsonBuf.clear();
jsonStarted = false;
}
if (jsonStarted && line.contains(jsonPrefix)) {
jsonBuf.writeln(line.substring(line.indexOf(jsonPrefix) + jsonPrefix.length));
}

View File

@ -24,11 +24,27 @@ TaskFunction createMicrobenchmarkTask({
await device.unlock();
await device.clearLogs();
final Directory appDir =
dir(path.join(flutterDirectory.path, 'dev/benchmarks/microbenchmarks'));
// Hard-uninstall any prior apps.
await inDirectory(appDir, () async {
section('Uninstall previous microbenchmarks app');
await flutter(
'install',
options: <String>[
'-v',
'--uninstall-only',
'-d',
device.deviceId,
],
);
});
Future<Map<String, double>> runMicrobench(String benchmarkPath) async {
Future<Map<String, double>> run() async {
print('Running $benchmarkPath');
final Directory appDir = dir(
path.join(flutterDirectory.path, 'dev/benchmarks/microbenchmarks'));
final Process flutterProcess = await inDirectory(appDir, () async {
final List<String> options = <String>[
'-v',
@ -54,27 +70,7 @@ TaskFunction createMicrobenchmarkTask({
}
final Map<String, double> allResults = <String, double>{
...await runMicrobench('lib/foundation/all_elements_bench.dart'),
...await runMicrobench('lib/foundation/change_notifier_bench.dart'),
...await runMicrobench('lib/foundation/clamp.dart'),
...await runMicrobench('lib/foundation/platform_asset_bundle.dart'),
...await runMicrobench('lib/foundation/standard_message_codec_bench.dart'),
...await runMicrobench('lib/foundation/standard_method_codec_bench.dart'),
...await runMicrobench('lib/foundation/timeline_bench.dart'),
...await runMicrobench('lib/foundation/decode_and_parse_asset_manifest.dart'),
...await runMicrobench('lib/geometry/matrix_utils_transform_bench.dart'),
...await runMicrobench('lib/geometry/rrect_contains_bench.dart'),
...await runMicrobench('lib/gestures/gesture_detector_bench.dart'),
...await runMicrobench('lib/gestures/velocity_tracker_bench.dart'),
...await runMicrobench('lib/language/compute_bench.dart'),
...await runMicrobench('lib/language/sync_star_bench.dart'),
...await runMicrobench('lib/language/sync_star_semantics_bench.dart'),
...await runMicrobench('lib/stocks/animation_bench.dart'),
...await runMicrobench('lib/stocks/build_bench_profiled.dart'),
...await runMicrobench('lib/stocks/build_bench.dart'),
...await runMicrobench('lib/stocks/layout_bench.dart'),
...await runMicrobench('lib/ui/image_bench.dart'),
...await runMicrobench('lib/layout/text_intrinsic_bench.dart'),
...await runMicrobench('lib/benchmark_collection.dart'),
};
return TaskResult.success(allResults,