flutter/dev/devicelab/lib/tasks/hot_mode_tests.dart
Jonah Williams 5dfe7e6d51
[flutter_tools] attempt to stabilize hot restart benchmark the old fashioned way (#67971)
A change which sped up hot restart locally caused many of the devicelab measures to regress. I think this is because we do not measure when the isolate is actually "ready", so starting a reload or restart prematurely can cause time spent doing initialization to be registered as part of the reload operation.

A fix for this would be to have the framework include some sort of "initialization complete" event ... but it is not clear what the correct trigger would be. Perhaps after the first frame is successfully registered?

(9a3a0dc caused the benchmark regression - possibly since we spend less time syncing files now so we start the restart earlier)
2020-10-12 17:00:26 -07:00

170 lines
6.9 KiB
Dart

// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:convert';
import 'dart:io';
import 'package:path/path.dart' as path;
import '../framework/adb.dart';
import '../framework/framework.dart';
import '../framework/task_result.dart';
import '../framework/utils.dart';
final Directory _editedFlutterGalleryDir = dir(path.join(Directory.systemTemp.path, 'edited_flutter_gallery'));
final Directory flutterGalleryDir = dir(path.join(flutterDirectory.path, 'dev/integration_tests/flutter_gallery'));
TaskFunction createHotModeTest({String deviceIdOverride, Map<String, String> environment}) {
return () async {
if (deviceIdOverride == null) {
final Device device = await devices.workingDevice;
await device.unlock();
deviceIdOverride = device.deviceId;
}
final File benchmarkFile = file(path.join(_editedFlutterGalleryDir.path, 'hot_benchmark.json'));
rm(benchmarkFile);
final List<String> options = <String>[
'--hot', '-d', deviceIdOverride, '--benchmark', '--verbose', '--resident', '--output-dill',
];
int hotReloadCount = 0;
Map<String, dynamic> twoReloadsData;
Map<String, dynamic> freshRestartReloadsData;
await inDirectory<void>(flutterDirectory, () async {
rmTree(_editedFlutterGalleryDir);
mkdirs(_editedFlutterGalleryDir);
recursiveCopy(flutterGalleryDir, _editedFlutterGalleryDir);
await inDirectory<void>(_editedFlutterGalleryDir, () async {
{
final Process clearProcess = await startProcess(
path.join(flutterDirectory.path, 'bin', 'flutter'),
flutterCommandArgs('clean', <String>[]),
environment: environment,
);
await clearProcess.exitCode;
final Process process = await startProcess(
path.join(flutterDirectory.path, 'bin', 'flutter'),
flutterCommandArgs('run', options),
environment: environment,
);
final Completer<void> stdoutDone = Completer<void>();
final Completer<void> stderrDone = Completer<void>();
process.stdout
.transform<String>(utf8.decoder)
.transform<String>(const LineSplitter())
.listen((String line) {
if (line.contains('] Reloaded ')) {
if (hotReloadCount == 0) {
// Update the file and reload again.
final File appDartSource = file(path.join(
_editedFlutterGalleryDir.path, 'lib/gallery/app.dart',
));
appDartSource.writeAsStringSync(
appDartSource.readAsStringSync().replaceFirst(
"'Flutter Gallery'", "'Updated Flutter Gallery'",
)
);
process.stdin.writeln('r');
++hotReloadCount;
} else {
// Quit after second hot reload.
process.stdin.writeln('q');
}
}
print('stdout: $line');
}, onDone: () {
stdoutDone.complete();
});
process.stderr
.transform<String>(utf8.decoder)
.transform<String>(const LineSplitter())
.listen((String line) {
print('stderr: $line');
}, onDone: () {
stderrDone.complete();
});
await Future.wait<void>(
<Future<void>>[stdoutDone.future, stderrDone.future]);
await process.exitCode;
twoReloadsData = json.decode(benchmarkFile.readAsStringSync()) as Map<String, dynamic>;
}
benchmarkFile.deleteSync();
// Start `flutter run` again to make sure it loads from the previous
// state. Frontend loads up from previously generated kernel files.
{
final Process process = await startProcess(
path.join(flutterDirectory.path, 'bin', 'flutter'),
flutterCommandArgs('run', options),
environment: environment,
);
final Completer<void> stdoutDone = Completer<void>();
final Completer<void> stderrDone = Completer<void>();
process.stdout
.transform<String>(utf8.decoder)
.transform<String>(const LineSplitter())
.listen((String line) {
if (line.contains('] Reloaded ')) {
process.stdin.writeln('q');
}
print('stdout: $line');
}, onDone: () {
stdoutDone.complete();
});
process.stderr
.transform<String>(utf8.decoder)
.transform<String>(const LineSplitter())
.listen((String line) {
print('stderr: $line');
}, onDone: () {
stderrDone.complete();
});
await Future.wait<void>(
<Future<void>>[stdoutDone.future, stderrDone.future]);
await process.exitCode;
freshRestartReloadsData =
json.decode(benchmarkFile.readAsStringSync()) as Map<String, dynamic>;
}
});
});
return TaskResult.success(
<String, dynamic> {
'hotReloadInitialDevFSSyncMilliseconds': twoReloadsData['hotReloadInitialDevFSSyncMilliseconds'][0],
'hotRestartMillisecondsToFrame': twoReloadsData['hotRestartMillisecondsToFrame'][0],
'hotReloadMillisecondsToFrame' : twoReloadsData['hotReloadMillisecondsToFrame'][0],
'hotReloadDevFSSyncMilliseconds': twoReloadsData['hotReloadDevFSSyncMilliseconds'][0],
'hotReloadFlutterReassembleMilliseconds': twoReloadsData['hotReloadFlutterReassembleMilliseconds'][0],
'hotReloadVMReloadMilliseconds': twoReloadsData['hotReloadVMReloadMilliseconds'][0],
'hotReloadMillisecondsToFrameAfterChange' : twoReloadsData['hotReloadMillisecondsToFrame'][1],
'hotReloadDevFSSyncMillisecondsAfterChange': twoReloadsData['hotReloadDevFSSyncMilliseconds'][1],
'hotReloadFlutterReassembleMillisecondsAfterChange': twoReloadsData['hotReloadFlutterReassembleMilliseconds'][1],
'hotReloadVMReloadMillisecondsAfterChange': twoReloadsData['hotReloadVMReloadMilliseconds'][1],
'hotReloadInitialDevFSSyncAfterRelaunchMilliseconds' : freshRestartReloadsData['hotReloadInitialDevFSSyncMilliseconds'][0],
},
benchmarkScoreKeys: <String>[
'hotReloadInitialDevFSSyncMilliseconds',
'hotRestartMillisecondsToFrame',
'hotReloadMillisecondsToFrame',
'hotReloadDevFSSyncMilliseconds',
'hotReloadFlutterReassembleMilliseconds',
'hotReloadVMReloadMilliseconds',
'hotReloadMillisecondsToFrameAfterChange',
'hotReloadDevFSSyncMillisecondsAfterChange',
'hotReloadFlutterReassembleMillisecondsAfterChange',
'hotReloadVMReloadMillisecondsAfterChange',
'hotReloadInitialDevFSSyncAfterRelaunchMilliseconds',
],
);
};
}