mirror of
https://github.com/flutter/flutter.git
synced 2025-06-03 00:51:18 +00:00
947367aaa2
This calculation is important if you want to calculate the probability that one thing is better than another.
407 lines
15 KiB
Dart
407 lines
15 KiB
Dart
// Copyright 2014 The Flutter Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
// found in the LICENSE file.
|
|
|
|
import 'dart:convert';
|
|
import 'dart:io';
|
|
import 'dart:math' as math;
|
|
|
|
import '../framework/devices.dart';
|
|
import '../framework/framework.dart';
|
|
import '../framework/task_result.dart';
|
|
import '../framework/utils.dart';
|
|
import 'build_test_task.dart';
|
|
|
|
final Directory galleryDirectory = dir('${flutterDirectory.path}/dev/integration_tests/flutter_gallery');
|
|
|
|
/// Temp function during gallery tests transition to build+test model.
|
|
///
|
|
/// https://github.com/flutter/flutter/issues/103542
|
|
TaskFunction createGalleryTransitionBuildTest(List<String> args, {bool semanticsEnabled = false}) {
|
|
return GalleryTransitionBuildTest(args, semanticsEnabled: semanticsEnabled).call;
|
|
}
|
|
|
|
TaskFunction createGalleryTransitionTest({bool semanticsEnabled = false}) {
|
|
return GalleryTransitionTest(semanticsEnabled: semanticsEnabled).call;
|
|
}
|
|
|
|
TaskFunction createGalleryTransitionE2EBuildTest(
|
|
List<String> args, {
|
|
bool semanticsEnabled = false,
|
|
bool? enableImpeller,
|
|
}) {
|
|
return GalleryTransitionBuildTest(
|
|
args,
|
|
testFile: semanticsEnabled ? 'transitions_perf_e2e_with_semantics' : 'transitions_perf_e2e',
|
|
needFullTimeline: false,
|
|
timelineSummaryFile: 'e2e_perf_summary',
|
|
transitionDurationFile: null,
|
|
timelineTraceFile: null,
|
|
driverFile: 'transitions_perf_e2e_test',
|
|
enableImpeller: enableImpeller,
|
|
).call;
|
|
}
|
|
|
|
TaskFunction createGalleryTransitionE2ETest({
|
|
bool semanticsEnabled = false,
|
|
bool? enableImpeller,
|
|
}) {
|
|
return GalleryTransitionTest(
|
|
testFile: semanticsEnabled
|
|
? 'transitions_perf_e2e_with_semantics'
|
|
: 'transitions_perf_e2e',
|
|
needFullTimeline: false,
|
|
timelineSummaryFile: 'e2e_perf_summary',
|
|
transitionDurationFile: null,
|
|
timelineTraceFile: null,
|
|
driverFile: 'transitions_perf_e2e_test',
|
|
enableImpeller: enableImpeller,
|
|
).call;
|
|
}
|
|
|
|
TaskFunction createGalleryTransitionHybridBuildTest(
|
|
List<String> args, {
|
|
bool semanticsEnabled = false,
|
|
}) {
|
|
return GalleryTransitionBuildTest(
|
|
args,
|
|
semanticsEnabled: semanticsEnabled,
|
|
driverFile: semanticsEnabled ? 'transitions_perf_hybrid_with_semantics_test' : 'transitions_perf_hybrid_test',
|
|
).call;
|
|
}
|
|
|
|
TaskFunction createGalleryTransitionHybridTest({bool semanticsEnabled = false}) {
|
|
return GalleryTransitionTest(
|
|
semanticsEnabled: semanticsEnabled,
|
|
driverFile: semanticsEnabled
|
|
? 'transitions_perf_hybrid_with_semantics_test'
|
|
: 'transitions_perf_hybrid_test',
|
|
).call;
|
|
}
|
|
|
|
class GalleryTransitionTest {
|
|
|
|
GalleryTransitionTest({
|
|
this.semanticsEnabled = false,
|
|
this.testFile = 'transitions_perf',
|
|
this.needFullTimeline = true,
|
|
this.timelineSummaryFile = 'transitions.timeline_summary',
|
|
this.timelineTraceFile = 'transitions.timeline',
|
|
this.transitionDurationFile = 'transition_durations.timeline',
|
|
this.driverFile,
|
|
this.measureCpuGpu = true,
|
|
this.measureMemory = true,
|
|
this.enableImpeller,
|
|
});
|
|
|
|
final bool semanticsEnabled;
|
|
final bool needFullTimeline;
|
|
final bool measureCpuGpu;
|
|
final bool measureMemory;
|
|
final bool? enableImpeller;
|
|
final String testFile;
|
|
final String timelineSummaryFile;
|
|
final String? timelineTraceFile;
|
|
final String? transitionDurationFile;
|
|
final String? driverFile;
|
|
|
|
Future<TaskResult> call() async {
|
|
final Device device = await devices.workingDevice;
|
|
await device.unlock();
|
|
final String deviceId = device.deviceId;
|
|
final Directory galleryDirectory = dir('${flutterDirectory.path}/dev/integration_tests/flutter_gallery');
|
|
await inDirectory<void>(galleryDirectory, () async {
|
|
String? applicationBinaryPath;
|
|
if (deviceOperatingSystem == DeviceOperatingSystem.android) {
|
|
section('BUILDING APPLICATION');
|
|
await flutter(
|
|
'build',
|
|
options: <String>[
|
|
'apk',
|
|
'--no-android-gradle-daemon',
|
|
'--profile',
|
|
'-t',
|
|
'test_driver/$testFile.dart',
|
|
'--target-platform',
|
|
'android-arm,android-arm64',
|
|
],
|
|
);
|
|
applicationBinaryPath = 'build/app/outputs/flutter-apk/app-profile.apk';
|
|
}
|
|
|
|
final String testDriver = driverFile ?? (semanticsEnabled
|
|
? '${testFile}_with_semantics_test'
|
|
: '${testFile}_test');
|
|
section('DRIVE START');
|
|
await flutter('drive', options: <String>[
|
|
'--no-dds',
|
|
'--profile',
|
|
if (enableImpeller != null && enableImpeller!) '--enable-impeller',
|
|
if (enableImpeller != null && !enableImpeller!) '--no-enable-impeller',
|
|
if (needFullTimeline)
|
|
'--trace-startup',
|
|
if (applicationBinaryPath != null)
|
|
'--use-application-binary=$applicationBinaryPath'
|
|
else
|
|
...<String>[
|
|
'-t',
|
|
'test_driver/$testFile.dart',
|
|
],
|
|
'--driver',
|
|
'test_driver/$testDriver.dart',
|
|
'-d',
|
|
deviceId,
|
|
'-v',
|
|
'--verbose-system-logs'
|
|
]);
|
|
});
|
|
|
|
final String testOutputDirectory = Platform.environment['FLUTTER_TEST_OUTPUTS_DIR'] ?? '${galleryDirectory.path}/build';
|
|
final Map<String, dynamic> summary = json.decode(
|
|
file('$testOutputDirectory/$timelineSummaryFile.json').readAsStringSync(),
|
|
) as Map<String, dynamic>;
|
|
|
|
if (transitionDurationFile != null) {
|
|
final Map<String, dynamic> original = json.decode(
|
|
file('$testOutputDirectory/$transitionDurationFile.json').readAsStringSync(),
|
|
) as Map<String, dynamic>;
|
|
final Map<String, List<int>> transitions = <String, List<int>>{};
|
|
for (final String key in original.keys) {
|
|
transitions[key] = List<int>.from(original[key] as List<dynamic>);
|
|
}
|
|
summary['transitions'] = transitions;
|
|
summary['missed_transition_count'] = _countMissedTransitions(transitions);
|
|
}
|
|
|
|
final bool isAndroid = deviceOperatingSystem == DeviceOperatingSystem.android;
|
|
return TaskResult.success(summary,
|
|
detailFiles: <String>[
|
|
if (transitionDurationFile != null)
|
|
'$testOutputDirectory/$transitionDurationFile.json',
|
|
if (timelineTraceFile != null)
|
|
'$testOutputDirectory/$timelineTraceFile.json',
|
|
],
|
|
benchmarkScoreKeys: <String>[
|
|
if (transitionDurationFile != null)
|
|
'missed_transition_count',
|
|
'average_frame_build_time_millis',
|
|
'worst_frame_build_time_millis',
|
|
'90th_percentile_frame_build_time_millis',
|
|
'99th_percentile_frame_build_time_millis',
|
|
'average_frame_rasterizer_time_millis',
|
|
'stddev_frame_rasterizer_time_millis',
|
|
'worst_frame_rasterizer_time_millis',
|
|
'90th_percentile_frame_rasterizer_time_millis',
|
|
'99th_percentile_frame_rasterizer_time_millis',
|
|
'average_layer_cache_count',
|
|
'90th_percentile_layer_cache_count',
|
|
'99th_percentile_layer_cache_count',
|
|
'worst_layer_cache_count',
|
|
'average_layer_cache_memory',
|
|
'90th_percentile_layer_cache_memory',
|
|
'99th_percentile_layer_cache_memory',
|
|
'worst_layer_cache_memory',
|
|
'average_picture_cache_count',
|
|
'90th_percentile_picture_cache_count',
|
|
'99th_percentile_picture_cache_count',
|
|
'worst_picture_cache_count',
|
|
'average_picture_cache_memory',
|
|
'90th_percentile_picture_cache_memory',
|
|
'99th_percentile_picture_cache_memory',
|
|
'worst_picture_cache_memory',
|
|
if (measureCpuGpu && !isAndroid) ...<String>[
|
|
// See https://github.com/flutter/flutter/issues/68888
|
|
if (summary['average_cpu_usage'] != null) 'average_cpu_usage',
|
|
if (summary['average_gpu_usage'] != null) 'average_gpu_usage',
|
|
],
|
|
if (measureMemory && !isAndroid) ...<String>[
|
|
// See https://github.com/flutter/flutter/issues/68888
|
|
if (summary['average_memory_usage'] != null) 'average_memory_usage',
|
|
if (summary['90th_percentile_memory_usage'] != null) '90th_percentile_memory_usage',
|
|
if (summary['99th_percentile_memory_usage'] != null) '99th_percentile_memory_usage',
|
|
],
|
|
],
|
|
);
|
|
}
|
|
}
|
|
|
|
class GalleryTransitionBuildTest extends BuildTestTask {
|
|
GalleryTransitionBuildTest(
|
|
super.args, {
|
|
this.semanticsEnabled = false,
|
|
this.testFile = 'transitions_perf',
|
|
this.needFullTimeline = true,
|
|
this.timelineSummaryFile = 'transitions.timeline_summary',
|
|
this.timelineTraceFile = 'transitions.timeline',
|
|
this.transitionDurationFile = 'transition_durations.timeline',
|
|
this.driverFile,
|
|
this.measureCpuGpu = true,
|
|
this.measureMemory = true,
|
|
this.enableImpeller,
|
|
}) : super(workingDirectory: galleryDirectory);
|
|
|
|
final bool semanticsEnabled;
|
|
final bool needFullTimeline;
|
|
final bool measureCpuGpu;
|
|
final bool measureMemory;
|
|
final bool? enableImpeller;
|
|
final String testFile;
|
|
final String timelineSummaryFile;
|
|
final String? timelineTraceFile;
|
|
final String? transitionDurationFile;
|
|
final String? driverFile;
|
|
|
|
final String testOutputDirectory = Platform.environment['FLUTTER_TEST_OUTPUTS_DIR'] ?? '${galleryDirectory.path}/build';
|
|
|
|
@override
|
|
void copyArtifacts() {
|
|
if (applicationBinaryPath == null) {
|
|
return;
|
|
}
|
|
if (deviceOperatingSystem == DeviceOperatingSystem.android) {
|
|
copy(
|
|
file('${galleryDirectory.path}/build/app/outputs/flutter-apk/app-profile.apk'),
|
|
Directory(applicationBinaryPath!),
|
|
);
|
|
} else if (deviceOperatingSystem == DeviceOperatingSystem.ios) {
|
|
recursiveCopy(
|
|
Directory('${galleryDirectory.path}/build/ios/iphoneos'),
|
|
Directory(applicationBinaryPath!),
|
|
);
|
|
}
|
|
}
|
|
|
|
@override
|
|
List<String> getBuildArgs(DeviceOperatingSystem deviceOperatingSystem) {
|
|
if (deviceOperatingSystem == DeviceOperatingSystem.android) {
|
|
return <String>[
|
|
'apk',
|
|
'--no-android-gradle-daemon',
|
|
'--profile',
|
|
'-t',
|
|
'test_driver/$testFile.dart',
|
|
'--target-platform',
|
|
'android-arm,android-arm64',
|
|
];
|
|
} else if (deviceOperatingSystem == DeviceOperatingSystem.ios) {
|
|
return <String>[
|
|
'ios',
|
|
'--codesign',
|
|
'--profile',
|
|
'-t',
|
|
'test_driver/$testFile.dart',
|
|
];
|
|
}
|
|
throw Exception('$deviceOperatingSystem has no build configuration');
|
|
}
|
|
|
|
@override
|
|
List<String> getTestArgs(DeviceOperatingSystem deviceOperatingSystem, String deviceId) {
|
|
final String testDriver = driverFile ?? (semanticsEnabled ? '${testFile}_with_semantics_test' : '${testFile}_test');
|
|
return <String>[
|
|
'--no-dds',
|
|
'--profile',
|
|
if (enableImpeller != null && enableImpeller!) '--enable-impeller',
|
|
if (enableImpeller != null && !enableImpeller!) '--no-enable-impeller',
|
|
if (needFullTimeline) '--trace-startup',
|
|
'-t',
|
|
'test_driver/$testFile.dart',
|
|
if (applicationBinaryPath != null) '--use-application-binary=${getApplicationBinaryPath()}',
|
|
'--driver',
|
|
'test_driver/$testDriver.dart',
|
|
'-d',
|
|
deviceId,
|
|
];
|
|
}
|
|
|
|
@override
|
|
Future<TaskResult> parseTaskResult() async {
|
|
final Map<String, dynamic> summary = json.decode(
|
|
file('$testOutputDirectory/$timelineSummaryFile.json').readAsStringSync(),
|
|
) as Map<String, dynamic>;
|
|
|
|
if (transitionDurationFile != null) {
|
|
final Map<String, dynamic> original = json.decode(
|
|
file('$testOutputDirectory/$transitionDurationFile.json').readAsStringSync(),
|
|
) as Map<String, dynamic>;
|
|
final Map<String, List<int>> transitions = <String, List<int>>{};
|
|
for (final String key in original.keys) {
|
|
transitions[key] = List<int>.from(original[key] as List<dynamic>);
|
|
}
|
|
summary['transitions'] = transitions;
|
|
summary['missed_transition_count'] = _countMissedTransitions(transitions);
|
|
}
|
|
|
|
final bool isAndroid = deviceOperatingSystem == DeviceOperatingSystem.android;
|
|
return TaskResult.success(
|
|
summary,
|
|
detailFiles: <String>[
|
|
if (transitionDurationFile != null) '$testOutputDirectory/$transitionDurationFile.json',
|
|
if (timelineTraceFile != null) '$testOutputDirectory/$timelineTraceFile.json',
|
|
],
|
|
benchmarkScoreKeys: <String>[
|
|
if (transitionDurationFile != null) 'missed_transition_count',
|
|
'average_frame_build_time_millis',
|
|
'worst_frame_build_time_millis',
|
|
'90th_percentile_frame_build_time_millis',
|
|
'99th_percentile_frame_build_time_millis',
|
|
'average_frame_rasterizer_time_millis',
|
|
'worst_frame_rasterizer_time_millis',
|
|
'90th_percentile_frame_rasterizer_time_millis',
|
|
'99th_percentile_frame_rasterizer_time_millis',
|
|
'average_layer_cache_count',
|
|
'90th_percentile_layer_cache_count',
|
|
'99th_percentile_layer_cache_count',
|
|
'worst_layer_cache_count',
|
|
'average_layer_cache_memory',
|
|
'90th_percentile_layer_cache_memory',
|
|
'99th_percentile_layer_cache_memory',
|
|
'worst_layer_cache_memory',
|
|
'average_picture_cache_count',
|
|
'90th_percentile_picture_cache_count',
|
|
'99th_percentile_picture_cache_count',
|
|
'worst_picture_cache_count',
|
|
'average_picture_cache_memory',
|
|
'90th_percentile_picture_cache_memory',
|
|
'99th_percentile_picture_cache_memory',
|
|
'worst_picture_cache_memory',
|
|
if (measureCpuGpu && !isAndroid) ...<String>[
|
|
// See https://github.com/flutter/flutter/issues/68888
|
|
if (summary['average_cpu_usage'] != null) 'average_cpu_usage',
|
|
if (summary['average_gpu_usage'] != null) 'average_gpu_usage',
|
|
],
|
|
if (measureMemory && !isAndroid) ...<String>[
|
|
// See https://github.com/flutter/flutter/issues/68888
|
|
if (summary['average_memory_usage'] != null) 'average_memory_usage',
|
|
if (summary['90th_percentile_memory_usage'] != null) '90th_percentile_memory_usage',
|
|
if (summary['99th_percentile_memory_usage'] != null) '99th_percentile_memory_usage',
|
|
],
|
|
],
|
|
);
|
|
}
|
|
|
|
@override
|
|
String getApplicationBinaryPath() {
|
|
if (deviceOperatingSystem == DeviceOperatingSystem.android) {
|
|
return '$applicationBinaryPath/app-profile.apk';
|
|
} else if (deviceOperatingSystem == DeviceOperatingSystem.ios) {
|
|
return '$applicationBinaryPath/Flutter Gallery.app';
|
|
} else {
|
|
return applicationBinaryPath!;
|
|
}
|
|
}
|
|
}
|
|
|
|
int _countMissedTransitions(Map<String, List<int>> transitions) {
|
|
const int kTransitionBudget = 100000; // µs
|
|
int count = 0;
|
|
transitions.forEach((String demoName, List<int> durations) {
|
|
final int longestDuration = durations.reduce(math.max);
|
|
if (longestDuration > kTransitionBudget) {
|
|
print('$demoName missed transition time budget ($longestDuration µs > $kTransitionBudget µs)');
|
|
count++;
|
|
}
|
|
});
|
|
return count;
|
|
}
|