// Copyright (c) 2016 The Chromium Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. import 'dart:async'; import 'dart:convert' show JSON; import '../framework/adb.dart'; import '../framework/framework.dart'; import '../framework/ios.dart'; import '../framework/utils.dart'; TaskFunction createComplexLayoutScrollPerfTest() { return new PerfTest( '${flutterDirectory.path}/dev/benchmarks/complex_layout', 'test_driver/scroll_perf.dart', 'complex_layout_scroll_perf', ); } TaskFunction createComplexLayoutScrollMemoryTest() { return new MemoryTest( '${flutterDirectory.path}/dev/benchmarks/complex_layout', 'com.yourcompany.complexLayout', testTarget: 'test_driver/scroll_perf.dart', ); } TaskFunction createFlutterGalleryStartupTest() { return new StartupTest( '${flutterDirectory.path}/examples/flutter_gallery', ); } TaskFunction createComplexLayoutStartupTest() { return new StartupTest( '${flutterDirectory.path}/dev/benchmarks/complex_layout', ); } TaskFunction createFlutterGalleryBuildTest() { return new BuildTest('${flutterDirectory.path}/examples/flutter_gallery'); } TaskFunction createComplexLayoutBuildTest() { return new BuildTest('${flutterDirectory.path}/dev/benchmarks/complex_layout'); } TaskFunction createHelloWorldMemoryTest() { return new MemoryTest( '${flutterDirectory.path}/examples/hello_world', 'io.flutter.examples.hello_world', ); } TaskFunction createGalleryNavigationMemoryTest() { return new MemoryTest( '${flutterDirectory.path}/examples/flutter_gallery', 'io.flutter.examples.gallery', testTarget: 'test_driver/memory_nav.dart', ); } TaskFunction createGalleryBackButtonMemoryTest() { return new AndroidBackButtonMemoryTest( '${flutterDirectory.path}/examples/flutter_gallery', 'io.flutter.examples.gallery', 'io.flutter.app.FlutterActivity', ); } TaskFunction createFlutterViewStartupTest() { return new StartupTest( '${flutterDirectory.path}/examples/flutter_view', reportMetrics: false, // This project has a non-standard CocoaPods Podfile. Run pod install // before building the project. runPodInstall: true, ); } /// Measure application startup performance. class StartupTest { static const Duration _startupTimeout = const Duration(minutes: 5); StartupTest(this.testDirectory, { this.reportMetrics: true, this.runPodInstall: false }); final String testDirectory; final bool reportMetrics; /// Used to trigger a `pod install` when the project has a custom Podfile and /// flutter build ios won't automatically run `pod install` via the managed /// plugin system. final bool runPodInstall; Future call() async { return await inDirectory(testDirectory, () async { final String deviceId = (await devices.workingDevice).deviceId; await flutter('packages', options: ['get']); if (deviceOperatingSystem == DeviceOperatingSystem.ios) { if (runPodInstall) await runPodInstallForCustomPodfile(testDirectory); await prepareProvisioningCertificates(testDirectory); // This causes an Xcode project to be created. await flutter('build', options: ['ios', '--profile']); } await flutter('run', options: [ '--verbose', '--profile', '--trace-startup', '-d', deviceId, ]).timeout(_startupTimeout); final Map data = JSON.decode(file('$testDirectory/build/start_up_info.json').readAsStringSync()); if (!reportMetrics) return new TaskResult.success(data); return new TaskResult.success(data, benchmarkScoreKeys: [ 'timeToFirstFrameMicros', ]); }); } } /// Measures application runtime performance, specifically per-frame /// performance. class PerfTest { PerfTest(this.testDirectory, this.testTarget, this.timelineFileName); final String testDirectory; final String testTarget; final String timelineFileName; Future call() { return inDirectory(testDirectory, () async { final Device device = await devices.workingDevice; await device.unlock(); final String deviceId = device.deviceId; await flutter('packages', options: ['get']); if (deviceOperatingSystem == DeviceOperatingSystem.ios) { await prepareProvisioningCertificates(testDirectory); // This causes an Xcode project to be created. await flutter('build', options: ['ios', '--profile']); } await flutter('drive', options: [ '-v', '--profile', '--trace-startup', // Enables "endless" timeline event buffering. '-t', testTarget, '-d', deviceId, ]); final Map data = JSON.decode(file('$testDirectory/build/$timelineFileName.timeline_summary.json').readAsStringSync()); if (data['frame_count'] < 5) { return new TaskResult.failure( 'Timeline contains too few frames: ${data['frame_count']}. Possibly ' 'trace events are not being captured.', ); } return new TaskResult.success(data, benchmarkScoreKeys: [ 'average_frame_build_time_millis', 'worst_frame_build_time_millis', 'missed_frame_build_budget_count', 'average_frame_rasterizer_time_millis', 'worst_frame_rasterizer_time_millis', 'missed_frame_rasterizer_budget_count', ]); }); } } class BuildTest { BuildTest(this.testDirectory); final String testDirectory; Future call() async { return await inDirectory(testDirectory, () async { final Device device = await devices.workingDevice; await device.unlock(); await flutter('packages', options: ['get']); final Stopwatch watch = new Stopwatch()..start(); final String buildLog = await evalFlutter('build', options: [ 'aot', '-v', '--profile', '--no-pub', '--target-platform', 'android-arm' // Generate blobs instead of assembly. ]); watch.stop(); final RegExp metricExpression = new RegExp(r'([a-zA-Z]+)\(CodeSize\)\: (\d+)'); final Map data = new Map.fromIterable( metricExpression.allMatches(buildLog), key: (Match m) => _sdkNameToMetricName(m.group(1)), value: (Match m) => int.parse(m.group(2)), ); data['aot_snapshot_build_millis'] = watch.elapsedMilliseconds; return new TaskResult.success(data, benchmarkScoreKeys: data.keys.toList()); }); } static String _sdkNameToMetricName(String sdkName) { const Map kSdkNameToMetricNameMapping = const { 'VMIsolate': 'aot_snapshot_size_vmisolate', 'Isolate': 'aot_snapshot_size_isolate', 'ReadOnlyData': 'aot_snapshot_size_rodata', 'Instructions': 'aot_snapshot_size_instructions', 'Total': 'aot_snapshot_size_total', }; if (!kSdkNameToMetricNameMapping.containsKey(sdkName)) throw 'Unrecognized SDK snapshot metric name: $sdkName'; return kSdkNameToMetricNameMapping[sdkName]; } } /// Measure application memory usage. class MemoryTest { MemoryTest(this.testDirectory, this.packageName, { this.testTarget }); final String testDirectory; final String packageName; /// Path to a flutter driver script that will run after starting the app. /// /// If not specified, then the test will start the app, gather statistics, and then exit. final String testTarget; Future call() { return inDirectory(testDirectory, () async { final Device device = await devices.workingDevice; await device.unlock(); final String deviceId = device.deviceId; await flutter('packages', options: ['get']); if (deviceOperatingSystem == DeviceOperatingSystem.ios) { await prepareProvisioningCertificates(testDirectory); // This causes an Xcode project to be created. await flutter('build', options: ['ios', '--profile']); } final int observatoryPort = await findAvailablePort(); final List runOptions = [ '-v', '--profile', '--trace-startup', // wait for the first frame to render '-d', deviceId, '--observatory-port', observatoryPort.toString(), ]; if (testTarget != null) runOptions.addAll(['-t', testTarget]); await flutter('run', options: runOptions); final Map startData = await device.getMemoryStats(packageName); final Map data = { 'start_total_kb': startData['total_kb'], }; if (testTarget != null) { await flutter('drive', options: [ '-v', '-t', testTarget, '-d', deviceId, '--use-existing-app=http://localhost:$observatoryPort', ]); final Map endData = await device.getMemoryStats(packageName); data['end_total_kb'] = endData['total_kb']; data['diff_total_kb'] = endData['total_kb'] - startData['total_kb']; } await device.stop(packageName); return new TaskResult.success(data, benchmarkScoreKeys: data.keys.toList()); }); } } /// Measure application memory usage after pausing and resuming the app /// with the Android back button. class AndroidBackButtonMemoryTest { final String testDirectory; final String packageName; final String activityName; AndroidBackButtonMemoryTest(this.testDirectory, this.packageName, this.activityName); Future call() { return inDirectory(testDirectory, () async { if (deviceOperatingSystem != DeviceOperatingSystem.android) { throw 'This test is only supported on Android'; } final AndroidDevice device = await devices.workingDevice; await device.unlock(); final String deviceId = device.deviceId; await flutter('packages', options: ['get']); await flutter('run', options: [ '-v', '--profile', '--trace-startup', // wait for the first frame to render '-d', deviceId, ]); final Map startData = await device.getMemoryStats(packageName); final Map data = { 'start_total_kb': startData['total_kb'], }; // Perform a series of back button suspend and resume cycles. for (int i = 0; i < 10; i++) { await device.shellExec('input', ['keyevent', 'KEYCODE_BACK']); await new Future.delayed(const Duration(milliseconds: 1000)); final String output = await device.shellEval('am', ['start', '-n', '$packageName/$activityName']); print(output); if (output.contains('Error')) return new TaskResult.failure('unable to launch activity'); await new Future.delayed(const Duration(milliseconds: 1000)); } final Map endData = await device.getMemoryStats(packageName); data['end_total_kb'] = endData['total_kb']; data['diff_total_kb'] = endData['total_kb'] - startData['total_kb']; await device.stop(packageName); return new TaskResult.success(data, benchmarkScoreKeys: data.keys.toList()); }); } }