diff --git a/dev/bots/test.dart b/dev/bots/test.dart index 2eb3596e230..1e467763125 100644 --- a/dev/bots/test.dart +++ b/dev/bots/test.dart @@ -333,11 +333,18 @@ Future _runBuildTests() async { ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'non_nullable'))) ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'ui'))); + final List devicelabBuildTasks = [ + 'flutter_gallery__transition_perf', + 'flutter_gallery_ios__transition_perf', + ]; + // The tests are randomly distributed into subshards so as to get a uniform // distribution of costs, but the seed is fixed so that issues are reproducible. final List tests = [ for (final FileSystemEntity exampleDirectory in exampleDirectories) () => _runExampleProjectBuildTests(exampleDirectory), + for (String devicelabBuildTask in devicelabBuildTasks) + () => _runDeviceLabBuildTask(devicelabBuildTask), ...[ // Web compilation tests. () => _flutterBuildDart2js( @@ -355,6 +362,26 @@ Future _runBuildTests() async { await _runShardRunnerIndexOfTotalSubshard(tests); } + +Future _runDeviceLabBuildTask(String task) async { + // Run the ios tasks + if (!Platform.isMacOS && task.contains('_ios_')) { + return; + } + + final String targetPlatform = (task.contains('_ios_')) ? 'ios' : 'android'; + + await runCommand(dart, [ + 'run', path.join('bin', 'test_runner.dart'), + 'test', + '--task', task, + '--task-args', 'build', + '--task-args', 'target-platform=$targetPlatform', + ], + workingDirectory: path.join('dev', 'devicelab')); +} + + Future _runExampleProjectBuildTests(FileSystemEntity exampleDirectory) async { // Only verify caching with flutter gallery. final bool verifyCaching = exampleDirectory.path.contains('flutter_gallery'); diff --git a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf.dart b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf.dart index 0fa2727e15c..750b4b66eff 100644 --- a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf.dart +++ b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf.dart @@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart'; import 'package:flutter_devicelab/framework/adb.dart'; import 'package:flutter_devicelab/framework/framework.dart'; -Future main() async { +Future main(List args) async { deviceOperatingSystem = DeviceOperatingSystem.android; - await task(createGalleryTransitionTest()); + await task(createGalleryTransitionTest(args)); } diff --git a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e.dart b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e.dart index ff14de41934..c2814bd92d3 100644 --- a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e.dart +++ b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e.dart @@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart'; import 'package:flutter_devicelab/framework/adb.dart'; import 'package:flutter_devicelab/framework/framework.dart'; -Future main() async { +Future main(List args) async { deviceOperatingSystem = DeviceOperatingSystem.android; - await task(createGalleryTransitionE2ETest()); + await task(createGalleryTransitionE2ETest(args)); } diff --git a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e_ios.dart b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e_ios.dart index eafa896484f..ac68d853126 100644 --- a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e_ios.dart +++ b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e_ios.dart @@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart'; import 'package:flutter_devicelab/framework/adb.dart'; import 'package:flutter_devicelab/framework/framework.dart'; -Future main() async { +Future main(List args) async { deviceOperatingSystem = DeviceOperatingSystem.ios; - await task(createGalleryTransitionE2ETest()); + await task(createGalleryTransitionE2ETest(args)); } diff --git a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e_ios32.dart b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e_ios32.dart index eafa896484f..ac68d853126 100644 --- a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e_ios32.dart +++ b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e_ios32.dart @@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart'; import 'package:flutter_devicelab/framework/adb.dart'; import 'package:flutter_devicelab/framework/framework.dart'; -Future main() async { +Future main(List args) async { deviceOperatingSystem = DeviceOperatingSystem.ios; - await task(createGalleryTransitionE2ETest()); + await task(createGalleryTransitionE2ETest(args)); } diff --git a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_hybrid.dart b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_hybrid.dart index 7a53970fe80..26175264bf8 100644 --- a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_hybrid.dart +++ b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_hybrid.dart @@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart'; import 'package:flutter_devicelab/framework/adb.dart'; import 'package:flutter_devicelab/framework/framework.dart'; -Future main() async { +Future main(List args) async { deviceOperatingSystem = DeviceOperatingSystem.android; - await task(createGalleryTransitionHybridTest()); + await task(createGalleryTransitionHybridTest(args)); } diff --git a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_with_semantics.dart b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_with_semantics.dart index 6af25a19845..cb3a5119eb6 100644 --- a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_with_semantics.dart +++ b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_with_semantics.dart @@ -7,11 +7,11 @@ import 'package:flutter_devicelab/framework/adb.dart'; import 'package:flutter_devicelab/framework/framework.dart'; import 'package:flutter_devicelab/framework/task_result.dart'; -Future main() async { +Future main(List args) async { deviceOperatingSystem = DeviceOperatingSystem.android; await task(() async { - final TaskResult withoutSemantics = await createGalleryTransitionTest()(); - final TaskResult withSemantics = await createGalleryTransitionTest(semanticsEnabled: true)(); + final TaskResult withoutSemantics = await createGalleryTransitionTest(args)(); + final TaskResult withSemantics = await createGalleryTransitionTest(args, semanticsEnabled: true)(); if (withSemantics.benchmarkScoreKeys.isEmpty || withoutSemantics.benchmarkScoreKeys.isEmpty) { String message = 'Lack of data'; if (withSemantics.benchmarkScoreKeys.isEmpty) { diff --git a/dev/devicelab/bin/tasks/flutter_gallery_ios__transition_perf.dart b/dev/devicelab/bin/tasks/flutter_gallery_ios__transition_perf.dart index 951828480e0..003947215b8 100644 --- a/dev/devicelab/bin/tasks/flutter_gallery_ios__transition_perf.dart +++ b/dev/devicelab/bin/tasks/flutter_gallery_ios__transition_perf.dart @@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart'; import 'package:flutter_devicelab/framework/adb.dart'; import 'package:flutter_devicelab/framework/framework.dart'; -Future main() async { +Future main(List args) async { deviceOperatingSystem = DeviceOperatingSystem.ios; - await task(createGalleryTransitionTest()); + await task(createGalleryTransitionTest(args)); } diff --git a/dev/devicelab/lib/command/test.dart b/dev/devicelab/lib/command/test.dart index 369fa80e697..98c5d91547d 100644 --- a/dev/devicelab/lib/command/test.dart +++ b/dev/devicelab/lib/command/test.dart @@ -65,7 +65,6 @@ class TestCommand extends Command { final List taskArgsRaw = argResults['task-args'] as List; // Prepend '--' to convert args to options when passed to task final List taskArgs = taskArgsRaw.map((String taskArg) => '--$taskArg').toList(); - print(taskArgs); await runTasks( [argResults['task'] as String], deviceId: argResults['device-id'] as String, diff --git a/dev/devicelab/lib/framework/adb.dart b/dev/devicelab/lib/framework/adb.dart index 4a0248a92f9..7b55e23b5e0 100644 --- a/dev/devicelab/lib/framework/adb.dart +++ b/dev/devicelab/lib/framework/adb.dart @@ -54,6 +54,26 @@ DeviceDiscovery get devices => DeviceDiscovery(); /// Device operating system the test is configured to test. enum DeviceOperatingSystem { android, androidArm, androidArm64 ,ios, fuchsia, fake } +/// Helper function to allow passing the target platform as a task arg instead +/// of hardcoding it in the task. +DeviceOperatingSystem deviceOperatingSystemFromString(String os) { + switch (os) { + case 'android': + return DeviceOperatingSystem.android; + case 'android_arm': + return DeviceOperatingSystem.androidArm; + case 'android_arm64': + return DeviceOperatingSystem.androidArm64; + case 'fake': + return DeviceOperatingSystem.fake; + case 'fuchsia': + return DeviceOperatingSystem.fuchsia; + case 'ios': + return DeviceOperatingSystem.ios; + } + throw UnimplementedError('$os is not defined in function deviceOperatingSystemFromString'); +} + /// Device OS to test on. DeviceOperatingSystem deviceOperatingSystem = DeviceOperatingSystem.android; diff --git a/dev/devicelab/lib/tasks/build_test_task.dart b/dev/devicelab/lib/tasks/build_test_task.dart index 8d4ad4a1ae9..3ff2f30212e 100644 --- a/dev/devicelab/lib/tasks/build_test_task.dart +++ b/dev/devicelab/lib/tasks/build_test_task.dart @@ -19,16 +19,23 @@ abstract class BuildTestTask { applicationBinaryPath = argResults[kApplicationBinaryPathOption] as String; buildOnly = argResults[kBuildOnlyFlag] as bool; testOnly = argResults[kTestOnlyFlag] as bool; - + if (argResults.wasParsed(kTargetPlatformOption)) { + // Override deviceOperatingSystem to prevent extra utilities from being used. + targetPlatform = deviceOperatingSystemFromString(argResults[kTargetPlatformOption] as String); + _originalDeviceOperatingSystem = deviceOperatingSystem; + deviceOperatingSystem = DeviceOperatingSystem.fake; + } } static const String kApplicationBinaryPathOption = 'application-binary-path'; static const String kBuildOnlyFlag = 'build'; + static const String kTargetPlatformOption = 'target-platform'; static const String kTestOnlyFlag = 'test'; final ArgParser argParser = ArgParser() ..addOption(kApplicationBinaryPathOption) ..addFlag(kBuildOnlyFlag) + ..addOption(kTargetPlatformOption) ..addFlag(kTestOnlyFlag); /// Args passed from the test runner via "--task-arg". @@ -37,6 +44,15 @@ abstract class BuildTestTask { /// If true, skip [test]. bool buildOnly = false; + /// The [DeviceOperatingSystem] being targeted for this build. + /// + /// If passed, no connected device checks are run as the current connected device + /// will be set as [DeviceOperatingSystem.fake]. + DeviceOperatingSystem targetPlatform; + + /// Original [deviceOperatingSystem] if [targetPlatform] is given. + DeviceOperatingSystem _originalDeviceOperatingSystem; + /// If true, skip [build]. bool testOnly = false; @@ -59,7 +75,7 @@ abstract class BuildTestTask { await flutter('clean'); } section('BUILDING APPLICATION'); - await flutter('build', options: getBuildArgs(deviceOperatingSystem)); + await flutter('build', options: getBuildArgs()); }); } @@ -68,21 +84,25 @@ abstract class BuildTestTask { /// /// This assumes that [applicationBinaryPath] exists. Future test() async { + // Ensure deviceOperatingSystem is the one set in bin/task. + if (deviceOperatingSystem == DeviceOperatingSystem.fake) { + deviceOperatingSystem = _originalDeviceOperatingSystem; + } final Device device = await devices.workingDevice; await device.unlock(); await inDirectory(workingDirectory, () async { section('DRIVE START'); - await flutter('drive', options: getTestArgs(deviceOperatingSystem, device.deviceId)); + await flutter('drive', options: getTestArgs(device.deviceId)); }); return parseTaskResult(); } /// Args passed to flutter build to build the application under test. - List getBuildArgs(DeviceOperatingSystem deviceOperatingSystem) => throw UnimplementedError('getBuildArgs is not implemented'); + List getBuildArgs() => throw UnimplementedError('getBuildArgs is not implemented'); /// Args passed to flutter drive to test the built application. - List getTestArgs(DeviceOperatingSystem deviceOperatingSystem, String deviceId) => throw UnimplementedError('getTestArgs is not implemented'); + List getTestArgs(String deviceId) => throw UnimplementedError('getTestArgs is not implemented'); /// Logic to construct [TaskResult] from this test's results. Future parseTaskResult() => throw UnimplementedError('parseTaskResult is not implemented'); @@ -106,7 +126,7 @@ abstract class BuildTestTask { } if (!testOnly) { - build(); + await build(); } if (buildOnly) { diff --git a/dev/devicelab/lib/tasks/gallery.dart b/dev/devicelab/lib/tasks/gallery.dart index a6994fb9ff8..dfe9391402b 100644 --- a/dev/devicelab/lib/tasks/gallery.dart +++ b/dev/devicelab/lib/tasks/gallery.dart @@ -11,13 +11,17 @@ import '../framework/adb.dart'; import '../framework/framework.dart'; import '../framework/task_result.dart'; import '../framework/utils.dart'; +import 'build_test_task.dart'; -TaskFunction createGalleryTransitionTest({bool semanticsEnabled = false}) { - return GalleryTransitionTest(semanticsEnabled: semanticsEnabled); +final Directory galleryDirectory = dir('${flutterDirectory.path}/dev/integration_tests/flutter_gallery'); + +TaskFunction createGalleryTransitionTest(List args, {bool semanticsEnabled = false}) { + return GalleryTransitionTest(args, semanticsEnabled: semanticsEnabled, workingDirectory: galleryDirectory,); } -TaskFunction createGalleryTransitionE2ETest({bool semanticsEnabled = false}) { +TaskFunction createGalleryTransitionE2ETest(List args, {bool semanticsEnabled = false}) { return GalleryTransitionTest( + args, testFile: semanticsEnabled ? 'transitions_perf_e2e_with_semantics' : 'transitions_perf_e2e', @@ -26,21 +30,23 @@ TaskFunction createGalleryTransitionE2ETest({bool semanticsEnabled = false}) { transitionDurationFile: null, timelineTraceFile: null, driverFile: 'transitions_perf_e2e_test', + workingDirectory: galleryDirectory, ); } -TaskFunction createGalleryTransitionHybridTest({bool semanticsEnabled = false}) { +TaskFunction createGalleryTransitionHybridTest(List args, {bool semanticsEnabled = false}) { return GalleryTransitionTest( + args, semanticsEnabled: semanticsEnabled, driverFile: semanticsEnabled ? 'transitions_perf_hybrid_with_semantics_test' : 'transitions_perf_hybrid_test', + workingDirectory: galleryDirectory, ); } -class GalleryTransitionTest { - - GalleryTransitionTest({ +class GalleryTransitionTest extends BuildTestTask { + GalleryTransitionTest(List args, { this.semanticsEnabled = false, this.testFile = 'transitions_perf', this.needFullTimeline = true, @@ -48,7 +54,8 @@ class GalleryTransitionTest { this.timelineTraceFile = 'transitions.timeline', this.transitionDurationFile = 'transition_durations.timeline', this.driverFile, - }); + Directory workingDirectory, + }) : super(args, workingDirectory: workingDirectory); final bool semanticsEnabled; final bool needFullTimeline; @@ -58,59 +65,58 @@ class GalleryTransitionTest { final String transitionDurationFile; final String driverFile; - Future call() async { - final Device device = await devices.workingDevice; - await device.unlock(); - final String deviceId = device.deviceId; - final Directory galleryDirectory = dir('${flutterDirectory.path}/dev/integration_tests/flutter_gallery'); - await inDirectory(galleryDirectory, () async { - String applicationBinaryPath; - if (deviceOperatingSystem == DeviceOperatingSystem.android) { - section('BUILDING APPLICATION'); - await flutter( - 'build', - options: [ - 'apk', - '--no-android-gradle-daemon', + @override + List getBuildArgs() { + switch (targetPlatform) { + case DeviceOperatingSystem.android: + return [ + 'apk', + '--no-android-gradle-daemon', + '--profile', + '-t', + 'test_driver/$testFile.dart', + '--target-platform', + 'android-arm,android-arm64', + ]; + case DeviceOperatingSystem.ios: + return [ + 'ios', + // Skip codesign on presubmit checks + if (targetPlatform != null) + '--no-codesign', '--profile', '-t', 'test_driver/$testFile.dart', - '--target-platform', - 'android-arm,android-arm64', - ], - ); - applicationBinaryPath = 'build/app/outputs/flutter-apk/app-profile.apk'; + ]; + default: + throw Exception('$deviceOperatingSystem has no build configuration'); } + } - final String testDriver = driverFile ?? (semanticsEnabled - ? '${testFile}_with_semantics_test' - : '${testFile}_test'); - section('DRIVE START'); - await flutter('drive', options: [ + @override + List getTestArgs(String deviceId) { + final String testDriver = driverFile ?? (semanticsEnabled + ? '${testFile}_with_semantics_test' + : '${testFile}_test'); + return [ '--profile', if (needFullTimeline) '--trace-startup', - if (applicationBinaryPath != null) - '--use-application-binary=$applicationBinaryPath' - else - ...[ - '-t', - 'test_driver/$testFile.dart', - ], - '--driver', - 'test_driver/$testDriver.dart', - '-d', - deviceId, - ]); - }); + '--use-application-binary="${getApplicationBinaryPath()}"', + '--driver', 'test_driver/$testDriver.dart', + '-d', deviceId, + ]; + } + @override + Future parseTaskResult() async { final Map summary = json.decode( - file('${galleryDirectory.path}/build/$timelineSummaryFile.json').readAsStringSync(), + file('${workingDirectory.path}/build/$timelineSummaryFile.json').readAsStringSync(), ) as Map; if (transitionDurationFile != null) { final Map original = json.decode( - file('${galleryDirectory.path}/build/$transitionDurationFile.json').readAsStringSync(), + file('${workingDirectory.path}/build/$transitionDurationFile.json').readAsStringSync(), ) as Map; final Map> transitions = >{}; for (final String key in original.keys) { @@ -123,9 +129,9 @@ class GalleryTransitionTest { return TaskResult.success(summary, detailFiles: [ if (transitionDurationFile != null) - '${galleryDirectory.path}/build/$transitionDurationFile.json', + '${workingDirectory.path}/build/$transitionDurationFile.json', if (timelineTraceFile != null) - '${galleryDirectory.path}/build/$timelineTraceFile.json' + '${workingDirectory.path}/build/$timelineTraceFile.json' ], benchmarkScoreKeys: [ if (transitionDurationFile != null) @@ -141,6 +147,22 @@ class GalleryTransitionTest { ], ); } + + @override + String getApplicationBinaryPath() { + if (applicationBinaryPath != null) { + return applicationBinaryPath; + } + + switch (targetPlatform) { + case DeviceOperatingSystem.android: + return 'build/app/outputs/flutter-apk/app-profile.apk'; + case DeviceOperatingSystem.ios: + return 'build/ios/iphoneos/Flutter Gallery.app'; + default: + throw UnimplementedError('getApplicationBinaryPath does not support $deviceOperatingSystem'); + } + } } int _countMissedTransitions(Map> transitions) {