diff --git a/dev/bots/test.dart b/dev/bots/test.dart index 1e467763125..2eb3596e230 100644 --- a/dev/bots/test.dart +++ b/dev/bots/test.dart @@ -333,18 +333,11 @@ Future _runBuildTests() async { ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'non_nullable'))) ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'ui'))); - final List devicelabBuildTasks = [ - 'flutter_gallery__transition_perf', - 'flutter_gallery_ios__transition_perf', - ]; - // The tests are randomly distributed into subshards so as to get a uniform // distribution of costs, but the seed is fixed so that issues are reproducible. final List tests = [ for (final FileSystemEntity exampleDirectory in exampleDirectories) () => _runExampleProjectBuildTests(exampleDirectory), - for (String devicelabBuildTask in devicelabBuildTasks) - () => _runDeviceLabBuildTask(devicelabBuildTask), ...[ // Web compilation tests. () => _flutterBuildDart2js( @@ -362,26 +355,6 @@ Future _runBuildTests() async { await _runShardRunnerIndexOfTotalSubshard(tests); } - -Future _runDeviceLabBuildTask(String task) async { - // Run the ios tasks - if (!Platform.isMacOS && task.contains('_ios_')) { - return; - } - - final String targetPlatform = (task.contains('_ios_')) ? 'ios' : 'android'; - - await runCommand(dart, [ - 'run', path.join('bin', 'test_runner.dart'), - 'test', - '--task', task, - '--task-args', 'build', - '--task-args', 'target-platform=$targetPlatform', - ], - workingDirectory: path.join('dev', 'devicelab')); -} - - Future _runExampleProjectBuildTests(FileSystemEntity exampleDirectory) async { // Only verify caching with flutter gallery. final bool verifyCaching = exampleDirectory.path.contains('flutter_gallery'); diff --git a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf.dart b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf.dart index 750b4b66eff..0fa2727e15c 100644 --- a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf.dart +++ b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf.dart @@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart'; import 'package:flutter_devicelab/framework/adb.dart'; import 'package:flutter_devicelab/framework/framework.dart'; -Future main(List args) async { +Future main() async { deviceOperatingSystem = DeviceOperatingSystem.android; - await task(createGalleryTransitionTest(args)); + await task(createGalleryTransitionTest()); } diff --git a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e.dart b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e.dart index c2814bd92d3..ff14de41934 100644 --- a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e.dart +++ b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e.dart @@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart'; import 'package:flutter_devicelab/framework/adb.dart'; import 'package:flutter_devicelab/framework/framework.dart'; -Future main(List args) async { +Future main() async { deviceOperatingSystem = DeviceOperatingSystem.android; - await task(createGalleryTransitionE2ETest(args)); + await task(createGalleryTransitionE2ETest()); } diff --git a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e_ios.dart b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e_ios.dart index ac68d853126..eafa896484f 100644 --- a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e_ios.dart +++ b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e_ios.dart @@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart'; import 'package:flutter_devicelab/framework/adb.dart'; import 'package:flutter_devicelab/framework/framework.dart'; -Future main(List args) async { +Future main() async { deviceOperatingSystem = DeviceOperatingSystem.ios; - await task(createGalleryTransitionE2ETest(args)); + await task(createGalleryTransitionE2ETest()); } diff --git a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e_ios32.dart b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e_ios32.dart index ac68d853126..eafa896484f 100644 --- a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e_ios32.dart +++ b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_e2e_ios32.dart @@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart'; import 'package:flutter_devicelab/framework/adb.dart'; import 'package:flutter_devicelab/framework/framework.dart'; -Future main(List args) async { +Future main() async { deviceOperatingSystem = DeviceOperatingSystem.ios; - await task(createGalleryTransitionE2ETest(args)); + await task(createGalleryTransitionE2ETest()); } diff --git a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_hybrid.dart b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_hybrid.dart index 26175264bf8..7a53970fe80 100644 --- a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_hybrid.dart +++ b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_hybrid.dart @@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart'; import 'package:flutter_devicelab/framework/adb.dart'; import 'package:flutter_devicelab/framework/framework.dart'; -Future main(List args) async { +Future main() async { deviceOperatingSystem = DeviceOperatingSystem.android; - await task(createGalleryTransitionHybridTest(args)); + await task(createGalleryTransitionHybridTest()); } diff --git a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_with_semantics.dart b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_with_semantics.dart index cb3a5119eb6..6af25a19845 100644 --- a/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_with_semantics.dart +++ b/dev/devicelab/bin/tasks/flutter_gallery__transition_perf_with_semantics.dart @@ -7,11 +7,11 @@ import 'package:flutter_devicelab/framework/adb.dart'; import 'package:flutter_devicelab/framework/framework.dart'; import 'package:flutter_devicelab/framework/task_result.dart'; -Future main(List args) async { +Future main() async { deviceOperatingSystem = DeviceOperatingSystem.android; await task(() async { - final TaskResult withoutSemantics = await createGalleryTransitionTest(args)(); - final TaskResult withSemantics = await createGalleryTransitionTest(args, semanticsEnabled: true)(); + final TaskResult withoutSemantics = await createGalleryTransitionTest()(); + final TaskResult withSemantics = await createGalleryTransitionTest(semanticsEnabled: true)(); if (withSemantics.benchmarkScoreKeys.isEmpty || withoutSemantics.benchmarkScoreKeys.isEmpty) { String message = 'Lack of data'; if (withSemantics.benchmarkScoreKeys.isEmpty) { diff --git a/dev/devicelab/bin/tasks/flutter_gallery_ios__transition_perf.dart b/dev/devicelab/bin/tasks/flutter_gallery_ios__transition_perf.dart index 003947215b8..951828480e0 100644 --- a/dev/devicelab/bin/tasks/flutter_gallery_ios__transition_perf.dart +++ b/dev/devicelab/bin/tasks/flutter_gallery_ios__transition_perf.dart @@ -6,7 +6,7 @@ import 'package:flutter_devicelab/tasks/gallery.dart'; import 'package:flutter_devicelab/framework/adb.dart'; import 'package:flutter_devicelab/framework/framework.dart'; -Future main(List args) async { +Future main() async { deviceOperatingSystem = DeviceOperatingSystem.ios; - await task(createGalleryTransitionTest(args)); + await task(createGalleryTransitionTest()); } diff --git a/dev/devicelab/lib/command/test.dart b/dev/devicelab/lib/command/test.dart index 98c5d91547d..369fa80e697 100644 --- a/dev/devicelab/lib/command/test.dart +++ b/dev/devicelab/lib/command/test.dart @@ -65,6 +65,7 @@ class TestCommand extends Command { final List taskArgsRaw = argResults['task-args'] as List; // Prepend '--' to convert args to options when passed to task final List taskArgs = taskArgsRaw.map((String taskArg) => '--$taskArg').toList(); + print(taskArgs); await runTasks( [argResults['task'] as String], deviceId: argResults['device-id'] as String, diff --git a/dev/devicelab/lib/framework/adb.dart b/dev/devicelab/lib/framework/adb.dart index 7b55e23b5e0..4a0248a92f9 100644 --- a/dev/devicelab/lib/framework/adb.dart +++ b/dev/devicelab/lib/framework/adb.dart @@ -54,26 +54,6 @@ DeviceDiscovery get devices => DeviceDiscovery(); /// Device operating system the test is configured to test. enum DeviceOperatingSystem { android, androidArm, androidArm64 ,ios, fuchsia, fake } -/// Helper function to allow passing the target platform as a task arg instead -/// of hardcoding it in the task. -DeviceOperatingSystem deviceOperatingSystemFromString(String os) { - switch (os) { - case 'android': - return DeviceOperatingSystem.android; - case 'android_arm': - return DeviceOperatingSystem.androidArm; - case 'android_arm64': - return DeviceOperatingSystem.androidArm64; - case 'fake': - return DeviceOperatingSystem.fake; - case 'fuchsia': - return DeviceOperatingSystem.fuchsia; - case 'ios': - return DeviceOperatingSystem.ios; - } - throw UnimplementedError('$os is not defined in function deviceOperatingSystemFromString'); -} - /// Device OS to test on. DeviceOperatingSystem deviceOperatingSystem = DeviceOperatingSystem.android; diff --git a/dev/devicelab/lib/tasks/build_test_task.dart b/dev/devicelab/lib/tasks/build_test_task.dart index 3ff2f30212e..8d4ad4a1ae9 100644 --- a/dev/devicelab/lib/tasks/build_test_task.dart +++ b/dev/devicelab/lib/tasks/build_test_task.dart @@ -19,23 +19,16 @@ abstract class BuildTestTask { applicationBinaryPath = argResults[kApplicationBinaryPathOption] as String; buildOnly = argResults[kBuildOnlyFlag] as bool; testOnly = argResults[kTestOnlyFlag] as bool; - if (argResults.wasParsed(kTargetPlatformOption)) { - // Override deviceOperatingSystem to prevent extra utilities from being used. - targetPlatform = deviceOperatingSystemFromString(argResults[kTargetPlatformOption] as String); - _originalDeviceOperatingSystem = deviceOperatingSystem; - deviceOperatingSystem = DeviceOperatingSystem.fake; - } + } static const String kApplicationBinaryPathOption = 'application-binary-path'; static const String kBuildOnlyFlag = 'build'; - static const String kTargetPlatformOption = 'target-platform'; static const String kTestOnlyFlag = 'test'; final ArgParser argParser = ArgParser() ..addOption(kApplicationBinaryPathOption) ..addFlag(kBuildOnlyFlag) - ..addOption(kTargetPlatformOption) ..addFlag(kTestOnlyFlag); /// Args passed from the test runner via "--task-arg". @@ -44,15 +37,6 @@ abstract class BuildTestTask { /// If true, skip [test]. bool buildOnly = false; - /// The [DeviceOperatingSystem] being targeted for this build. - /// - /// If passed, no connected device checks are run as the current connected device - /// will be set as [DeviceOperatingSystem.fake]. - DeviceOperatingSystem targetPlatform; - - /// Original [deviceOperatingSystem] if [targetPlatform] is given. - DeviceOperatingSystem _originalDeviceOperatingSystem; - /// If true, skip [build]. bool testOnly = false; @@ -75,7 +59,7 @@ abstract class BuildTestTask { await flutter('clean'); } section('BUILDING APPLICATION'); - await flutter('build', options: getBuildArgs()); + await flutter('build', options: getBuildArgs(deviceOperatingSystem)); }); } @@ -84,25 +68,21 @@ abstract class BuildTestTask { /// /// This assumes that [applicationBinaryPath] exists. Future test() async { - // Ensure deviceOperatingSystem is the one set in bin/task. - if (deviceOperatingSystem == DeviceOperatingSystem.fake) { - deviceOperatingSystem = _originalDeviceOperatingSystem; - } final Device device = await devices.workingDevice; await device.unlock(); await inDirectory(workingDirectory, () async { section('DRIVE START'); - await flutter('drive', options: getTestArgs(device.deviceId)); + await flutter('drive', options: getTestArgs(deviceOperatingSystem, device.deviceId)); }); return parseTaskResult(); } /// Args passed to flutter build to build the application under test. - List getBuildArgs() => throw UnimplementedError('getBuildArgs is not implemented'); + List getBuildArgs(DeviceOperatingSystem deviceOperatingSystem) => throw UnimplementedError('getBuildArgs is not implemented'); /// Args passed to flutter drive to test the built application. - List getTestArgs(String deviceId) => throw UnimplementedError('getTestArgs is not implemented'); + List getTestArgs(DeviceOperatingSystem deviceOperatingSystem, String deviceId) => throw UnimplementedError('getTestArgs is not implemented'); /// Logic to construct [TaskResult] from this test's results. Future parseTaskResult() => throw UnimplementedError('parseTaskResult is not implemented'); @@ -126,7 +106,7 @@ abstract class BuildTestTask { } if (!testOnly) { - await build(); + build(); } if (buildOnly) { diff --git a/dev/devicelab/lib/tasks/gallery.dart b/dev/devicelab/lib/tasks/gallery.dart index dfe9391402b..a6994fb9ff8 100644 --- a/dev/devicelab/lib/tasks/gallery.dart +++ b/dev/devicelab/lib/tasks/gallery.dart @@ -11,17 +11,13 @@ import '../framework/adb.dart'; import '../framework/framework.dart'; import '../framework/task_result.dart'; import '../framework/utils.dart'; -import 'build_test_task.dart'; -final Directory galleryDirectory = dir('${flutterDirectory.path}/dev/integration_tests/flutter_gallery'); - -TaskFunction createGalleryTransitionTest(List args, {bool semanticsEnabled = false}) { - return GalleryTransitionTest(args, semanticsEnabled: semanticsEnabled, workingDirectory: galleryDirectory,); +TaskFunction createGalleryTransitionTest({bool semanticsEnabled = false}) { + return GalleryTransitionTest(semanticsEnabled: semanticsEnabled); } -TaskFunction createGalleryTransitionE2ETest(List args, {bool semanticsEnabled = false}) { +TaskFunction createGalleryTransitionE2ETest({bool semanticsEnabled = false}) { return GalleryTransitionTest( - args, testFile: semanticsEnabled ? 'transitions_perf_e2e_with_semantics' : 'transitions_perf_e2e', @@ -30,23 +26,21 @@ TaskFunction createGalleryTransitionE2ETest(List args, {bool semanticsEn transitionDurationFile: null, timelineTraceFile: null, driverFile: 'transitions_perf_e2e_test', - workingDirectory: galleryDirectory, ); } -TaskFunction createGalleryTransitionHybridTest(List args, {bool semanticsEnabled = false}) { +TaskFunction createGalleryTransitionHybridTest({bool semanticsEnabled = false}) { return GalleryTransitionTest( - args, semanticsEnabled: semanticsEnabled, driverFile: semanticsEnabled ? 'transitions_perf_hybrid_with_semantics_test' : 'transitions_perf_hybrid_test', - workingDirectory: galleryDirectory, ); } -class GalleryTransitionTest extends BuildTestTask { - GalleryTransitionTest(List args, { +class GalleryTransitionTest { + + GalleryTransitionTest({ this.semanticsEnabled = false, this.testFile = 'transitions_perf', this.needFullTimeline = true, @@ -54,8 +48,7 @@ class GalleryTransitionTest extends BuildTestTask { this.timelineTraceFile = 'transitions.timeline', this.transitionDurationFile = 'transition_durations.timeline', this.driverFile, - Directory workingDirectory, - }) : super(args, workingDirectory: workingDirectory); + }); final bool semanticsEnabled; final bool needFullTimeline; @@ -65,58 +58,59 @@ class GalleryTransitionTest extends BuildTestTask { final String transitionDurationFile; final String driverFile; - @override - List getBuildArgs() { - switch (targetPlatform) { - case DeviceOperatingSystem.android: - return [ - 'apk', - '--no-android-gradle-daemon', - '--profile', - '-t', - 'test_driver/$testFile.dart', - '--target-platform', - 'android-arm,android-arm64', - ]; - case DeviceOperatingSystem.ios: - return [ - 'ios', - // Skip codesign on presubmit checks - if (targetPlatform != null) - '--no-codesign', + Future call() async { + final Device device = await devices.workingDevice; + await device.unlock(); + final String deviceId = device.deviceId; + final Directory galleryDirectory = dir('${flutterDirectory.path}/dev/integration_tests/flutter_gallery'); + await inDirectory(galleryDirectory, () async { + String applicationBinaryPath; + if (deviceOperatingSystem == DeviceOperatingSystem.android) { + section('BUILDING APPLICATION'); + await flutter( + 'build', + options: [ + 'apk', + '--no-android-gradle-daemon', '--profile', '-t', 'test_driver/$testFile.dart', - ]; - default: - throw Exception('$deviceOperatingSystem has no build configuration'); + '--target-platform', + 'android-arm,android-arm64', + ], + ); + applicationBinaryPath = 'build/app/outputs/flutter-apk/app-profile.apk'; } - } - @override - List getTestArgs(String deviceId) { - final String testDriver = driverFile ?? (semanticsEnabled - ? '${testFile}_with_semantics_test' - : '${testFile}_test'); - return [ + final String testDriver = driverFile ?? (semanticsEnabled + ? '${testFile}_with_semantics_test' + : '${testFile}_test'); + section('DRIVE START'); + await flutter('drive', options: [ '--profile', if (needFullTimeline) '--trace-startup', - '--use-application-binary="${getApplicationBinaryPath()}"', - '--driver', 'test_driver/$testDriver.dart', - '-d', deviceId, - ]; - } + if (applicationBinaryPath != null) + '--use-application-binary=$applicationBinaryPath' + else + ...[ + '-t', + 'test_driver/$testFile.dart', + ], + '--driver', + 'test_driver/$testDriver.dart', + '-d', + deviceId, + ]); + }); - @override - Future parseTaskResult() async { final Map summary = json.decode( - file('${workingDirectory.path}/build/$timelineSummaryFile.json').readAsStringSync(), + file('${galleryDirectory.path}/build/$timelineSummaryFile.json').readAsStringSync(), ) as Map; if (transitionDurationFile != null) { final Map original = json.decode( - file('${workingDirectory.path}/build/$transitionDurationFile.json').readAsStringSync(), + file('${galleryDirectory.path}/build/$transitionDurationFile.json').readAsStringSync(), ) as Map; final Map> transitions = >{}; for (final String key in original.keys) { @@ -129,9 +123,9 @@ class GalleryTransitionTest extends BuildTestTask { return TaskResult.success(summary, detailFiles: [ if (transitionDurationFile != null) - '${workingDirectory.path}/build/$transitionDurationFile.json', + '${galleryDirectory.path}/build/$transitionDurationFile.json', if (timelineTraceFile != null) - '${workingDirectory.path}/build/$timelineTraceFile.json' + '${galleryDirectory.path}/build/$timelineTraceFile.json' ], benchmarkScoreKeys: [ if (transitionDurationFile != null) @@ -147,22 +141,6 @@ class GalleryTransitionTest extends BuildTestTask { ], ); } - - @override - String getApplicationBinaryPath() { - if (applicationBinaryPath != null) { - return applicationBinaryPath; - } - - switch (targetPlatform) { - case DeviceOperatingSystem.android: - return 'build/app/outputs/flutter-apk/app-profile.apk'; - case DeviceOperatingSystem.ios: - return 'build/ios/iphoneos/Flutter Gallery.app'; - default: - throw UnimplementedError('getApplicationBinaryPath does not support $deviceOperatingSystem'); - } - } } int _countMissedTransitions(Map> transitions) {