From 99f5eebc6ba898f98bc51fdb8d0a8af8bd0bcb58 Mon Sep 17 00:00:00 2001 From: "Ming Lyu (CareF)" Date: Fri, 26 Jun 2020 09:29:46 -0400 Subject: [PATCH] Add --device-id option for devicelab/bin/run.dart (#59276) * Implement device selection for devicelab/run.dart * Add test to --device-id option for devicelab/run * Update dev/devicelab/bin/run.dart by jonahwilliam * Rename deviceOperatingSystem enum mock -> fake Co-authored-by: Jonah Williams --- dev/devicelab/bin/run.dart | 16 ++ .../bin/tasks/smoke_test_device.dart | 27 +++ dev/devicelab/lib/framework/adb.dart | 192 +++++++++++++++++- dev/devicelab/lib/framework/runner.dart | 23 ++- dev/devicelab/lib/tasks/perf_tests.dart | 4 + dev/devicelab/test/run_test.dart | 20 +- 6 files changed, 271 insertions(+), 11 deletions(-) create mode 100644 dev/devicelab/bin/tasks/smoke_test_device.dart diff --git a/dev/devicelab/bin/run.dart b/dev/devicelab/bin/run.dart index 5923b14571d..a67cc2ea1ea 100644 --- a/dev/devicelab/bin/run.dart +++ b/dev/devicelab/bin/run.dart @@ -32,6 +32,9 @@ String localEngineSrcPath; /// Whether to exit on first test failure. bool exitOnFirstTestFailure; +/// The device-id to run test on. +String deviceId; + /// Runs tasks. /// /// The tasks are chosen depending on the command-line options @@ -75,6 +78,7 @@ Future main(List rawArgs) async { localEngine = args['local-engine'] as String; localEngineSrcPath = args['local-engine-src-path'] as String; exitOnFirstTestFailure = args['exit'] as bool; + deviceId = args['device-id'] as String; if (args.wasParsed('ab')) { await _runABTest(); @@ -91,6 +95,7 @@ Future _runTasks() async { silent: silent, localEngine: localEngine, localEngineSrcPath: localEngineSrcPath, + deviceId: deviceId, ); print('Task result:'); @@ -133,6 +138,7 @@ Future _runABTest() async { final Map defaultEngineResult = await runTask( taskName, silent: silent, + deviceId: deviceId, ); print('Default engine result:'); @@ -151,6 +157,7 @@ Future _runABTest() async { silent: silent, localEngine: localEngine, localEngineSrcPath: localEngineSrcPath, + deviceId: deviceId, ); print('Task localEngineResult:'); @@ -253,6 +260,15 @@ final ArgParser _argParser = ArgParser() } }, ) + ..addOption( + 'device-id', + abbr: 'd', + help: 'Target device id (prefixes are allowed, names are not supported).\n' + 'The option will be ignored if the test target does not run on a\n' + 'mobile device. This still respects the device operating system\n' + 'settings in the test case, and will results in error if no device\n' + 'with given ID/ID prefix is found.', + ) ..addOption( 'ab', help: 'Runs an A/B test comparing the default engine with the local\n' diff --git a/dev/devicelab/bin/tasks/smoke_test_device.dart b/dev/devicelab/bin/tasks/smoke_test_device.dart new file mode 100644 index 00000000000..d1a8c939aaa --- /dev/null +++ b/dev/devicelab/bin/tasks/smoke_test_device.dart @@ -0,0 +1,27 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/framework/adb.dart'; + +/// Smoke test of a successful task. +Future main() async { + deviceOperatingSystem = DeviceOperatingSystem.fake; + await task(() async { + final Device device = await devices.workingDevice; + if (device.deviceId == 'FAKE_SUCCESS') + return TaskResult.success({ + 'metric1': 42, + 'metric2': 123, + 'not_a_metric': 'something', + }, benchmarkScoreKeys: [ + 'metric1', + 'metric2', + ]); + else + return TaskResult.failure('Failed'); + }); +} diff --git a/dev/devicelab/lib/framework/adb.dart b/dev/devicelab/lib/framework/adb.dart index 55df99aa393..a9f908676be 100644 --- a/dev/devicelab/lib/framework/adb.dart +++ b/dev/devicelab/lib/framework/adb.dart @@ -12,6 +12,8 @@ import 'package:path/path.dart' as path; import 'utils.dart'; +const String DeviceIdEnvName = 'FLUTTER_DEVICELAB_DEVICEID'; + class DeviceException implements Exception { const DeviceException(this.message); @@ -31,11 +33,26 @@ String getArtifactPath() { ); } +/// Return the item is in idList if find a match, otherwise return null +String _findMatchId(List idList, String idPattern) { + String candidate; + idPattern = idPattern.toLowerCase(); + for(final String id in idList) { + if (id.toLowerCase() == idPattern) { + return id; + } + if (id.toLowerCase().startsWith(idPattern)) { + candidate ??= id; + } + } + return candidate; +} + /// The root of the API for controlling devices. DeviceDiscovery get devices => DeviceDiscovery(); /// Device operating system the test is configured to test. -enum DeviceOperatingSystem { android, ios, fuchsia } +enum DeviceOperatingSystem { android, ios, fuchsia, fake } /// Device OS to test on. DeviceOperatingSystem deviceOperatingSystem = DeviceOperatingSystem.android; @@ -50,8 +67,12 @@ abstract class DeviceDiscovery { return IosDeviceDiscovery(); case DeviceOperatingSystem.fuchsia: return FuchsiaDeviceDiscovery(); + case DeviceOperatingSystem.fake: + print('Looking for fake devices!' + 'You should not see this in release builds.'); + return FakeDeviceDiscovery(); default: - throw const DeviceException('Unsupported device operating system: {config.deviceOperatingSystem}'); + throw DeviceException('Unsupported device operating system: $deviceOperatingSystem'); } } @@ -62,6 +83,9 @@ abstract class DeviceDiscovery { /// returned. For such behavior see [workingDevice]. Future chooseWorkingDevice(); + /// Select the device with ID strati with deviceId, return the device. + Future chooseWorkingDeviceById(String deviceId); + /// A device to work with. /// /// Returns the same device when called repeatedly (unlike @@ -147,6 +171,11 @@ class AndroidDeviceDiscovery implements DeviceDiscovery { @override Future get workingDevice async { if (_workingDevice == null) { + if (Platform.environment.containsKey(DeviceIdEnvName)) { + final String deviceId = Platform.environment[DeviceIdEnvName]; + await chooseWorkingDeviceById(deviceId); + return _workingDevice; + } await chooseWorkingDevice(); } @@ -169,6 +198,20 @@ class AndroidDeviceDiscovery implements DeviceDiscovery { print('Device chosen: $_workingDevice'); } + @override + Future chooseWorkingDeviceById(String deviceId) async { + final String matchedId = _findMatchId(await discoverDevices(), deviceId); + if (matchedId != null) { + _workingDevice = AndroidDevice(deviceId: matchedId); + print('Choose device by ID: $matchedId'); + return; + } + throw DeviceException( + 'Device with ID $deviceId is not found for operating system: ' + '$deviceOperatingSystem' + ); + } + @override Future> discoverDevices() async { final List output = (await eval(adbPath, ['devices', '-l'], canFail: false)) @@ -250,6 +293,11 @@ class FuchsiaDeviceDiscovery implements DeviceDiscovery { @override Future get workingDevice async { if (_workingDevice == null) { + if (Platform.environment.containsKey(DeviceIdEnvName)) { + final String deviceId = Platform.environment[DeviceIdEnvName]; + await chooseWorkingDeviceById(deviceId); + return _workingDevice; + } await chooseWorkingDevice(); } return _workingDevice; @@ -269,6 +317,20 @@ class FuchsiaDeviceDiscovery implements DeviceDiscovery { print('Device chosen: $_workingDevice'); } + @override + Future chooseWorkingDeviceById(String deviceId) async { + final String matchedId = _findMatchId(await discoverDevices(), deviceId); + if (deviceId != null) { + _workingDevice = FuchsiaDevice(deviceId: matchedId); + print('Choose device by ID: $matchedId'); + return; + } + throw DeviceException( + 'Device with ID $deviceId is not found for operating system: ' + '$deviceOperatingSystem' + ); + } + @override Future> discoverDevices() async { final List output = (await eval(_devFinder, ['list', '-full'])) @@ -529,6 +591,11 @@ class IosDeviceDiscovery implements DeviceDiscovery { @override Future get workingDevice async { if (_workingDevice == null) { + if (Platform.environment.containsKey(DeviceIdEnvName)) { + final String deviceId = Platform.environment[DeviceIdEnvName]; + await chooseWorkingDeviceById(deviceId); + return _workingDevice; + } await chooseWorkingDevice(); } @@ -551,6 +618,20 @@ class IosDeviceDiscovery implements DeviceDiscovery { print('Device chosen: $_workingDevice'); } + @override + Future chooseWorkingDeviceById(String deviceId) async { + final String matchedId = _findMatchId(await discoverDevices(), deviceId); + if (matchedId != null) { + _workingDevice = IosDevice(deviceId: matchedId); + print('Choose device by ID: $matchedId'); + return; + } + throw DeviceException( + 'Device with ID $deviceId is not found for operating system: ' + '$deviceOperatingSystem' + ); + } + @override Future> discoverDevices() async { final List results = json.decode(await eval( @@ -723,3 +804,110 @@ String get adbPath { return path.absolute(adbPath); } + +class FakeDevice extends Device { + const FakeDevice({ @required this.deviceId }); + + @override + final String deviceId; + + @override + Future isAwake() async => true; + + @override + Future isAsleep() async => false; + + @override + Future wakeUp() async {} + + @override + Future sendToSleep() async {} + + @override + Future togglePower() async {} + + @override + Future unlock() async {} + + @override + Future tap(int x, int y) async { + throw UnimplementedError(); + } + + @override + Future> getMemoryStats(String packageName) async { + throw UnimplementedError(); + } + + @override + Stream get logcat { + throw UnimplementedError(); + } + + @override + Future stop(String packageName) async {} +} + +class FakeDeviceDiscovery implements DeviceDiscovery { + factory FakeDeviceDiscovery() { + return _instance ??= FakeDeviceDiscovery._(); + } + + FakeDeviceDiscovery._(); + + static FakeDeviceDiscovery _instance; + + FakeDevice _workingDevice; + + @override + Future get workingDevice async { + if (_workingDevice == null) { + if (Platform.environment.containsKey(DeviceIdEnvName)) { + final String deviceId = Platform.environment[DeviceIdEnvName]; + await chooseWorkingDeviceById(deviceId); + return _workingDevice; + } + await chooseWorkingDevice(); + } + + return _workingDevice; + } + + /// The Fake is only available for by ID device discovery. + @override + Future chooseWorkingDevice() async { + throw const DeviceException('No fake devices detected'); + } + + @override + Future chooseWorkingDeviceById(String deviceId) async { + final String matchedId = _findMatchId(await discoverDevices(), deviceId); + if (matchedId != null) { + _workingDevice = FakeDevice(deviceId: matchedId); + print('Choose device by ID: $matchedId'); + return; + } + throw DeviceException( + 'Device with ID $deviceId is not found for operating system: ' + '$deviceOperatingSystem' + ); + } + + @override + Future> discoverDevices() async { + return ['FAKE_SUCCESS', 'THIS_IS_A_FAKE']; + } + + @override + Future> checkDevices() async { + final Map results = {}; + for (final String deviceId in await discoverDevices()) { + results['fake-device-$deviceId'] = HealthCheckResult.success(); + } + return results; + } + + @override + Future performPreflightTasks() async { + } +} diff --git a/dev/devicelab/lib/framework/runner.dart b/dev/devicelab/lib/framework/runner.dart index 8e511eefde2..09e3b0eb811 100644 --- a/dev/devicelab/lib/framework/runner.dart +++ b/dev/devicelab/lib/framework/runner.dart @@ -10,6 +10,7 @@ import 'package:path/path.dart' as path; import 'package:vm_service_client/vm_service_client.dart'; import 'package:flutter_devicelab/framework/utils.dart'; +import 'package:flutter_devicelab/framework/adb.dart' show DeviceIdEnvName; /// Runs a task in a separate Dart VM and collects the result using the VM /// service protocol. @@ -24,19 +25,27 @@ Future> runTask( bool silent = false, String localEngine, String localEngineSrcPath, + String deviceId, }) async { final String taskExecutable = 'bin/tasks/$taskName.dart'; if (!file(taskExecutable).existsSync()) throw 'Executable Dart file not found: $taskExecutable'; - final Process runner = await startProcess(dartBin, [ - '--enable-vm-service=0', // zero causes the system to choose a free port - '--no-pause-isolates-on-exit', - if (localEngine != null) '-DlocalEngine=$localEngine', - if (localEngineSrcPath != null) '-DlocalEngineSrcPath=$localEngineSrcPath', - taskExecutable, - ]); + final Process runner = await startProcess( + dartBin, + [ + '--enable-vm-service=0', // zero causes the system to choose a free port + '--no-pause-isolates-on-exit', + if (localEngine != null) '-DlocalEngine=$localEngine', + if (localEngineSrcPath != null) '-DlocalEngineSrcPath=$localEngineSrcPath', + taskExecutable, + ], + environment: { + if (deviceId != null) + DeviceIdEnvName: deviceId, + }, + ); bool runnerFinished = false; diff --git a/dev/devicelab/lib/tasks/perf_tests.dart b/dev/devicelab/lib/tasks/perf_tests.dart index bb22c710b7a..98b007a6944 100644 --- a/dev/devicelab/lib/tasks/perf_tests.dart +++ b/dev/devicelab/lib/tasks/perf_tests.dart @@ -657,6 +657,8 @@ class CompileTest { break; case DeviceOperatingSystem.fuchsia: throw Exception('Unsupported option for Fuchsia devices'); + case DeviceOperatingSystem.fake: + throw Exception('Unsupported option for fake devices'); } metrics.addAll({ @@ -681,6 +683,8 @@ class CompileTest { break; case DeviceOperatingSystem.fuchsia: throw Exception('Unsupported option for Fuchsia devices'); + case DeviceOperatingSystem.fake: + throw Exception('Unsupported option for fake devices'); } watch.start(); await flutter('build', options: options); diff --git a/dev/devicelab/test/run_test.dart b/dev/devicelab/test/run_test.dart index 1b0a52204c7..815a94053d1 100644 --- a/dev/devicelab/test/run_test.dart +++ b/dev/devicelab/test/run_test.dart @@ -28,8 +28,13 @@ void main() { } Future expectScriptResult( - List testNames, int expectedExitCode) async { - final ProcessResult result = await runScript(testNames); + List testNames, + int expectedExitCode, + {String deviceId} + ) async { + final ProcessResult result = await runScript(testNames, [ + if (deviceId != null) ...['-d', deviceId], + ]); expect(result.exitCode, expectedExitCode, reason: '[ stderr from test process ]\n\n${result.stderr}\n\n[ end of stderr ]' @@ -71,6 +76,17 @@ void main() { ); }); + test('exits with code 0 when provided a valid device ID', () async { + await expectScriptResult(['smoke_test_device'], 0, + deviceId: 'FAKE'); + }); + + test('exits with code 1 when provided a bad device ID', () async { + await expectScriptResult(['smoke_test_device'], 1, + deviceId: 'THIS_IS_NOT_VALID'); + }); + + test('runs A/B test', () async { final ProcessResult result = await runScript( ['smoke_test_success'],