Add --device-id option for devicelab/bin/run.dart (#59276)

* Implement device selection for devicelab/run.dart

* Add test to --device-id option for devicelab/run

* Update dev/devicelab/bin/run.dart by jonahwilliam

* Rename deviceOperatingSystem enum mock -> fake

Co-authored-by: Jonah Williams <jonahwilliams@google.com>

dev/devicelab/bin/run.dart

@@ -32,6 +32,9 @@ String localEngineSrcPath;
 /// Whether to exit on first test failure.
 bool exitOnFirstTestFailure;
 
+/// The device-id to run test on.
+String deviceId;
+
 /// Runs tasks.
 ///
 /// The tasks are chosen depending on the command-line options
@@ -75,6 +78,7 @@ Future<void> main(List<String> rawArgs) async {
   localEngine = args['local-engine'] as String;
   localEngineSrcPath = args['local-engine-src-path'] as String;
   exitOnFirstTestFailure = args['exit'] as bool;
+  deviceId = args['device-id'] as String;
 
   if (args.wasParsed('ab')) {
     await _runABTest();
@@ -91,6 +95,7 @@ Future<void> _runTasks() async {
       silent: silent,
       localEngine: localEngine,
       localEngineSrcPath: localEngineSrcPath,
+      deviceId: deviceId,
     );
 
     print('Task result:');
@@ -133,6 +138,7 @@ Future<void> _runABTest() async {
     final Map<String, dynamic> defaultEngineResult = await runTask(
       taskName,
       silent: silent,
+      deviceId: deviceId,
     );
 
     print('Default engine result:');
@@ -151,6 +157,7 @@ Future<void> _runABTest() async {
       silent: silent,
       localEngine: localEngine,
       localEngineSrcPath: localEngineSrcPath,
+      deviceId: deviceId,
     );
 
     print('Task localEngineResult:');
@@ -253,6 +260,15 @@ final ArgParser _argParser = ArgParser()
       }
     },
   )
+  ..addOption(
+    'device-id',
+    abbr: 'd',
+    help: 'Target device id (prefixes are allowed, names are not supported).\n'
+          'The option will be ignored if the test target does not run on a\n'
+          'mobile device. This still respects the device operating system\n'
+          'settings in the test case, and will results in error if no device\n'
+          'with given ID/ID prefix is found.',
+  )
   ..addOption(
     'ab',
     help: 'Runs an A/B test comparing the default engine with the local\n'

dev/devicelab/bin/tasks/smoke_test_device.dart

@@ -0,0 +1,27 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/framework/adb.dart';
+
+/// Smoke test of a successful task.
+Future<void> main() async {
+  deviceOperatingSystem = DeviceOperatingSystem.fake;
+  await task(() async {
+    final Device device = await devices.workingDevice;
+    if (device.deviceId == 'FAKE_SUCCESS')
+      return TaskResult.success(<String, dynamic>{
+        'metric1': 42,
+        'metric2': 123,
+        'not_a_metric': 'something',
+      }, benchmarkScoreKeys: <String>[
+        'metric1',
+        'metric2',
+      ]);
+    else
+      return TaskResult.failure('Failed');
+  });
+}

dev/devicelab/lib/framework/adb.dart

@@ -12,6 +12,8 @@ import 'package:path/path.dart' as path;
 
 import 'utils.dart';
 
+const String DeviceIdEnvName = 'FLUTTER_DEVICELAB_DEVICEID';
+
 class DeviceException implements Exception {
   const DeviceException(this.message);
 
@@ -31,11 +33,26 @@ String getArtifactPath() {
     );
 }
 
+/// Return the item is in idList if find a match, otherwise return null
+String _findMatchId(List<String> idList, String idPattern) {
+  String candidate;
+  idPattern = idPattern.toLowerCase();
+  for(final String id in idList) {
+    if (id.toLowerCase() == idPattern) {
+      return id;
+    }
+    if (id.toLowerCase().startsWith(idPattern)) {
+      candidate ??= id;
+    }
+  }
+  return candidate;
+}
+
 /// The root of the API for controlling devices.
 DeviceDiscovery get devices => DeviceDiscovery();
 
 /// Device operating system the test is configured to test.
-enum DeviceOperatingSystem { android, ios, fuchsia }
+enum DeviceOperatingSystem { android, ios, fuchsia, fake }
 
 /// Device OS to test on.
 DeviceOperatingSystem deviceOperatingSystem = DeviceOperatingSystem.android;
@@ -50,8 +67,12 @@ abstract class DeviceDiscovery {
         return IosDeviceDiscovery();
       case DeviceOperatingSystem.fuchsia:
         return FuchsiaDeviceDiscovery();
+      case DeviceOperatingSystem.fake:
+        print('Looking for fake devices!'
+              'You should not see this in release builds.');
+        return FakeDeviceDiscovery();
       default:
-        throw const DeviceException('Unsupported device operating system: {config.deviceOperatingSystem}');
+        throw DeviceException('Unsupported device operating system: $deviceOperatingSystem');
     }
   }
 
@@ -62,6 +83,9 @@ abstract class DeviceDiscovery {
   /// returned. For such behavior see [workingDevice].
   Future<void> chooseWorkingDevice();
 
+  /// Select the device with ID strati with deviceId, return the device.
+  Future<void> chooseWorkingDeviceById(String deviceId);
+
   /// A device to work with.
   ///
   /// Returns the same device when called repeatedly (unlike
@@ -147,6 +171,11 @@ class AndroidDeviceDiscovery implements DeviceDiscovery {
   @override
   Future<AndroidDevice> get workingDevice async {
     if (_workingDevice == null) {
+      if (Platform.environment.containsKey(DeviceIdEnvName)) {
+        final String deviceId = Platform.environment[DeviceIdEnvName];
+        await chooseWorkingDeviceById(deviceId);
+        return _workingDevice;
+      }
       await chooseWorkingDevice();
     }
 
@@ -169,6 +198,20 @@ class AndroidDeviceDiscovery implements DeviceDiscovery {
     print('Device chosen: $_workingDevice');
   }
 
+  @override
+  Future<void> chooseWorkingDeviceById(String deviceId) async {
+    final String matchedId = _findMatchId(await discoverDevices(), deviceId);
+    if (matchedId != null) {
+      _workingDevice = AndroidDevice(deviceId: matchedId);
+      print('Choose device by ID: $matchedId');
+      return;
+    }
+    throw DeviceException(
+      'Device with ID $deviceId is not found for operating system: '
+      '$deviceOperatingSystem'
+      );
+  }
+
   @override
   Future<List<String>> discoverDevices() async {
     final List<String> output = (await eval(adbPath, <String>['devices', '-l'], canFail: false))
@@ -250,6 +293,11 @@ class FuchsiaDeviceDiscovery implements DeviceDiscovery {
   @override
   Future<FuchsiaDevice> get workingDevice async {
     if (_workingDevice == null) {
+      if (Platform.environment.containsKey(DeviceIdEnvName)) {
+        final String deviceId = Platform.environment[DeviceIdEnvName];
+        await chooseWorkingDeviceById(deviceId);
+        return _workingDevice;
+      }
       await chooseWorkingDevice();
     }
     return _workingDevice;
@@ -269,6 +317,20 @@ class FuchsiaDeviceDiscovery implements DeviceDiscovery {
     print('Device chosen: $_workingDevice');
   }
 
+  @override
+  Future<void> chooseWorkingDeviceById(String deviceId) async {
+    final String matchedId = _findMatchId(await discoverDevices(), deviceId);
+    if (deviceId != null) {
+      _workingDevice = FuchsiaDevice(deviceId: matchedId);
+      print('Choose device by ID: $matchedId');
+      return;
+    }
+    throw DeviceException(
+      'Device with ID $deviceId is not found for operating system: '
+      '$deviceOperatingSystem'
+      );
+  }
+
   @override
   Future<List<String>> discoverDevices() async {
     final List<String> output = (await eval(_devFinder, <String>['list', '-full']))
@@ -529,6 +591,11 @@ class IosDeviceDiscovery implements DeviceDiscovery {
   @override
   Future<IosDevice> get workingDevice async {
     if (_workingDevice == null) {
+      if (Platform.environment.containsKey(DeviceIdEnvName)) {
+        final String deviceId = Platform.environment[DeviceIdEnvName];
+        await chooseWorkingDeviceById(deviceId);
+        return _workingDevice;
+      }
       await chooseWorkingDevice();
     }
 
@@ -551,6 +618,20 @@ class IosDeviceDiscovery implements DeviceDiscovery {
     print('Device chosen: $_workingDevice');
   }
 
+  @override
+  Future<void> chooseWorkingDeviceById(String deviceId) async {
+    final String matchedId = _findMatchId(await discoverDevices(), deviceId);
+    if (matchedId != null) {
+      _workingDevice = IosDevice(deviceId: matchedId);
+      print('Choose device by ID: $matchedId');
+      return;
+    }
+    throw DeviceException(
+      'Device with ID $deviceId is not found for operating system: '
+      '$deviceOperatingSystem'
+      );
+  }
+
   @override
   Future<List<String>> discoverDevices() async {
     final List<dynamic> results = json.decode(await eval(
@@ -723,3 +804,110 @@ String get adbPath {
 
   return path.absolute(adbPath);
 }
+
+class FakeDevice extends Device {
+  const FakeDevice({ @required this.deviceId });
+
+  @override
+  final String deviceId;
+
+  @override
+  Future<bool> isAwake() async => true;
+
+  @override
+  Future<bool> isAsleep() async => false;
+
+  @override
+  Future<void> wakeUp() async {}
+
+  @override
+  Future<void> sendToSleep() async {}
+
+  @override
+  Future<void> togglePower() async {}
+
+  @override
+  Future<void> unlock() async {}
+
+  @override
+  Future<void> tap(int x, int y) async {
+    throw UnimplementedError();
+  }
+
+  @override
+  Future<Map<String, dynamic>> getMemoryStats(String packageName) async {
+    throw UnimplementedError();
+  }
+
+  @override
+  Stream<String> get logcat {
+    throw UnimplementedError();
+  }
+
+  @override
+  Future<void> stop(String packageName) async {}
+}
+
+class FakeDeviceDiscovery implements DeviceDiscovery {
+  factory FakeDeviceDiscovery() {
+    return _instance ??= FakeDeviceDiscovery._();
+  }
+
+  FakeDeviceDiscovery._();
+
+  static FakeDeviceDiscovery _instance;
+
+  FakeDevice _workingDevice;
+
+  @override
+  Future<FakeDevice> get workingDevice async {
+    if (_workingDevice == null) {
+      if (Platform.environment.containsKey(DeviceIdEnvName)) {
+        final String deviceId = Platform.environment[DeviceIdEnvName];
+        await chooseWorkingDeviceById(deviceId);
+        return _workingDevice;
+      }
+      await chooseWorkingDevice();
+    }
+
+    return _workingDevice;
+  }
+
+  /// The Fake is only available for by ID device discovery.
+  @override
+  Future<void> chooseWorkingDevice() async {
+    throw const DeviceException('No fake devices detected');
+  }
+
+  @override
+  Future<void> chooseWorkingDeviceById(String deviceId) async {
+    final String matchedId = _findMatchId(await discoverDevices(), deviceId);
+    if (matchedId != null) {
+      _workingDevice = FakeDevice(deviceId: matchedId);
+      print('Choose device by ID: $matchedId');
+      return;
+    }
+    throw DeviceException(
+      'Device with ID $deviceId is not found for operating system: '
+      '$deviceOperatingSystem'
+      );
+  }
+
+  @override
+  Future<List<String>> discoverDevices() async {
+    return <String>['FAKE_SUCCESS', 'THIS_IS_A_FAKE'];
+  }
+
+  @override
+  Future<Map<String, HealthCheckResult>> checkDevices() async {
+    final Map<String, HealthCheckResult> results = <String, HealthCheckResult>{};
+    for (final String deviceId in await discoverDevices()) {
+      results['fake-device-$deviceId'] = HealthCheckResult.success();
+    }
+    return results;
+  }
+
+  @override
+  Future<void> performPreflightTasks() async {
+  }
+}

dev/devicelab/lib/framework/runner.dart

@@ -10,6 +10,7 @@ import 'package:path/path.dart' as path;
 import 'package:vm_service_client/vm_service_client.dart';
 
 import 'package:flutter_devicelab/framework/utils.dart';
+import 'package:flutter_devicelab/framework/adb.dart' show DeviceIdEnvName;
 
 /// Runs a task in a separate Dart VM and collects the result using the VM
 /// service protocol.
@@ -24,19 +25,27 @@ Future<Map<String, dynamic>> runTask(
   bool silent = false,
   String localEngine,
   String localEngineSrcPath,
+  String deviceId,
 }) async {
   final String taskExecutable = 'bin/tasks/$taskName.dart';
 
   if (!file(taskExecutable).existsSync())
     throw 'Executable Dart file not found: $taskExecutable';
 
-  final Process runner = await startProcess(dartBin, <String>[
-    '--enable-vm-service=0', // zero causes the system to choose a free port
-    '--no-pause-isolates-on-exit',
-    if (localEngine != null) '-DlocalEngine=$localEngine',
-    if (localEngineSrcPath != null) '-DlocalEngineSrcPath=$localEngineSrcPath',
-    taskExecutable,
-  ]);
+  final Process runner = await startProcess(
+    dartBin,
+    <String>[
+      '--enable-vm-service=0', // zero causes the system to choose a free port
+      '--no-pause-isolates-on-exit',
+      if (localEngine != null) '-DlocalEngine=$localEngine',
+      if (localEngineSrcPath != null) '-DlocalEngineSrcPath=$localEngineSrcPath',
+      taskExecutable,
+    ],
+    environment: <String, String>{
+      if (deviceId != null)
+        DeviceIdEnvName: deviceId,
+    },
+  );
 
   bool runnerFinished = false;
 

dev/devicelab/lib/tasks/perf_tests.dart

@@ -657,6 +657,8 @@ class CompileTest {
         break;
       case DeviceOperatingSystem.fuchsia:
         throw Exception('Unsupported option for Fuchsia devices');
+      case DeviceOperatingSystem.fake:
+        throw Exception('Unsupported option for fake devices');
     }
 
     metrics.addAll(<String, dynamic>{
@@ -681,6 +683,8 @@ class CompileTest {
         break;
       case DeviceOperatingSystem.fuchsia:
         throw Exception('Unsupported option for Fuchsia devices');
+      case DeviceOperatingSystem.fake:
+        throw Exception('Unsupported option for fake devices');
     }
     watch.start();
     await flutter('build', options: options);

dev/devicelab/test/run_test.dart

@@ -28,8 +28,13 @@ void main() {
     }
 
     Future<void> expectScriptResult(
-        List<String> testNames, int expectedExitCode) async {
-      final ProcessResult result = await runScript(testNames);
+        List<String> testNames,
+        int expectedExitCode,
+        {String deviceId}
+      ) async {
+      final ProcessResult result = await runScript(testNames, <String>[
+        if (deviceId != null) ...<String>['-d', deviceId],
+      ]);
       expect(result.exitCode, expectedExitCode,
           reason:
               '[ stderr from test process ]\n\n${result.stderr}\n\n[ end of stderr ]'
@@ -71,6 +76,17 @@ void main() {
       );
     });
 
+    test('exits with code 0 when provided a valid device ID', () async {
+      await expectScriptResult(<String>['smoke_test_device'], 0,
+        deviceId: 'FAKE');
+    });
+
+    test('exits with code 1 when provided a bad device ID', () async {
+      await expectScriptResult(<String>['smoke_test_device'], 1,
+        deviceId: 'THIS_IS_NOT_VALID');
+    });
+
+
     test('runs A/B test', () async {
       final ProcessResult result = await runScript(
         <String>['smoke_test_success'],