Reduce noise on the memory benchmarks (#19630) (#20163)

- Check memory usage in release builds, not profile.
- Use multiple runs and average the results.

dev/benchmarks/complex_layout/test_memory/scroll_perf.dart

@@ -0,0 +1,72 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+
+import 'package:complex_layout/main.dart';
+import 'package:flutter/material.dart';
+import 'package:flutter/scheduler.dart';
+import 'package:flutter_test/flutter_test.dart';
+
+/// The speed, in pixels per second, that the drag gestures should end with.
+const double speed = 1500.0;
+
+/// The number of down drags and the number of up drags. The total number of
+/// gestures is twice this number.
+const int maxIterations = 4;
+
+/// The time that is allowed between gestures for the fling effect to settle.
+const Duration pauses = Duration(milliseconds: 500);
+
+Future<void> main() async {
+  final Completer<void> ready = new Completer<void>();
+  runApp(new GestureDetector(
+    onTap: () {
+      debugPrint('Received tap.');
+      ready.complete();
+    },
+    behavior: HitTestBehavior.opaque,
+    child: new IgnorePointer(
+      ignoring: true,
+      child: new ComplexLayoutApp(),
+    ),
+  ));
+  await SchedulerBinding.instance.endOfFrame;
+
+  /// Wait 50ms to allow the GPU thread to actually put up the frame. (The
+  /// endOfFrame future ends when we send the data to the engine, before the GPU
+  /// thread has had a chance to rasterize, etc.)
+  await new Future<Null>.delayed(const Duration(milliseconds: 50));
+  debugPrint('==== MEMORY BENCHMARK ==== READY ====');
+
+  await ready.future; // waits for tap sent by devicelab task
+  debugPrint('Continuing...');
+
+  // remove onTap handler, enable pointer events for app
+  runApp(new GestureDetector(
+    child: new IgnorePointer(
+      ignoring: false,
+      child: new ComplexLayoutApp(),
+    ),
+  ));
+  await SchedulerBinding.instance.endOfFrame;
+
+  final WidgetController controller = new LiveWidgetController(WidgetsBinding.instance);
+
+  // Scroll down
+  for (int iteration = 0; iteration < maxIterations; iteration += 1) {
+    debugPrint('Scroll down... $iteration/$maxIterations');
+    await controller.fling(find.byType(ListView), const Offset(0.0, -700.0), speed);
+    await new Future<Null>.delayed(pauses);
+  }
+
+  // Scroll up
+  for (int iteration = 0; iteration < maxIterations; iteration += 1) {
+    debugPrint('Scroll up... $iteration/$maxIterations');
+    await controller.fling(find.byType(ListView), const Offset(0.0, 300.0), speed);
+    await new Future<Null>.delayed(pauses);
+  }
+
+  debugPrint('==== MEMORY BENCHMARK ==== DONE ====');
+}

dev/devicelab/bin/tasks/complex_layout_scroll_perf__memory.dart

@@ -4,11 +4,16 @@
 
 import 'dart:async';
 
-import 'package:flutter_devicelab/tasks/perf_tests.dart';
 import 'package:flutter_devicelab/framework/adb.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/framework/utils.dart';
+import 'package:flutter_devicelab/tasks/perf_tests.dart';
 
 Future<Null> main() async {
   deviceOperatingSystem = DeviceOperatingSystem.android;
-  await task(createComplexLayoutScrollMemoryTest());
+  await task(new MemoryTest(
+    '${flutterDirectory.path}/dev/benchmarks/complex_layout',
+    'test_memory/scroll_perf.dart',
+    'com.yourcompany.complexLayout',
+  ).run);
 }

dev/devicelab/bin/tasks/flutter_gallery__back_button_memory.dart

@@ -2,11 +2,57 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+/// Measure application memory usage after pausing and resuming the app
+/// with the Android back button.
+
 import 'dart:async';
 
-import 'package:flutter_devicelab/tasks/perf_tests.dart';
+import 'package:flutter_devicelab/framework/adb.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/framework/utils.dart';
+import 'package:flutter_devicelab/tasks/perf_tests.dart';
+
+const String packageName = 'io.flutter.demo.gallery';
+const String activityName = 'io.flutter.demo.gallery.MainActivity';
+
+class BackButtonMemoryTest extends MemoryTest {
+  BackButtonMemoryTest() : super('${flutterDirectory.path}/examples/flutter_gallery', 'test_memory/back_button.dart', packageName);
+
+  @override
+  AndroidDevice get device => super.device;
+
+  /// Perform a series of back button suspend and resume cycles.
+  @override
+  Future<void> useMemory() async {
+    await launchApp();
+    await recordStart();
+    for (int iteration = 0; iteration < 10; iteration += 1) {
+      print('back/forward iteration $iteration');
+
+      // Push back button, wait for it to be seen by the Flutter app.
+      prepareForNextMessage('AppLifecycleState.paused');
+      await device.shellExec('input', <String>['keyevent', 'KEYCODE_BACK']);
+      await receivedNextMessage;
+
+      // Give Android time to settle (e.g. run GCs) after closing the app.
+      await new Future<Null>.delayed(const Duration(milliseconds: 100));
+
+      // Relaunch the app, wait for it to launch.
+      prepareForNextMessage('READY');
+      final String output = await device.shellEval('am', <String>['start', '-n', '$packageName/$activityName']);
+      print('adb shell am start: $output');
+      if (output.contains('Error'))
+        fail('unable to launch activity');
+      await receivedNextMessage;
+
+      // Wait for the Flutter app to settle (e.g. run GCs).
+      await new Future<Null>.delayed(const Duration(milliseconds: 100));
+    }
+    await recordEnd();
+  }
+}
 
 Future<Null> main() async {
-  await task(createGalleryBackButtonMemoryTest());
+  deviceOperatingSystem = DeviceOperatingSystem.android;
+  await task(new BackButtonMemoryTest().run);
 }

dev/devicelab/bin/tasks/flutter_gallery__memory_nav.dart

@@ -4,9 +4,14 @@
 
 import 'dart:async';
 
-import 'package:flutter_devicelab/tasks/perf_tests.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/framework/utils.dart';
+import 'package:flutter_devicelab/tasks/perf_tests.dart';
 
 Future<Null> main() async {
-  await task(createGalleryNavigationMemoryTest());
+  await task(new MemoryTest(
+    '${flutterDirectory.path}/examples/flutter_gallery',
+    'test_memory/memory_nav.dart',
+    'io.flutter.demo.gallery',
+  ).run);
 }

dev/devicelab/bin/tasks/hello_world__memory.dart

@@ -4,9 +4,36 @@
 
 import 'dart:async';
 
-import 'package:flutter_devicelab/tasks/perf_tests.dart';
 import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/framework/utils.dart';
+import 'package:flutter_devicelab/tasks/perf_tests.dart';
+
+class HelloWorldMemoryTest extends MemoryTest {
+  HelloWorldMemoryTest() : super(
+    '${flutterDirectory.path}/examples/hello_world',
+    'lib/main.dart',
+    'io.flutter.examples.hello_world',
+  );
+
+  /// Launch an app with no instrumentation and measure its memory usage after
+  /// 1.5s and 3.0s.
+  @override
+  Future<void> useMemory() async {
+    print('launching $project$test on device...');
+    await flutter('run', options: <String>[
+      '--verbose',
+      '--release',
+      '--no-resident',
+      '-d', device.deviceId,
+      test,
+    ]);
+    await new Future<Null>.delayed(const Duration(milliseconds: 1500));
+    await recordStart();
+    await new Future<Null>.delayed(const Duration(milliseconds: 3000));
+    await recordEnd();
+  }
+}
 
 Future<Null> main() async {
-  await task(createHelloWorldMemoryTest());
+  await task(new HelloWorldMemoryTest().run);
 }

dev/devicelab/lib/framework/adb.dart

@@ -83,9 +83,18 @@ abstract class Device {
   /// Assumes the device doesn't have a secure unlock pattern.
   Future<Null> unlock();
 
+  /// Emulate a tap on the touch screen.
+  Future<Null> tap(int x, int y);
+
   /// Read memory statistics for a process.
   Future<Map<String, dynamic>> getMemoryStats(String packageName);
 
+  /// Stream the system log from the device.
+  ///
+  /// Flutter applications' `print` statements end up in this log
+  /// with some prefix.
+  Stream<String> get logcat;
+
   /// Stop a process.
   Future<Null> stop(String packageName);
 }
@@ -237,6 +246,11 @@ class AndroidDevice implements Device {
     await shellExec('input', const <String>['keyevent', '82']);
   }
 
+  @override
+  Future<Null> tap(int x, int y) async {
+    await shellExec('input', <String>['tap', '$x', '$y']);
+  }
+
   /// Retrieves device's wakefulness state.
   ///
   /// See: https://android.googlesource.com/platform/frameworks/base/+/master/core/java/android/os/PowerManagerInternal.java
@@ -271,6 +285,63 @@ class AndroidDevice implements Device {
     };
   }
 
+  @override
+  Stream<String> get logcat {
+    final Completer<void> stdoutDone = new Completer<void>();
+    final Completer<void> stderrDone = new Completer<void>();
+    final Completer<void> processDone = new Completer<void>();
+    final Completer<void> abort = new Completer<void>();
+    bool aborted = false;
+    StreamController<String> stream;
+    stream = new StreamController<String>(
+      onListen: () async {
+        await adb(<String>['logcat', '--clear']);
+        final Process process = await startProcess(adbPath, <String>['-s', deviceId, 'logcat']);
+        process.stdout
+          .transform(utf8.decoder)
+          .transform(const LineSplitter())
+          .listen((String line) {
+            print('adb logcat: $line');
+            stream.sink.add(line);
+          }, onDone: () { stdoutDone.complete(); });
+        process.stderr
+          .transform(utf8.decoder)
+          .transform(const LineSplitter())
+          .listen((String line) {
+            print('adb logcat stderr: $line');
+          }, onDone: () { stderrDone.complete(); });
+        process.exitCode.then((int exitCode) {
+          print('adb logcat process terminated with exit code $exitCode');
+          if (!aborted) {
+            stream.addError(BuildFailedError('adb logcat failed with exit code $exitCode.'));
+            processDone.complete();
+          }
+        });
+        await Future.any<dynamic>(<Future<dynamic>>[
+          Future.wait<void>(<Future<void>>[
+            stdoutDone.future,
+            stderrDone.future,
+            processDone.future,
+          ]),
+          abort.future,
+        ]);
+        aborted = true;
+        print('terminating adb logcat');
+        process.kill();
+        print('closing logcat stream');
+        await stream.close();
+      },
+      onCancel: () {
+        if (!aborted) {
+          print('adb logcat aborted');
+          aborted = true;
+          abort.complete();
+        }
+      },
+    );
+    return stream.stream;
+  }
+
   @override
   Future<Null> stop(String packageName) async {
     return shellExec('am', <String>['force-stop', packageName]);
@@ -371,11 +442,21 @@ class IosDevice implements Device {
   @override
   Future<Null> unlock() async {}
 
+  @override
+  Future<Null> tap(int x, int y) async {
+    throw 'Not implemented';
+  }
+
   @override
   Future<Map<String, dynamic>> getMemoryStats(String packageName) async {
     throw 'Not implemented';
   }
 
+  @override
+  Stream<String> get logcat {
+    throw 'Not implemented';
+  }
+
   @override
   Future<Null> stop(String packageName) async {}
 }

dev/devicelab/lib/framework/framework.dart

@@ -81,12 +81,15 @@ class _TaskRunner {
   Future<TaskResult> run(Duration taskTimeout) async {
     try {
       _taskStarted = true;
+      print('Running task.');
       final TaskResult result = await _performTask().timeout(taskTimeout);
       _completer.complete(result);
       return result;
     } on TimeoutException catch (_) {
+      print('Task timed out after $taskTimeout.');
       return new TaskResult.failure('Task timed out after $taskTimeout');
     } finally {
+      print('Cleaning up after task...');
       await forceQuitRunningProcesses();
       _closeKeepAlivePort();
     }

dev/devicelab/lib/framework/utils.dart

@@ -244,7 +244,7 @@ Future<Null> forceQuitRunningProcesses() async {
 
   // Whatever's left, kill it.
   for (ProcessInfo p in _runningProcesses) {
-    print('Force quitting process:\n$p');
+    print('Force-quitting process:\n$p');
     if (!p.process.kill()) {
       print('Failed to force quit process');
     }
@@ -528,8 +528,6 @@ int parseServicePort(String line, {
   // e.g. "An Observatory debugger and profiler on ... is available at: http://127.0.0.1:8100/"
   final RegExp pattern = new RegExp('$prefix(\\S+:(\\d+)/\\S*)\$', multiLine: multiLine);
   final Match match = pattern.firstMatch(line);
-  print(pattern);
-  print(match);
   return match == null ? null : int.parse(match.group(2));
 }
 

dev/devicelab/lib/tasks/perf_tests.dart

@@ -7,7 +7,7 @@ import 'dart:convert' show json;
 import 'dart:io';
 
 import 'package:meta/meta.dart';
-import 'package:path/path.dart' as p;
+import 'package:path/path.dart' as path;
 
 import '../framework/adb.dart';
 import '../framework/framework.dart';
@@ -30,14 +30,6 @@ TaskFunction createTilesScrollPerfTest() {
   ).run;
 }
 
-TaskFunction createComplexLayoutScrollMemoryTest() {
-  return new MemoryTest(
-    '${flutterDirectory.path}/dev/benchmarks/complex_layout',
-    'com.yourcompany.complexLayout',
-    testTarget: 'test_driver/scroll_perf.dart',
-  ).run;
-}
-
 TaskFunction createFlutterGalleryStartupTest() {
   return new StartupTest(
     '${flutterDirectory.path}/examples/flutter_gallery',
@@ -62,29 +54,6 @@ TaskFunction createComplexLayoutCompileTest() {
   return new CompileTest('${flutterDirectory.path}/dev/benchmarks/complex_layout').run;
 }
 
-TaskFunction createHelloWorldMemoryTest() {
-  return new MemoryTest(
-    '${flutterDirectory.path}/examples/hello_world',
-    'io.flutter.examples.hello_world',
-  ).run;
-}
-
-TaskFunction createGalleryNavigationMemoryTest() {
-  return new MemoryTest(
-    '${flutterDirectory.path}/examples/flutter_gallery',
-    'io.flutter.demo.gallery',
-    testTarget: 'test_driver/memory_nav.dart',
-  ).run;
-}
-
-TaskFunction createGalleryBackButtonMemoryTest() {
-  return new AndroidBackButtonMemoryTest(
-    '${flutterDirectory.path}/examples/flutter_gallery',
-    'io.flutter.demo.gallery',
-    'io.flutter.demo.gallery.MainActivity',
-  ).run;
-}
-
 TaskFunction createFlutterViewStartupTest() {
   return new StartupTest(
       '${flutterDirectory.path}/examples/flutter_view',
@@ -366,15 +335,15 @@ class CompileTest {
 
   static Future<Map<String, dynamic>> getSizesFromIosApp(String appPath) async {
     // Thin the binary to only contain one architecture.
-    final String xcodeBackend = p.join(flutterDirectory.path, 'packages', 'flutter_tools', 'bin', 'xcode_backend.sh');
+    final String xcodeBackend = path.join(flutterDirectory.path, 'packages', 'flutter_tools', 'bin', 'xcode_backend.sh');
     await exec(xcodeBackend, <String>['thin'], environment: <String, String>{
       'ARCHS': 'arm64',
-      'WRAPPER_NAME': p.basename(appPath),
+      'WRAPPER_NAME': path.basename(appPath),
-      'TARGET_BUILD_DIR': p.dirname(appPath),
+      'TARGET_BUILD_DIR': path.dirname(appPath),
     });
 
-    final File appFramework = new File(p.join(appPath, 'Frameworks', 'App.framework', 'App'));
+    final File appFramework = new File(path.join(appPath, 'Frameworks', 'App.framework', 'App'));
-    final File flutterFramework = new File(p.join(appPath, 'Frameworks', 'Flutter.framework', 'Flutter'));
+    final File flutterFramework = new File(path.join(appPath, 'Frameworks', 'Flutter.framework', 'Flutter'));
 
     return <String, dynamic>{
       'app_framework_uncompressed_bytes': await appFramework.length(),
@@ -420,123 +389,167 @@ class CompileTest {
 
 /// Measure application memory usage.
 class MemoryTest {
-  const MemoryTest(this.testDirectory, this.packageName, { this.testTarget });
+  MemoryTest(this.project, this.test, this.package);
 
-  final String testDirectory;
+  final String project;
-  final String packageName;
+  final String test;
+  final String package;
 
-  /// Path to a flutter driver script that will run after starting the app.
+  /// Completes when the log line specified in the last call to
-  ///
+  /// [prepareForNextMessage] is seen by `adb logcat`.
-  /// If not specified, then the test will start the app, gather statistics, and then exit.
+  Future<void> get receivedNextMessage => _receivedNextMessage?.future;
-  final String testTarget;
+  Completer<void> _receivedNextMessage;
+  String _nextMessage;
+
+  /// Prepares the [receivedNextMessage] future such that it will complete
+  /// when `adb logcat` sees a log line with the given `message`.
+  void prepareForNextMessage(String message) {
+    _nextMessage = message;
+    _receivedNextMessage = new Completer<void>();
+  }
+
+  int get iterationCount => 15;
+
+  Device get device => _device;
+  Device _device;
 
   Future<TaskResult> run() {
-    return inDirectory(testDirectory, () async {
+    return inDirectory(project, () async {
-      final Device device = await devices.workingDevice;
+      // This test currently only works on Android, because device.logcat,
+      // device.getMemoryStats, etc, aren't implemented for iOS.
+
+      _device = await devices.workingDevice;
       await device.unlock();
-      final String deviceId = device.deviceId;
       await flutter('packages', options: <String>['get']);
 
       if (deviceOperatingSystem == DeviceOperatingSystem.ios)
-        await prepareProvisioningCertificates(testDirectory);
+        await prepareProvisioningCertificates(project);
 
-      final List<String> runOptions = <String>[
+      final StreamSubscription<String> adb = device.logcat.listen(
-        '-v',
+        (String data) {
-        '--profile',
+          if (data.contains('==== MEMORY BENCHMARK ==== $_nextMessage ===='))
-        '--trace-startup', // wait for the first frame to render
+            _receivedNextMessage.complete();
-        '-d',
+        },
-        deviceId,
+      );
-        '--observatory-port',
-        '0',
-      ];
-      if (testTarget != null)
-        runOptions.addAll(<String>['-t', testTarget]);
-      final String output = await evalFlutter('run', options: runOptions);
-      final int observatoryPort = parseServicePort(output, prefix: 'Successfully connected to service protocol: ', multiLine: true);
-      if (observatoryPort == null)
-        throw new Exception('Could not find observatory port in "flutter run" output.');
 
-      final Map<String, dynamic> startData = await device.getMemoryStats(packageName);
+      for (int iteration = 0; iteration < iterationCount; iteration += 1) {
-
+        print('running memory test iteration $iteration...');
-      final Map<String, dynamic> data = <String, dynamic>{
+        _startMemoryUsage = null;
-         'start_total_kb': startData['total_kb'],
+        await useMemory();
-      };
+        assert(_startMemoryUsage != null);
-
+        assert(_startMemory.length == iteration + 1);
-      if (testTarget != null) {
+        assert(_endMemory.length == iteration + 1);
-        await flutter('drive', options: <String>[
+        assert(_diffMemory.length == iteration + 1);
-          '-v',
+        print('terminating...');
-          '-t',
+        await device.stop(package);
-          testTarget,
+        await new Future<void>.delayed(const Duration(milliseconds: 10));
-          '-d',
-          deviceId,
-          '--use-existing-app=http://localhost:$observatoryPort',
-        ]);
-
-        final Map<String, dynamic> endData = await device.getMemoryStats(packageName);
-        data['end_total_kb'] = endData['total_kb'];
-        data['diff_total_kb'] = endData['total_kb'] - startData['total_kb'];
       }
 
-      await device.stop(packageName);
+      await adb.cancel();
 
-      return new TaskResult.success(data, benchmarkScoreKeys: data.keys.toList());
+      final ListStatistics startMemoryStatistics = new ListStatistics(_startMemory);
+      final ListStatistics endMemoryStatistics = new ListStatistics(_endMemory);
+      final ListStatistics diffMemoryStatistics = new ListStatistics(_diffMemory);
+
+      final Map<String, dynamic> memoryUsage = <String, dynamic>{};
+      memoryUsage.addAll(startMemoryStatistics.asMap('start'));
+      memoryUsage.addAll(endMemoryStatistics.asMap('end'));
+      memoryUsage.addAll(diffMemoryStatistics.asMap('diff'));
+
+      _device = null;
+      _startMemory.clear();
+      _endMemory.clear();
+      _diffMemory.clear();
+
+      return new TaskResult.success(memoryUsage, benchmarkScoreKeys: memoryUsage.keys.toList());
     });
   }
+
+  /// Starts the app specified by [test] on the [device].
+  ///
+  /// The [run] method will terminate it by its package name ([package]).
+  Future<void> launchApp() async {
+    prepareForNextMessage('READY');
+    print('launching $project$test on device...');
+    await flutter('run', options: <String>[
+      '--verbose',
+      '--release',
+      '--no-resident',
+      '-d', device.deviceId,
+      test,
+    ]);
+    print('awaiting "ready" message...');
+    await receivedNextMessage;
+  }
+
+  /// To change the behaviour of the test, override this.
+  ///
+  /// Make sure to call recordStart() and recordEnd() once each in that order.
+  ///
+  /// By default it just launches the app, records memory usage, taps the device,
+  /// awaits a DONE notification, and records memory usage again.
+  Future<void> useMemory() async {
+    await launchApp();
+    await recordStart();
+
+    prepareForNextMessage('DONE');
+    print('tapping device...');
+    await device.tap(100, 100);
+    print('awaiting "done" message...');
+    await receivedNextMessage;
+
+    await recordEnd();
+  }
+
+  final List<int> _startMemory = <int>[];
+  final List<int> _endMemory = <int>[];
+  final List<int> _diffMemory = <int>[];
+
+  Map<String, dynamic> _startMemoryUsage;
+
+  @protected
+  Future<void> recordStart() async {
+    assert(_startMemoryUsage == null);
+    print('snapshotting memory usage...');
+    _startMemoryUsage = await device.getMemoryStats(package);
+  }
+
+  @protected
+  Future<void> recordEnd() async {
+    assert(_startMemoryUsage != null);
+    print('snapshotting memory usage...');
+    final Map<String, dynamic> endMemoryUsage = await device.getMemoryStats(package);
+    _startMemory.add(_startMemoryUsage['total_kb']);
+    _endMemory.add(endMemoryUsage['total_kb']);
+    _diffMemory.add(endMemoryUsage['total_kb'] - _startMemoryUsage['total_kb']);
+  }
 }
 
-/// Measure application memory usage after pausing and resuming the app
+/// Holds simple statistics of an odd-lengthed list of integers.
-/// with the Android back button.
+class ListStatistics {
-class AndroidBackButtonMemoryTest {
+  factory ListStatistics(Iterable<int> data) {
-  const AndroidBackButtonMemoryTest(this.testDirectory, this.packageName, this.activityName);
+    assert(data.isNotEmpty);
+    assert(data.length % 2 == 1);
+    final List<int> sortedData = data.toList()..sort();
+    return new ListStatistics._(
+      sortedData.first,
+      sortedData.last,
+      sortedData[(sortedData.length - 1) ~/ 2],
+    );
+  }
 
-  final String testDirectory;
+  const ListStatistics._(this.min, this.max, this.median);
-  final String packageName;
-  final String activityName;
 
-  Future<TaskResult> run() {
+  final int min;
-    return inDirectory(testDirectory, () async {
+  final int max;
-      if (deviceOperatingSystem != DeviceOperatingSystem.android) {
+  final int median;
-        throw 'This test is only supported on Android';
-      }
 
-      final AndroidDevice device = await devices.workingDevice;
+  Map<String, int> asMap(String prefix) {
-      await device.unlock();
+    return <String, int>{
-      final String deviceId = device.deviceId;
+      '$prefix-min': min,
-      await flutter('packages', options: <String>['get']);
+      '$prefix-max': max,
-
+      '$prefix-median': median,
-      await flutter('run', options: <String>[
+    };
-        '-v',
-        '--profile',
-        '--trace-startup', // wait for the first frame to render
-        '-d',
-        deviceId,
-      ]);
-
-      final Map<String, dynamic> startData = await device.getMemoryStats(packageName);
-
-      final Map<String, dynamic> data = <String, dynamic>{
-         'start_total_kb': startData['total_kb'],
-      };
-
-      // Perform a series of back button suspend and resume cycles.
-      for (int i = 0; i < 10; i++) {
-        await device.shellExec('input', <String>['keyevent', 'KEYCODE_BACK']);
-        await new Future<Null>.delayed(const Duration(milliseconds: 1000));
-        final String output = await device.shellEval('am', <String>['start', '-n', '$packageName/$activityName']);
-        print(output);
-        if (output.contains('Error'))
-          return new TaskResult.failure('unable to launch activity');
-        await new Future<Null>.delayed(const Duration(milliseconds: 1000));
-      }
-
-      final Map<String, dynamic> endData = await device.getMemoryStats(packageName);
-      data['end_total_kb'] = endData['total_kb'];
-      data['diff_total_kb'] = endData['total_kb'] - startData['total_kb'];
-
-      await device.stop(packageName);
-
-      return new TaskResult.success(data, benchmarkScoreKeys: data.keys.toList());
-    });
   }
 }
 

dev/devicelab/test/adb_test.dart

@@ -93,6 +93,15 @@ void main() {
         ]);
       });
     });
+
+    group('adb', () {
+      test('tap', () async {
+        await device.tap(100, 200);
+        expectLog(<CommandArgs>[
+          cmd(command: 'input', arguments: <String>['tap', '100', '200']),
+        ]);
+      });
+    });
   });
 }
 

examples/flutter_gallery/lib/gallery/home.dart

@@ -363,7 +363,7 @@ class _GalleryHomeState extends State<GalleryHome> with SingleTickerProviderStat
                 ? const Text('Flutter gallery')
                 : new Text(_category.name),
             ),
-            frontHeading: widget.testMode ? null: new Container(height: 24.0),
+            frontHeading: widget.testMode ? null : new Container(height: 24.0),
             frontLayer: new AnimatedSwitcher(
               duration: _kFrontLayerSwitchDuration,
               switchOutCurve: switchOutCurve,

examples/flutter_gallery/test/live_smoketest.dart

@@ -44,7 +44,7 @@ Future<Null> main() async {
       fail('Unrecognized demo names in _kSkippedDemoTitles: $_kSkippedDemoTitles');
 
     runApp(const GalleryApp(testMode: true));
-    final _LiveWidgetController controller = new _LiveWidgetController();
+    final _LiveWidgetController controller = new _LiveWidgetController(WidgetsBinding.instance);
     for (GalleryDemoCategory category in kAllGalleryDemoCategories) {
       await controller.tap(find.text(category.name));
       for (GalleryDemo demo in kGalleryCategoryToDemos[category]) {
@@ -73,9 +73,8 @@ Future<Null> main() async {
   }
 }
 
-class _LiveWidgetController {
+class _LiveWidgetController extends LiveWidgetController {
-
+  _LiveWidgetController(WidgetsBinding binding) : super(binding);
-  final WidgetController _controller = new WidgetController(WidgetsBinding.instance);
 
   /// With [frameSync] enabled, Flutter Driver will wait to perform an action
   /// until there are no pending frames in the app under test.
@@ -107,8 +106,9 @@ class _LiveWidgetController {
     return finder;
   }
 
-  Future<Null> tap(Finder finder) async {
+  @override
-    await _controller.tap(await _waitForElement(finder));
+  Future<Null> tap(Finder finder, { int pointer }) async {
+    await tap(await _waitForElement(finder), pointer: pointer);
   }
 
   Future<Null> scrollIntoView(Finder finder, {double alignment}) async {

examples/flutter_gallery/test_driver/memory_nav.dart

@@ -1,15 +0,0 @@
-// Copyright 2016 The Chromium Authors. All rights reserved.
-// Use of this source code is governed by a BSD-style license that can be
-// found in the LICENSE file.
-
-import 'package:flutter_driver/driver_extension.dart';
-import 'package:flutter_gallery/gallery/app.dart' show GalleryApp;
-import 'package:flutter/material.dart';
-
-void main() {
-  enableFlutterDriverExtension();
-  // As in lib/main.dart: overriding https://github.com/flutter/flutter/issues/13736
-  // for better visual effect at the cost of performance.
-  MaterialPageRoute.debugEnableFadingRoutes = true; // ignore: deprecated_member_use
-  runApp(const GalleryApp(testMode: true));
-}

examples/flutter_gallery/test_driver/memory_nav_test.dart

@@ -1,32 +0,0 @@
-import 'package:flutter_driver/flutter_driver.dart';
-import 'package:test/test.dart';
-
-void main() {
-  group('flutter gallery transitions', () {
-    FlutterDriver driver;
-    setUpAll(() async {
-      driver = await FlutterDriver.connect();
-    });
-
-    tearDownAll(() async {
-      if (driver != null)
-        await driver.close();
-    });
-
-    test('navigation', () async {
-      await driver.tap(find.text('Material'));
-
-      final SerializableFinder demoList = find.byValueKey('GalleryDemoList');
-      final SerializableFinder demoItem = find.text('Text fields');
-      await driver.scrollUntilVisible(demoList, demoItem,
-        dyScroll: -300.0,
-        alignment: 0.5,
-        timeout: const Duration(minutes: 1),
-      );
-      for (int i = 0; i < 15; i++) {
-        await driver.tap(demoItem);
-        await driver.tap(find.byTooltip('Back'));
-      }
-    });
-  });
-}

examples/flutter_gallery/test_memory/back_button.dart

@@ -0,0 +1,37 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// See //dev/devicelab/bin/tasks/flutter_gallery__memory_nav.dart
+
+import 'dart:async';
+
+import 'package:flutter/material.dart';
+import 'package:flutter/scheduler.dart';
+import 'package:flutter_gallery/gallery/app.dart' show GalleryApp;
+import 'package:flutter_test/flutter_test.dart';
+
+Future<void> endOfAnimation() async {
+  do {
+    await SchedulerBinding.instance.endOfFrame;
+  } while (SchedulerBinding.instance.hasScheduledFrame);
+}
+
+int iteration = 0;
+
+class LifecycleObserver extends WidgetsBindingObserver {
+   @override
+   void didChangeAppLifecycleState(AppLifecycleState state) {
+     debugPrint('==== MEMORY BENCHMARK ==== $state ====');
+     debugPrint('This was lifecycle event number $iteration in this instance');
+   }
+}
+
+Future<void> main() async {
+  MaterialPageRoute.debugEnableFadingRoutes = true; // ignore: deprecated_member_use
+  runApp(const GalleryApp());
+  await endOfAnimation();
+  await new Future<Null>.delayed(const Duration(milliseconds: 50));
+  debugPrint('==== MEMORY BENCHMARK ==== READY ====');
+  WidgetsBinding.instance.addObserver(new LifecycleObserver());
+}

examples/flutter_gallery/test_memory/memory_nav.dart

@@ -0,0 +1,72 @@
+// Copyright 2016 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+// See //dev/devicelab/bin/tasks/flutter_gallery__memory_nav.dart
+
+import 'dart:async';
+
+import 'package:flutter/material.dart';
+import 'package:flutter/scheduler.dart';
+import 'package:flutter_gallery/gallery/app.dart' show GalleryApp;
+import 'package:flutter_test/flutter_test.dart';
+
+Future<void> endOfAnimation() async {
+  do {
+    await SchedulerBinding.instance.endOfFrame;
+  } while (SchedulerBinding.instance.hasScheduledFrame);
+}
+
+Future<void> main() async {
+  MaterialPageRoute.debugEnableFadingRoutes = true; // ignore: deprecated_member_use
+  final Completer<void> ready = new Completer<void>();
+  runApp(new GestureDetector(
+    onTap: () {
+      debugPrint('Received tap.');
+      ready.complete();
+    },
+    behavior: HitTestBehavior.opaque,
+    child: const IgnorePointer(
+      ignoring: true,
+      child: GalleryApp(testMode: true),
+    ),
+  ));
+  await SchedulerBinding.instance.endOfFrame;
+  await new Future<Null>.delayed(const Duration(milliseconds: 50));
+  debugPrint('==== MEMORY BENCHMARK ==== READY ====');
+
+  await ready.future;
+  debugPrint('Continuing...');
+
+  // remove onTap handler, enable pointer events for app
+  runApp(new GestureDetector(
+    child: const IgnorePointer(
+      ignoring: false,
+      child: GalleryApp(testMode: true),
+    ),
+  ));
+  await SchedulerBinding.instance.endOfFrame;
+
+  final WidgetController controller = new LiveWidgetController(WidgetsBinding.instance);
+
+  debugPrint('Navigating...');
+  await controller.tap(find.text('Material'));
+  await new Future<Null>.delayed(const Duration(milliseconds: 150));
+  final Finder demoList = find.byKey(const Key('GalleryDemoList'));
+  final Finder demoItem = find.text('Text fields');
+  do {
+    await controller.drag(demoList, const Offset(0.0, -300.0));
+    await new Future<Null>.delayed(const Duration(milliseconds: 20));
+  } while (!demoItem.precache());
+
+  for (int iteration = 0; iteration < 15; iteration += 1) {
+    debugPrint('Tapping... (iteration $iteration)');
+    await controller.tap(demoItem);
+    await endOfAnimation();
+    debugPrint('Backing out...');
+    await controller.tap(find.byTooltip('Back').last);
+    await endOfAnimation();
+  }
+
+  debugPrint('==== MEMORY BENCHMARK ==== DONE ====');
+}

packages/flutter_driver/lib/src/driver/driver.dart

@@ -447,19 +447,19 @@ class FlutterDriver {
   /// ensure the item's final position matches [alignment].
   ///
   /// The [scrollable] must locate the scrolling widget that contains [item].
-  /// Typically `find.byType('ListView') or `find.byType('CustomScrollView')`.
+  /// Typically `find.byType('ListView')` or `find.byType('CustomScrollView')`.
   ///
-  /// Atleast one of [dxScroll] and [dyScroll] must be non-zero.
+  /// At least one of [dxScroll] and [dyScroll] must be non-zero.
   ///
   /// If [item] is below the currently visible items, then specify a negative
   /// value for [dyScroll] that's a small enough increment to expose [item]
   /// without potentially scrolling it up and completely out of view. Similarly
-  /// if [item] is above, then specify a positve value for [dyScroll].
+  /// if [item] is above, then specify a positive value for [dyScroll].
   ///
-  /// If [item] is to the right of the the currently visible items, then
+  /// If [item] is to the right of the currently visible items, then
   /// specify a negative value for [dxScroll] that's a small enough increment to
   /// expose [item] without potentially scrolling it up and completely out of
-  /// view. Similarly if [item] is to the left, then specify a positve value
+  /// view. Similarly if [item] is to the left, then specify a positive value
   /// for [dyScroll].
   ///
   /// The [timeout] value should be long enough to accommodate as many scrolls
@@ -483,7 +483,7 @@ class FlutterDriver {
     // the chance to complete if the item is already onscreen; if not, scroll
     // repeatedly until we either find the item or time out.
     bool isVisible = false;
-    waitFor(item, timeout: timeout).then((Null _) { isVisible = true; });
+    waitFor(item, timeout: timeout).then((Null value) { isVisible = true; });
     await new Future<Null>.delayed(const Duration(milliseconds: 500));
     while (!isVisible) {
       await scroll(scrollable, dxScroll, dyScroll, const Duration(milliseconds: 100));

packages/flutter_driver/lib/src/extension/extension.dart

@@ -146,7 +146,7 @@ class FlutterDriverExtension {
 
   static final Logger _log = new Logger('FlutterDriverExtension');
 
-  final WidgetController _prober = new WidgetController(WidgetsBinding.instance);
+  final WidgetController _prober = new LiveWidgetController(WidgetsBinding.instance);
   final Map<String, CommandHandlerCallback> _commandHandlers = <String, CommandHandlerCallback>{};
   final Map<String, CommandDeserializerCallback> _commandDeserializers = <String, CommandDeserializerCallback>{};
   final Map<String, FinderConstructor> _finders = <String, FinderConstructor>{};

packages/flutter_test/lib/src/controller.dart

@@ -15,8 +15,12 @@ import 'test_pointer.dart';
 
 /// Class that programmatically interacts with widgets.
 ///
-/// For a variant of this class suited specifically for unit tests, see [WidgetTester].
+/// For a variant of this class suited specifically for unit tests, see
-class WidgetController {
+/// [WidgetTester]. For one suitable for live tests on a device, consider
+/// [LiveWidgetController].
+///
+/// Concrete subclasses must implement the [pump] method.
+abstract class WidgetController {
   /// Creates a widget controller that uses the given binding.
   WidgetController(this.binding);
 
@@ -392,10 +396,11 @@ class WidgetController {
   /// This is invoked by [flingFrom], for instance, so that the sequence of
   /// pointer events occurs over time.
   ///
-  /// The default implementation does nothing.
-  ///
   /// The [WidgetTester] subclass implements this by deferring to the [binding].
-  Future<Null> pump(Duration duration) => new Future<Null>.value(null);
+  ///
+  /// See also [SchedulerBinding.endOfFrame], which returns a future that could
+  /// be appropriate to return in the implementation of this method.
+  Future<Null> pump(Duration duration);
 
   /// Attempts to drag the given widget by the given offset, by
   /// starting a drag in the middle of the widget.
@@ -516,3 +521,20 @@ class WidgetController {
   /// the widget's render object has been laid out at least once.
   Rect getRect(Finder finder) => getTopLeft(finder) & getSize(finder);
 }
+
+/// Variant of [WidgetController] that can be used in tests running
+/// on a device.
+///
+/// This is used, for instance, by [FlutterDriver].
+class LiveWidgetController extends WidgetController {
+  /// Creates a widget controller that uses the given binding.
+  LiveWidgetController(WidgetsBinding binding) : super(binding);
+
+  @override
+  Future<Null> pump(Duration duration) async {
+    if (duration != null)
+      await new Future<void>.delayed(duration);
+    binding.scheduleFrame();
+    await binding.endOfFrame;
+  }
+}

packages/flutter_tools/lib/src/android/gradle.dart

@@ -361,11 +361,11 @@ Future<Null> _buildGradleProjectV2(
 
   command.add(assembleTask);
   final int exitCode = await runCommandAndStreamOutput(
-      command,
+    command,
-      workingDirectory: flutterProject.android.directory.path,
+    workingDirectory: flutterProject.android.directory.path,
-      allowReentrantFlutter: true,
+    allowReentrantFlutter: true,
-      environment: _gradleEnv,
+    environment: _gradleEnv,
-      filter: logger.isVerbose ? null : ndkMessageFilter,
+    filter: logger.isVerbose ? null : ndkMessageFilter,
   );
   status.stop();
 

packages/flutter_tools/lib/src/base/process.dart

@@ -302,10 +302,11 @@ String runSync(List<String> cmd, {
 
 void _traceCommand(List<String> args, { String workingDirectory }) {
   final String argsText = args.join(' ');
-  if (workingDirectory == null)
+  if (workingDirectory == null) {
-    printTrace(argsText);
+    printTrace('executing: $argsText');
-  else
+  } else {
-    printTrace('[$workingDirectory${fs.path.separator}] $argsText');
+    printTrace('executing: [$workingDirectory${fs.path.separator}] $argsText');
+  }
 }
 
 String _runWithLoggingSync(List<String> cmd, {