Add frameSync mechanism to flutter_driver. (#7471)

With frameSync enabled, flutter_driver actions will only be performed
when there are no pending frames in the app under test. This helps with
reducing flakiness.

examples/flutter_gallery/test_driver/memory_nav_test.dart

@@ -3,6 +3,8 @@ import 'dart:async';
 import 'package:flutter_driver/flutter_driver.dart';
 import 'package:test/test.dart';
 
+const Duration kWaitBetweenActions = const Duration(milliseconds: 250);
+
 void main() {
   group('flutter gallery transitions', () {
     FlutterDriver driver;
@@ -18,13 +20,13 @@ void main() {
     test('navigation', () async {
       SerializableFinder menuItem = find.text('Text fields');
       await driver.scrollIntoView(menuItem);
-      await new Future<Null>.delayed(new Duration(milliseconds: 500));
+      await new Future<Null>.delayed(kWaitBetweenActions);
 
       for (int i = 0; i < 15; i++) {
         await driver.tap(menuItem);
-        await new Future<Null>.delayed(new Duration(milliseconds: 1000));
+        await new Future<Null>.delayed(kWaitBetweenActions);
         await driver.tap(find.byTooltip('Back'));
-        await new Future<Null>.delayed(new Duration(milliseconds: 1000));
+        await new Future<Null>.delayed(kWaitBetweenActions);
       }
     }, timeout: new Timeout(new Duration(minutes: 1)));
   });

examples/flutter_gallery/test_driver/transitions_perf_test.dart

@@ -59,8 +59,15 @@ final List<String> demoTitles = <String>[
   'Typography'
 ];
 
+// Subset of [demoTitles] that needs frameSync turned off.
+final List<String> unsynchedDemoTitles = <String>[
+  'Progress indicators',
+];
+
 final FileSystem _fs = new LocalFileSystem();
 
+const Duration kWaitBetweenActions = const Duration(milliseconds: 250);
+
 /// Extracts event data from [events] recorded by timeline, validates it, turns
 /// it into a histogram, and saves to a JSON file.
 Future<Null> saveDurationsHistogram(List<Map<String, dynamic>> events, String outputPath) async {
@@ -144,7 +151,7 @@ void main() {
         // Expand the demo category submenus.
         for (String category in demoCategories.reversed) {
           await driver.tap(find.text(category));
-          await new Future<Null>.delayed(new Duration(milliseconds: 500));
+          await new Future<Null>.delayed(kWaitBetweenActions);
         }
         // Scroll each demo menu item into view, launch the demo and
         // return to the demo menu 2x.
@@ -152,13 +159,20 @@ void main() {
           print('Testing "$demoTitle" demo');
           SerializableFinder menuItem = find.text(demoTitle);
           await driver.scrollIntoView(menuItem);
-          await new Future<Null>.delayed(new Duration(milliseconds: 500));
+          await new Future<Null>.delayed(kWaitBetweenActions);
 
           for(int i = 0; i < 2; i += 1) {
             await driver.tap(menuItem); // Launch the demo
-            await new Future<Null>.delayed(new Duration(milliseconds: 500));
+            await new Future<Null>.delayed(kWaitBetweenActions);
+            if (!unsynchedDemoTitles.contains(demoTitle)) {
               await driver.tap(find.byTooltip('Back'));
-            await new Future<Null>.delayed(new Duration(milliseconds: 1000));
+            } else {
+              await driver.runUnsynchronized(() async {
+                await new Future<Null>.delayed(kWaitBetweenActions);
+                await driver.tap(find.byTooltip('Back'));
+              });
+            }
+            await new Future<Null>.delayed(kWaitBetweenActions);
           }
           print('Success');
         }

packages/flutter_driver/lib/src/driver.dart

@@ -17,6 +17,7 @@ import 'health.dart';
 import 'input.dart';
 import 'message.dart';
 import 'render_tree.dart';
+import 'frame_sync.dart';
 import 'timeline.dart';
 
 /// Timeline stream identifier.
@@ -395,6 +396,33 @@ class FlutterDriver {
     return stopTracingAndDownloadTimeline();
   }
 
+  /// [action] will be executed with the frame sync mechanism disabled.
+  ///
+  /// By default, Flutter Driver waits until there is no pending frame scheduled
+  /// in the app under test before executing an action. This mechanism is called
+  /// "frame sync". It greatly reduces flakiness because Flutter Driver will not
+  /// execute an action while the app under test is undergoing a transition.
+  ///
+  /// Having said that, sometimes it is necessary to disable the frame sync
+  /// mechanism (e.g. if there is an ongoing animation in the app, it will
+  /// never reach a state where there are no pending frames scheduled and the
+  /// action will time out). For these cases, the sync mechanism can be disabled
+  /// by wrapping the actions to be performed by this [runUnsynchronized] method.
+  ///
+  /// With frame sync disabled, its the responsibility of the test author to
+  /// ensure that no action is performed while the app is undergoing a
+  /// transition to avoid flakiness.
+  Future<dynamic/*=T*/> runUnsynchronized/*<T>*/(Future<dynamic/*=T*/> action()) async {
+    await _sendCommand(new SetFrameSync(false));
+    dynamic/*=T*/ result;
+    try {
+      result = await action();
+    } finally {
+      await _sendCommand(new SetFrameSync(true));
+    }
+    return result;
+  }
+
   /// Closes the underlying connection to the VM service.
   ///
   /// Returns a [Future] that fires once the connection has been closed.

packages/flutter_driver/lib/src/extension.dart

@@ -18,6 +18,7 @@ import 'health.dart';
 import 'input.dart';
 import 'message.dart';
 import 'render_tree.dart';
+import 'frame_sync.dart';
 
 const String _extensionMethodName = 'driver';
 const String _extensionMethod = 'ext.flutter.$_extensionMethodName';
@@ -65,6 +66,7 @@ class _FlutterDriverExtension {
       'get_render_tree': _getRenderTree,
       'tap': _tap,
       'get_text': _getText,
+      'set_frame_sync': _setFrameSync,
       'scroll': _scroll,
       'scrollIntoView': _scrollIntoView,
       'setInputText': _setInputText,
@@ -73,15 +75,16 @@ class _FlutterDriverExtension {
     });
 
     _commandDeserializers.addAll(<String, CommandDeserializerCallback>{
-      'get_health': (Map<String, dynamic> json) => new GetHealth.deserialize(json),
-      'get_render_tree': (Map<String, dynamic> json) => new GetRenderTree.deserialize(json),
-      'tap': (Map<String, dynamic> json) => new Tap.deserialize(json),
-      'get_text': (Map<String, dynamic> json) => new GetText.deserialize(json),
-      'scroll': (Map<String, dynamic> json) => new Scroll.deserialize(json),
-      'scrollIntoView': (Map<String, dynamic> json) => new ScrollIntoView.deserialize(json),
-      'setInputText': (Map<String, dynamic> json) => new SetInputText.deserialize(json),
-      'submitInputText': (Map<String, dynamic> json) => new SubmitInputText.deserialize(json),
-      'waitFor': (Map<String, dynamic> json) => new WaitFor.deserialize(json),
+      'get_health': (Map<String, String> params) => new GetHealth.deserialize(params),
+      'get_render_tree': (Map<String, String> params) => new GetRenderTree.deserialize(params),
+      'tap': (Map<String, String> params) => new Tap.deserialize(params),
+      'get_text': (Map<String, String> params) => new GetText.deserialize(params),
+      'set_frame_sync': (Map<String, String> params) => new SetFrameSync.deserialize(params),
+      'scroll': (Map<String, String> params) => new Scroll.deserialize(params),
+      'scrollIntoView': (Map<String, String> params) => new ScrollIntoView.deserialize(params),
+      'setInputText': (Map<String, String> params) => new SetInputText.deserialize(params),
+      'submitInputText': (Map<String, String> params) => new SubmitInputText.deserialize(params),
+      'waitFor': (Map<String, String> params) => new WaitFor.deserialize(params),
     });
 
     _finders.addAll(<String, FinderConstructor>{
@@ -96,6 +99,10 @@ class _FlutterDriverExtension {
   final Map<String, CommandDeserializerCallback> _commandDeserializers = <String, CommandDeserializerCallback>{};
   final Map<String, FinderConstructor> _finders = <String, FinderConstructor>{};
 
+  /// With [_frameSync] enabled, Flutter Driver will wait to perform an action
+  /// until there are no pending frames in the app under test.
+  bool _frameSync = true;
+
   /// Processes a driver command configured by [params] and returns a result
   /// as an arbitrary JSON object.
   ///
@@ -135,46 +142,38 @@ class _FlutterDriverExtension {
     };
   }
 
-  Stream<Duration> _onFrameReadyStream;
-  Stream<Duration> get _onFrameReady {
-    if (_onFrameReadyStream == null) {
-      // Lazy-initialize the frame callback because the renderer is not yet
-      // available at the time the extension is registered.
-      StreamController<Duration> frameReadyController = new StreamController<Duration>.broadcast(sync: true);
-      SchedulerBinding.instance.addPersistentFrameCallback((Duration timestamp) {
-        frameReadyController.add(timestamp);
-      });
-      _onFrameReadyStream = frameReadyController.stream;
-    }
-    return _onFrameReadyStream;
-  }
-
   Future<Health> _getHealth(Command command) async => new Health(HealthStatus.ok);
 
   Future<RenderTree> _getRenderTree(Command command) async {
     return new RenderTree(RendererBinding.instance?.renderView?.toStringDeep());
   }
 
-  /// Runs `finder` repeatedly until it finds one or more [Element]s.
-  Future<Finder> _waitForElement(Finder finder) {
-    // Short-circuit if the element is already on the UI
-    if (finder.precache())
-      return new Future<Finder>.value(finder);
-
-    // No element yet, so we retry on frames rendered in the future.
-    Completer<Finder> completer = new Completer<Finder>();
-    StreamSubscription<Duration> subscription;
-
-    subscription = _onFrameReady.listen((Duration duration) {
-      if (finder.precache()) {
-        subscription.cancel();
-        completer.complete(finder);
-      }
+  // Waits until at the end of a frame the provided [condition] is [true].
+  Future<Null> _waitUntilFrame(bool condition(), [Completer<Null> completer]) {
+    completer ??= new Completer<Null>();
+    if (!condition()) {
+      SchedulerBinding.instance.addPostFrameCallback((Duration timestamp) {
+        _waitUntilFrame(condition, completer);
       });
-
+    } else {
+      completer.complete();
+    }
     return completer.future;
   }
 
+  /// Runs `finder` repeatedly until it finds one or more [Element]s.
+  Future<Finder> _waitForElement(Finder finder) async {
+    if (_frameSync)
+      await _waitUntilFrame(() => SchedulerBinding.instance.transientCallbackCount == 0);
+
+    await _waitUntilFrame(() => finder.precache());
+
+    if (_frameSync)
+      await _waitUntilFrame(() => SchedulerBinding.instance.transientCallbackCount == 0);
+
+    return finder;
+  }
+
   Finder _createByTextFinder(ByText arguments) {
     return find.text(arguments.text);
   }
@@ -242,7 +241,7 @@ class _FlutterDriverExtension {
   Future<ScrollResult> _scrollIntoView(Command command) async {
     ScrollIntoView scrollIntoViewCommand = command;
     Finder target = await _waitForElement(_createFinder(scrollIntoViewCommand.finder));
-    await Scrollable.ensureVisible(target.evaluate().single);
+    await Scrollable.ensureVisible(target.evaluate().single, duration: const Duration(milliseconds: 100));
     return new ScrollResult();
   }
 
@@ -269,4 +268,10 @@ class _FlutterDriverExtension {
     Text text = target.evaluate().single.widget;
     return new GetTextResult(text.data);
   }
+
+  Future<SetFrameSyncResult> _setFrameSync(Command command) async {
+    SetFrameSync setFrameSyncCommand = command;
+    _frameSync = setFrameSyncCommand.enabled;
+    return new SetFrameSyncResult();
+  }
 }

packages/flutter_driver/lib/src/frame_sync.dart

@@ -0,0 +1,37 @@
+// Copyright 2017 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'message.dart';
+
+/// Enables or disables the FrameSync mechanism.
+class SetFrameSync extends Command {
+  @override
+  final String kind = 'set_frame_sync';
+
+  /// Whether frameSync should be enabled or disabled.
+  final bool enabled;
+
+  SetFrameSync(this.enabled) : super();
+
+  /// Deserializes this command from the value generated by [serialize].
+  SetFrameSync.deserialize(Map<String, String> params)
+      : this.enabled = params['enabled'].toLowerCase() == 'true',
+        super.deserialize(params);
+
+  @override
+  Map<String, String> serialize() => super.serialize()..addAll(<String, String>{
+    'enabled': '$enabled',
+  });
+}
+
+/// The result of a [SetFrameSync] command.
+class SetFrameSyncResult extends Result {
+  /// Deserializes this result from JSON.
+  static SetFrameSyncResult fromJson(Map<String, dynamic> json) {
+    return new SetFrameSyncResult();
+  }
+
+  @override
+  Map<String, dynamic> toJson() => <String, dynamic>{};
+}