Refactor android_engine_test, make it easier to debug/deflake locally. (#161534)

The goal here is to have a great standalone `android_engine_test` suite
that [replaces
`scenario_app/android`](https://github.com/flutter/flutter/pull/160992).

No test is _functionally_ changed in this PR, but overview of changes:
- Finished renaming the suite `android_engine_tests` instead of
`flutter_driver_android`
- Added instructions and an environment variable for local generation of
golden-files (`UPDATE_GOLDENS=1`)
- Added explanations of the individual tests, where they live, and how
to run them locally
- Added a hybrid-composition (HC, not TLHC, which is already tested)
test
- Renamed "other_smiley" to "surface_texture_smiley" (and renamed the
original to "surface_producer_smiley")
- Removed unnecessary ".android" suffix (we will not run this on
anything but Android)
- Added a `tool/deflake.dart` to run a test suite 10x (or custom) times
locally to try and determine flakiness

After this PR, I'll add flags to let you control variants and name the
screenshots accordingly, i.e.:
- API v34 or v35
- OpenGLES or Vulkan (will require an `AndroidManifest.xml` edit during
the test instrumentation)

.ci.yaml

@@ -1528,7 +1528,7 @@ targets:
     bringup: true
     timeout: 60
     properties:
-      shard: flutter_driver_android
+      shard: android_engine_tests
       tags: >
          ["framework", "hostonly", "shard", "linux"]
       dependencies: >-
@@ -1541,7 +1541,7 @@ targets:
     recipe: flutter/flutter_drone
     timeout: 60
     properties:
-      shard: flutter_driver_android
+      shard: android_engine_tests
       tags: >
          ["framework", "hostonly", "shard", "linux"]
       dependencies: >-

dev/bots/suite_runners/run_android_engine_tests.dart

@@ -17,13 +17,21 @@ import '../utils.dart';
 /// 3. Run the following command from the root of the Flutter repository:
 ///
 /// ```sh
-/// SHARD=flutter_driver_android bin/cache/dart-sdk/bin/dart dev/bots/test.dart
+/// # Generate a baseline of local golden files.
+/// SHARD=android_engine_tests UPDATE_GOLDENS=1 bin/cache/dart-sdk/bin/dart dev/bots/test.dart
 /// ```
 ///
-/// For debugging, you need to instead run and launch these tests
+/// 4. Then, re-run the command against the baseline images:
-/// individually _in_ the `dev/integration_tests/android_engine_test` directory.
+///
-/// Comparisons against goldens cant happen locally.
+/// ```sh
-Future<void> runFlutterDriverAndroidTests() async {
+/// SHARD=android_engine_tests bin/cache/dart-sdk/bin/dart dev/bots/test.dart
+/// ```
+///
+/// If you are trying to debug a commit, you will want to run step (3) first,
+/// then apply the commit (or flag), and then run step (4). If you are trying
+/// to determine flakiness in the *same* state, or want better debugging, see
+/// `dev/integration_tests/android_engine_test/README.md`.
+Future<void> runAndroidEngineTests() async {
   print('Running Flutter Driver Android tests...');
 
   final String androidEngineTestPath = path.join('dev', 'integration_tests', 'android_engine_test');

dev/bots/test.dart

@@ -52,11 +52,11 @@ import 'package:path/path.dart' as path;
 import 'run_command.dart';
 import 'suite_runners/run_add_to_app_life_cycle_tests.dart';
 import 'suite_runners/run_analyze_tests.dart';
+import 'suite_runners/run_android_engine_tests.dart';
 import 'suite_runners/run_android_java11_integration_tool_tests.dart';
 import 'suite_runners/run_android_preview_integration_tool_tests.dart';
 import 'suite_runners/run_customer_testing_tests.dart';
 import 'suite_runners/run_docs_tests.dart';
-import 'suite_runners/run_flutter_driver_android_tests.dart';
 import 'suite_runners/run_flutter_packages_tests.dart';
 import 'suite_runners/run_framework_coverage_tests.dart';
 import 'suite_runners/run_framework_tests.dart';
@@ -139,7 +139,7 @@ Future<void> main(List<String> args) async {
       'web_skwasm_tests': webTestsSuite.runWebSkwasmUnitTests,
       // All web integration tests
       'web_long_running_tests': webTestsSuite.webLongRunningTestsRunner,
-      'flutter_driver_android': runFlutterDriverAndroidTests,
+      'android_engine_tests': runAndroidEngineTests,
       'flutter_plugins': flutterPackagesRunner,
       'skp_generator': skpGeneratorTestsRunner,
       'customer_testing': customerTestingRunner,

dev/integration_tests/android_engine_test/README.md

@@ -5,10 +5,35 @@ This directory contains a sample app and tests that demonstrate how to use the
 Android devices or emulators, interact with and capture screenshots of the app,
 and compare the screenshots against golden images.
 
+> [!CAUTION]
+> This test suite is a _very_ end-to-end suite that is testing a combination of
+> the graphics backend, the Android embedder, the Flutter framework, and Flutter
+> tools, and only useful when the documentation and naming stays up to date and
+> is clearly actionable.
+>
+> Please take extra care when updating the test suite to also update the REAMDE.
+
+## How it runs on CI (LUCI)
+
+See [`dev/bots/suite_runners/run_android_engine_tests.dart`](../../bots/suite_runners/run_android_engine_tests.dart), but tl;dr:
+
+```sh
+# TIP: If golden-files do not exist locally, this command will fail locally.
+SHARD=android_engine_tests bin/cache/dart-sdk/bin/dart dev/bots/test.dart
+```
+
 ## Running the apps and tests
 
 Each `lib/{prefix}_main.dart` file is a standalone Flutter app that you can run
-on an Android device or emulator:
+on an Android device or emulator.
+
+- [`flutter_rendered_blue_rectangle`](#flutter_rendered_blue_rectangle)
+- [`external_texture/surface_producer_smiley_face`](#external_texturesurface_producer_smiley_face)
+- [`external_texture/surface_texture_smiley_face`](#external_texturesurface_texture_smiley_face)
+- [`platform_view/hybrid_composition_platform_view`](#platform_viewhybrid_composition_platform_view)
+- [`platform_view/texture_layer_hybrid_composition_platform_view`](#platform_viewtexture_layer_hybrid_composition_platform_view)
+- [`platform_view/virtual_display_platform_view`](#platform_viewvirtual_display_platform_view)
+- [`platform_view_tap_color_change`](#platform_view_tap_color_change)
 
 ### `flutter_rendered_blue_rectangle`
 
@@ -25,13 +50,97 @@ $ flutter run lib/flutter_rendered_blue_rectangle_main.dart
 $ flutter drive lib/flutter_rendered_blue_rectangle_main.dart
 ```
 
-Files of significance:
+### `external_texture/surface_producer_smiley_face`
 
-- [Entrypoint](lib/flutter_rendered_blue_rectangle_main.dart)
+This app displays a full screen rectangular deformed smiley face with a yellow
-- [Test](test_driver/flutter_rendered_blue_rectangle_main_test.dart)
+background. It tests the [`SurfaceProducer`](https://api.flutter.dev/javadoc/io/flutter/view/TextureRegistry.SurfaceProducer.html) API end-to-end, including historic regression cases around
+backgrounding the app, trimming memory, and resuming the app.
 
-## Debugging tips
+```sh
+# Run the app
+$ flutter run lib/external_texture/surface_producer_smiley_face_main.dart
 
-- Use `flutter drive --keep-app-running` to keep the app running after the test.
+# Run the test
-- USe `flutter run` followed by `flutter drive --use-existing-app` for faster
+$ flutter drive lib/external_texture/surface_producer_smiley_face_main.dart
-  test iterations.
+```
+
+### `external_texture/surface_texture_smiley_face`
+
+This app displays a full screen rectangular deformed smiley face with a yellow
+background. It tests the [`SurfaceTexture`](https://api.flutter.dev/javadoc/io/flutter/view/TextureRegistry.SurfaceTexture.html) API end-to-end.
+
+```sh
+# Run the app
+$ flutter run lib/external_texture/surface_texture_smiley_face_main.dart
+
+# Run the test
+$ flutter drive lib/external_texture/surface_texture_smiley_face_main.dart
+```
+
+### `platform_view/hybrid_composition_platform_view`
+
+This app displays a blue orange gradient, the app is backgrounded, and then
+resumed. It tests the [Hybrid Composition](../../../docs/platforms/android/Android-Platform-Views.md#hybrid-composition) implementation.
+
+```sh
+# Run the app
+$ flutter run lib/platform_view/hybrid_composition_platform_view_main.dart
+
+# Run the test
+$ flutter drive lib/platform_view/hybrid_composition_platform_view_main.dart
+```
+
+### `platform_view/texture_layer_hybrid_composition_platform_view`
+
+This app displays a blue orange gradient, the app is backgrounded, and then
+resumed. It tests the [Texture Layer Hybrid Composition](../../../docs/platforms/android/Android-Platform-Views.md#texture-layer-hybrid-composition) implementation.
+
+```sh
+# Run the app
+$ flutter run lib/platform_view/texture_layer_hybrid_composition_platform_view_main.dart
+
+# Run the test
+$ flutter drive lib/platform_view/texture_layer_hybrid_composition_platform_view_main.dart
+```
+
+### `platform_view/virtual_display_platform_view`
+
+This app displays a blue orange gradient, the app is backgrounded, and then
+resumed. It tests the [Virtual Display](../../../docs/platforms/android/Android-Platform-Views.md#virtual-display) implementation.
+
+```sh
+# Run the app
+$ flutter run lib/platform_view/virtual_display_platform_view_main.dart
+
+# Run the test
+$ flutter drive lib/platform_view/virtual_display_platform_view_main.dart
+```
+
+### `platform_view_tap_color_change`
+
+This app displays a blue rectangle, using platform views, which upon
+being tapped (natively, not by Flutter), changes from blue to red.
+
+```sh
+# Run the app
+$ flutter run lib/platform_view_tap_color_change_main.dart
+
+# Run the test
+$ flutter drive lib/platform_view_tap_color_change_main_test.dart
+```
+
+## Deflaking
+
+Use `tool/deflake.dart <path/to/lib/main.dart>` to, in 1-command:
+
+- Build an APK.
+- Establish a baseline set of golden-files locally.
+- Run N tests (by default, 10) in the same state, asserting the same output.
+
+For example:
+
+```sh
+dart tool/deflake.dart lib/flutter_rendered_blue_rectangle_main.dart
+```
+
+For more options, see `dart tool/deflake.dart --help`.

dev/integration_tests/android_engine_test/lib/external_texture/surface_producer_smiley_face_main.dart

@@ -9,7 +9,7 @@ import 'package:flutter/material.dart';
 import 'package:flutter/services.dart';
 import 'package:flutter_driver/driver_extension.dart';
 
-import 'src/allow_list_devices.dart';
+import '../src/allow_list_devices.dart';
 
 const MethodChannel _channel = MethodChannel('smiley_face_texture');
 Future<int> _fetchTexture(int width, int height) async {

dev/integration_tests/android_engine_test/lib/external_texture/surface_texture_smiley_face_main.dart

@@ -9,7 +9,7 @@ import 'package:flutter/material.dart';
 import 'package:flutter/services.dart';
 import 'package:flutter_driver/driver_extension.dart';
 
-import 'src/allow_list_devices.dart';
+import '../src/allow_list_devices.dart';
 
 const MethodChannel _channel = MethodChannel('other_face_texture');
 Future<int> _fetchTexture(int width, int height) async {

dev/integration_tests/android_engine_test/lib/platform_view/hybrid_composition_platform_view_main.dart

@@ -0,0 +1,64 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:android_driver_extensions/extension.dart';
+import 'package:flutter/foundation.dart';
+import 'package:flutter/gestures.dart';
+import 'package:flutter/material.dart';
+import 'package:flutter/rendering.dart';
+import 'package:flutter/services.dart';
+import 'package:flutter_driver/driver_extension.dart';
+
+import '../src/allow_list_devices.dart';
+
+void main() async {
+  ensureAndroidDevice();
+  enableFlutterDriverExtension(commands: <CommandExtension>[nativeDriverCommands]);
+
+  // Run on full screen.
+  await SystemChrome.setEnabledSystemUIMode(SystemUiMode.immersive);
+  runApp(const MainApp());
+}
+
+final class MainApp extends StatelessWidget {
+  const MainApp({super.key});
+
+  @override
+  Widget build(BuildContext context) {
+    return const MaterialApp(
+      debugShowCheckedModeBanner: false,
+      home: _HybridCompositionAndroidPlatformView(viewType: 'blue_orange_gradient_platform_view'),
+    );
+  }
+}
+
+final class _HybridCompositionAndroidPlatformView extends StatelessWidget {
+  const _HybridCompositionAndroidPlatformView({required this.viewType});
+
+  final String viewType;
+
+  @override
+  Widget build(BuildContext context) {
+    return PlatformViewLink(
+      viewType: viewType,
+      surfaceFactory: (BuildContext context, PlatformViewController controller) {
+        return AndroidViewSurface(
+          controller: controller as AndroidViewController,
+          gestureRecognizers: const <Factory<OneSequenceGestureRecognizer>>{},
+          hitTestBehavior: PlatformViewHitTestBehavior.opaque,
+        );
+      },
+      onCreatePlatformView: (PlatformViewCreationParams params) {
+        return PlatformViewsService.initExpensiveAndroidView(
+            id: params.id,
+            viewType: viewType,
+            layoutDirection: TextDirection.ltr,
+            creationParamsCodec: const StandardMessageCodec(),
+          )
+          ..addOnPlatformViewCreatedListener(params.onPlatformViewCreated)
+          ..create();
+      },
+    );
+  }
+}

dev/integration_tests/android_engine_test/test_driver/external_texture/surface_producer_smiley_face_main_test.dart

@@ -7,19 +7,29 @@ import 'package:android_driver_extensions/skia_gold.dart';
 import 'package:flutter_driver/flutter_driver.dart';
 import 'package:test/test.dart';
 
-import '_luci_skia_gold_prelude.dart';
+import '../_luci_skia_gold_prelude.dart';
 
+/// For local debugging, a (local) golden-file is required as a baseline:
+///
+/// ```sh
+/// # Checkout HEAD, i.e. *before* changes you want to test.
+/// UPDATE_GOLDENS=1 flutter drive lib/external_texture/surface_producer_smiley_face_main.dart
+///
+/// # Make your changes.
+///
+/// # Run the test against baseline.
+/// flutter drive lib/external_texture/surface_producer_smiley_face_main.dart
+/// ```
+///
+/// For a convenient way to deflake a test, see `tool/deflake.dart`.
 void main() async {
-  // To test the golden file generation locally, comment out the following line.
-  // autoUpdateGoldenFiles = true;
-
   const String appName = 'com.example.android_engine_test';
   late final FlutterDriver flutterDriver;
   late final NativeDriver nativeDriver;
 
   setUpAll(() async {
     if (isLuci) {
-      await enableSkiaGoldComparator();
+      await enableSkiaGoldComparator(namePrefix: 'android_engine_test');
     }
     flutterDriver = await FlutterDriver.connect();
     nativeDriver = await AndroidNativeDriver.connect(flutterDriver);
@@ -32,8 +42,6 @@ void main() async {
   });
 
   test('should screenshot and match an external smiley face texture', () async {
-    await flutterDriver.waitFor(find.byType('Texture'));
-
     // On Android: Background the app, trim memory, and restore the app.
     if (nativeDriver case final AndroidNativeDriver nativeDriver) {
       print('Backgrounding the app, trimming memory, and resuming the app.');
@@ -48,7 +56,7 @@ void main() async {
 
     await expectLater(
       nativeDriver.screenshot(),
-      matchesGoldenFile('external_texture_smiley_face.android.png'),
+      matchesGoldenFile('external_texture_surface_producer_smiley_face.png'),
     );
   }, timeout: Timeout.none);
 }

dev/integration_tests/android_engine_test/test_driver/external_texture/surface_texture_smiley_face_main_test.dart

@@ -7,17 +7,28 @@ import 'package:android_driver_extensions/skia_gold.dart';
 import 'package:flutter_driver/flutter_driver.dart';
 import 'package:test/test.dart';
 
-import '_luci_skia_gold_prelude.dart';
+import '../_luci_skia_gold_prelude.dart';
 
+/// For local debugging, a (local) golden-file is required as a baseline:
+///
+/// ```sh
+/// # Checkout HEAD, i.e. *before* changes you want to test.
+/// UPDATE_GOLDENS=1 flutter drive lib/external_texture/surface_texture_smiley_face_main.dart
+///
+/// # Make your changes.
+///
+/// # Run the test against baseline.
+/// flutter drive lib/external_texture/surface_texture_smiley_face_main.dart
+/// ```
+///
+/// For a convenient way to deflake a test, see `tool/deflake.dart`.
 void main() async {
-  // To test the golden file generation locally, comment out the following line.
-  // autoUpdateGoldenFiles = true;
   late final FlutterDriver flutterDriver;
   late final NativeDriver nativeDriver;
 
   setUpAll(() async {
     if (isLuci) {
-      await enableSkiaGoldComparator();
+      await enableSkiaGoldComparator(namePrefix: 'android_engine_test');
     }
     flutterDriver = await FlutterDriver.connect();
     nativeDriver = await AndroidNativeDriver.connect(flutterDriver);
@@ -30,11 +41,9 @@ void main() async {
   });
 
   test('should screenshot and match a smiley face texture using the trampoline', () async {
-    await flutterDriver.waitFor(find.byType('Texture'));
-
     await expectLater(
       nativeDriver.screenshot(),
-      matchesGoldenFile('external_texture_other_face.android.png'),
+      matchesGoldenFile('external_texture_surface_texture_smiley_face.png'),
     );
   }, timeout: Timeout.none);
 }

dev/integration_tests/android_engine_test/test_driver/flutter_rendered_blue_rectangle_main_test.dart

@@ -9,16 +9,26 @@ import 'package:test/test.dart';
 
 import '_luci_skia_gold_prelude.dart';
 
+/// For local debugging, a (local) golden-file is required as a baseline:
+///
+/// ```sh
+/// # Checkout HEAD, i.e. *before* changes you want to test.
+/// UPDATE_GOLDENS=1 flutter drive lib/flutter_rendered_blue_rectangle_main.dart
+///
+/// # Make your changes.
+///
+/// # Run the test against baseline.
+/// flutter drive lib/flutter_rendered_blue_rectangle_main.dart
+/// ```
+///
+/// For a convenient way to deflake a test, see `tool/deflake.dart`.
 void main() async {
-  // To test the golden file generation locally, comment out the following line.
-  // autoUpdateGoldenFiles = true;
-
   late final FlutterDriver flutterDriver;
   late final NativeDriver nativeDriver;
 
   setUpAll(() async {
     if (isLuci) {
-      await enableSkiaGoldComparator();
+      await enableSkiaGoldComparator(namePrefix: 'android_engine_test');
     }
     flutterDriver = await FlutterDriver.connect();
     nativeDriver = await AndroidNativeDriver.connect(flutterDriver);
@@ -31,10 +41,9 @@ void main() async {
   });
 
   test('should screenshot and match a full-screen blue rectangle', () async {
-    await flutterDriver.waitFor(find.byType('DecoratedBox'));
     await expectLater(
       nativeDriver.screenshot(),
-      matchesGoldenFile('fluttered_rendered_blue_rectangle.android.png'),
+      matchesGoldenFile('fluttered_rendered_blue_rectangle.png'),
     );
   }, timeout: Timeout.none);
 }

dev/integration_tests/android_engine_test/test_driver/platform_view/hybrid_composition_platform_view_main_test.dart

@@ -0,0 +1,66 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:android_driver_extensions/native_driver.dart';
+import 'package:android_driver_extensions/skia_gold.dart';
+import 'package:flutter_driver/flutter_driver.dart';
+import 'package:test/test.dart';
+
+import '../_luci_skia_gold_prelude.dart';
+
+/// For local debugging, a (local) golden-file is required as a baseline:
+///
+/// ```sh
+/// # Checkout HEAD, i.e. *before* changes you want to test.
+/// UPDATE_GOLDENS=1 flutter drive lib/platform_view/hybrid_compoisition_platform_view_main.dart
+///
+/// # Make your changes.
+///
+/// # Run the test against baseline.
+/// flutter drive lib/platform_view/hybrid_compoisition_platform_view_main.dart
+/// ```
+///
+/// For a convenient way to deflake a test, see `tool/deflake.dart`.
+void main() async {
+  const String goldenPrefix = 'hybrid_composition_platform_view';
+
+  late final FlutterDriver flutterDriver;
+  late final NativeDriver nativeDriver;
+
+  setUpAll(() async {
+    if (isLuci) {
+      await enableSkiaGoldComparator(namePrefix: 'android_engine_test');
+    }
+    flutterDriver = await FlutterDriver.connect();
+    nativeDriver = await AndroidNativeDriver.connect(flutterDriver);
+    await nativeDriver.configureForScreenshotTesting();
+    await flutterDriver.waitUntilFirstFrameRasterized();
+  });
+
+  tearDownAll(() async {
+    await nativeDriver.close();
+    await flutterDriver.close();
+  });
+
+  test('should screenshot and match a blue -> orange gradient', () async {
+    await expectLater(
+      nativeDriver.screenshot(),
+      matchesGoldenFile('$goldenPrefix.blue_orange_gradient_portrait.png'),
+    );
+  }, timeout: Timeout.none);
+
+  test('should rotate landscape and screenshot the gradient', () async {
+    await nativeDriver.rotateToLandscape();
+    await expectLater(
+      nativeDriver.screenshot(),
+      matchesGoldenFile('$goldenPrefix.blue_orange_gradient_landscape_rotated.png'),
+    );
+
+    await nativeDriver.rotateResetDefault();
+    await expectLater(
+      nativeDriver.screenshot(),
+      matchesGoldenFile('$goldenPrefix.blue_orange_gradient_portait_rotated_back.png'),
+    );
+  }, timeout: Timeout.none);
+}

dev/integration_tests/android_engine_test/test_driver/platform_view/texture_layer_hybrid_composition_platform_view_main_test.dart

@@ -9,10 +9,20 @@ import 'package:test/test.dart';
 
 import '../_luci_skia_gold_prelude.dart';
 
+/// For local debugging, a (local) golden-file is required as a baseline:
+///
+/// ```sh
+/// # Checkout HEAD, i.e. *before* changes you want to test.
+/// UPDATE_GOLDENS=1 flutter drive lib/platform_view/texture_layer_hybrid_composition_platform_view_main.dart
+///
+/// # Make your changes.
+///
+/// # Run the test against baseline.
+/// flutter drive lib/platform_view/texture_layer_hybrid_composition_platform_view_main.dart
+/// ```
+///
+/// For a convenient way to deflake a test, see `tool/deflake.dart`.
 void main() async {
-  // To test the golden file generation locally, comment out the following line.
-  // autoUpdateGoldenFiles = true;
-
   const String goldenPrefix = 'texture_layer_hybrid_composition_platform_view';
 
   late final FlutterDriver flutterDriver;
@@ -20,7 +30,7 @@ void main() async {
 
   setUpAll(() async {
     if (isLuci) {
-      await enableSkiaGoldComparator();
+      await enableSkiaGoldComparator(namePrefix: 'android_engine_test');
     }
     flutterDriver = await FlutterDriver.connect();
     nativeDriver = await AndroidNativeDriver.connect(flutterDriver);
@@ -40,25 +50,23 @@ void main() async {
   });
 
   test('should screenshot and match a blue -> orange gradient', () async {
-    await flutterDriver.waitFor(find.byType('AndroidView'));
     await expectLater(
       nativeDriver.screenshot(),
-      matchesGoldenFile('$goldenPrefix.blue_orange_gradient_portrait.android.png'),
+      matchesGoldenFile('$goldenPrefix.blue_orange_gradient_portrait.png'),
     );
   }, timeout: Timeout.none);
 
   test('should rotate landscape and screenshot the gradient', () async {
-    await flutterDriver.waitFor(find.byType('AndroidView'));
     await nativeDriver.rotateToLandscape();
     await expectLater(
       nativeDriver.screenshot(),
-      matchesGoldenFile('$goldenPrefix.blue_orange_gradient_landscape_rotated.android.png'),
+      matchesGoldenFile('$goldenPrefix.blue_orange_gradient_landscape_rotated.png'),
     );
 
     await nativeDriver.rotateResetDefault();
     await expectLater(
       nativeDriver.screenshot(),
-      matchesGoldenFile('$goldenPrefix.blue_orange_gradient_portait_rotated_back.android.png'),
+      matchesGoldenFile('$goldenPrefix.blue_orange_gradient_portait_rotated_back.png'),
     );
   }, timeout: Timeout.none);
 }

dev/integration_tests/android_engine_test/test_driver/platform_view/virtual_display_platform_view_main_test.dart

@@ -9,10 +9,20 @@ import 'package:test/test.dart';
 
 import '../_luci_skia_gold_prelude.dart';
 
+/// For local debugging, a (local) golden-file is required as a baseline:
+///
+/// ```sh
+/// # Checkout HEAD, i.e. *before* changes you want to test.
+/// UPDATE_GOLDENS=1 flutter drive lib/platform_view/virtual_display_platform_view_main.dart
+///
+/// # Make your changes.
+///
+/// # Run the test against baseline.
+/// flutter drive lib/platform_view/virtual_display_platform_view_main.dart
+/// ```
+///
+/// For a convenient way to deflake a test, see `tool/deflake.dart`.
 void main() async {
-  // To test the golden file generation locally, comment out the following line.
-  // autoUpdateGoldenFiles = true;
-
   const String goldenPrefix = 'virtual_display_platform_view';
 
   late final FlutterDriver flutterDriver;
@@ -20,7 +30,7 @@ void main() async {
 
   setUpAll(() async {
     if (isLuci) {
-      await enableSkiaGoldComparator();
+      await enableSkiaGoldComparator(namePrefix: 'android_engine_test');
     }
     flutterDriver = await FlutterDriver.connect();
     nativeDriver = await AndroidNativeDriver.connect(flutterDriver);
@@ -40,25 +50,23 @@ void main() async {
   });
 
   test('should screenshot and match a blue -> orange gradient', () async {
-    await flutterDriver.waitFor(find.byType('AndroidView'));
     await expectLater(
       nativeDriver.screenshot(),
-      matchesGoldenFile('$goldenPrefix.blue_orange_gradient_portrait.android.png'),
+      matchesGoldenFile('$goldenPrefix.blue_orange_gradient_portrait.png'),
     );
   }, timeout: Timeout.none);
 
   test('should rotate landscape and screenshot the gradient', () async {
-    await flutterDriver.waitFor(find.byType('AndroidView'));
     await nativeDriver.rotateToLandscape();
     await expectLater(
       nativeDriver.screenshot(),
-      matchesGoldenFile('$goldenPrefix.blue_orange_gradient_landscape_rotated.android.png'),
+      matchesGoldenFile('$goldenPrefix.blue_orange_gradient_landscape_rotated.png'),
     );
 
     await nativeDriver.rotateResetDefault();
     await expectLater(
       nativeDriver.screenshot(),
-      matchesGoldenFile('$goldenPrefix.blue_orange_gradient_portait_rotated_back.android.png'),
+      matchesGoldenFile('$goldenPrefix.blue_orange_gradient_portait_rotated_back.png'),
     );
   }, timeout: Timeout.none);
 }

dev/integration_tests/android_engine_test/test_driver/platform_view_tap_color_change_main_test.dart

@@ -9,16 +9,26 @@ import 'package:test/test.dart';
 
 import '_luci_skia_gold_prelude.dart';
 
+/// For local debugging, a (local) golden-file is required as a baseline:
+///
+/// ```sh
+/// # Checkout HEAD, i.e. *before* changes you want to test.
+/// UPDATE_GOLDENS=1 flutter drive lib/platform_view_tap_color_change_main.dart
+///
+/// # Make your changes.
+///
+/// # Run the test against baseline.
+/// flutter drive lib/platform_view_tap_color_change_main.dart
+/// ```
+///
+/// For a convenient way to deflake a test, see `tool/deflake.dart`.
 void main() async {
-  // To test the golden file generation locally, comment out the following line.
-  // autoUpdateGoldenFiles = true;
-
   late final FlutterDriver flutterDriver;
   late final NativeDriver nativeDriver;
 
   setUpAll(() async {
     if (isLuci) {
-      await enableSkiaGoldComparator();
+      await enableSkiaGoldComparator(namePrefix: 'android_engine_test');
     }
     flutterDriver = await FlutterDriver.connect();
     nativeDriver = await AndroidNativeDriver.connect(flutterDriver);
@@ -31,16 +41,15 @@ void main() async {
   });
 
   test('should screenshot a rectangle that becomes blue after a tap', () async {
-    await flutterDriver.waitFor(find.byType('AndroidView'));
     await expectLater(
       nativeDriver.screenshot(),
-      matchesGoldenFile('platform_view_tap_color_change_initial.android.png'),
+      matchesGoldenFile('platform_view_tap_color_change_initial.png'),
     );
 
     await nativeDriver.tap(const ByNativeAccessibilityLabel('Change color'));
     await expectLater(
       nativeDriver.screenshot(),
-      matchesGoldenFile('platform_view_tap_color_change_tapped.android.png'),
+      matchesGoldenFile('platform_view_tap_color_change_tapped.png'),
     );
   }, timeout: Timeout.none);
 }

dev/integration_tests/android_engine_test/tool/deflake.dart

@@ -0,0 +1,163 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:convert';
+import 'dart:io' as io;
+
+import 'package:args/args.dart';
+import 'package:path/path.dart' as p;
+
+final ArgParser _argParser =
+    ArgParser()
+      ..addFlag('help', abbr: 'h', help: 'Display usage information.', negatable: false)
+      ..addFlag('verbose', abbr: 'v', help: 'Show noisy output while running', negatable: false)
+      ..addFlag(
+        'generate-initial-golden',
+        help:
+            'Whether an initial run (not part of "runs") should generate the '
+            'base golden file. If false, it is assumed the golden file wasl already generated.',
+        defaultsTo: true,
+      )
+      ..addFlag(
+        'build-app-once',
+        help:
+            'Whether to use flutter build and --use-application-binary instead of rebuilding every iteration.',
+        defaultsTo: true,
+      )
+      ..addOption('runs', abbr: 'n', help: 'How many times to run the test.', defaultsTo: '10');
+
+/// Builds, establishes a baseline, and runs a golden-file test N number of times.
+///
+/// Example use:
+/// ```sh
+/// dart ./tool/deflake.dart lib/external_texture/surface_texture_smiley_face_main.dart
+/// ```
+///
+/// By default it will:
+/// - Build the app once (and reuse the APK);
+/// - Generate a baseline (local) golden-file, overwriting your local file system;
+/// - Run N (by default, 10) subsequent tests, asserting the generated golden exactly matches.
+///
+/// For advanced usage, see `dart ./tool/deflake.dart --help`.
+void main(List<String> args) async {
+  final ArgResults argResults = _argParser.parse(args);
+  if (argResults.flag('help')) {
+    return _printUsage();
+  }
+
+  final List<String> testFiles = argResults.rest;
+  if (testFiles.length != 1) {
+    io.stderr.writeln('Exactly one test-file must be specified');
+    _printUsage();
+    io.exitCode = 1;
+    return;
+  }
+
+  final io.File testFile = io.File(testFiles.single);
+  if (!testFile.existsSync()) {
+    io.stderr.writeln('Not a file: ${testFile.path}');
+    _printUsage();
+    io.exitCode = 1;
+    return;
+  }
+
+  final bool generateInitialGolden = argResults.flag('generate-initial-golden');
+  final bool buildAppOnce = argResults.flag('build-app-once');
+  final bool verbose = argResults.flag('verbose');
+  final int runs;
+  {
+    final String rawRuns = argResults.option('runs')!;
+    final int? parsedRuns = int.tryParse(rawRuns);
+    if (parsedRuns == null || parsedRuns < 1) {
+      io.stderr.writeln('--runs must be a positive number: "$rawRuns".');
+      io.exitCode = 1;
+      return;
+    }
+    runs = parsedRuns;
+  }
+
+  final List<String> driverArgs;
+  if (buildAppOnce) {
+    io.stderr.writeln('Building initial app with "flutter build apk --debug...');
+    final io.Process proccess = await io.Process.start('flutter', <String>[
+      'build',
+      'apk',
+      '--debug',
+      testFile.path,
+    ], mode: verbose ? io.ProcessStartMode.inheritStdio : io.ProcessStartMode.normal);
+    if (await proccess.exitCode case final int exitCode when exitCode != 0) {
+      io.stderr.writeln('Failed to build (exit code = $exitCode).');
+      io.stderr.writeln(_collectStdOut(proccess));
+      io.exitCode = 1;
+      return;
+    }
+
+    // Strictly speaking, it would be better to parse stdout for:
+    // "✓ Built build/app/outputs/flutter-apk/app-debug.apk"
+    //
+    // ... _or_ specify the expected out ourselves and rely on that.
+    driverArgs = <String>[
+      'drive',
+      '--use-application-binary',
+      p.join('build', 'app', 'outputs', 'flutter-apk', 'app-debug.apk'),
+      testFile.path,
+    ];
+  } else {
+    // I can't imagine wanting to do this, but here is the option anyway!
+    driverArgs = <String>['drive', testFile.path];
+  }
+
+  Future<bool> runDriverTest({Map<String, String>? environment}) async {
+    final io.Process proccess = await io.Process.start(
+      'flutter',
+      driverArgs,
+      mode: verbose ? io.ProcessStartMode.inheritStdio : io.ProcessStartMode.normal,
+      environment: environment,
+    );
+    if (await proccess.exitCode case final int exitCode when exitCode != 0) {
+      io.stderr.writeln('Failed to build (exit code = $exitCode).');
+      io.stderr.writeln(_collectStdOut(proccess));
+      return false;
+    }
+    return true;
+  }
+
+  // Do an initial baseline run.
+  if (generateInitialGolden) {
+    io.stderr.writeln('Generating a baseline set of golden-files...');
+    await runDriverTest(environment: <String, String>{'UPDATE_GOLDENS': '1'});
+  }
+
+  // Now run.
+  int totalFailed = 0;
+  for (int i = 0; i < runs; i++) {
+    io.stderr.writeln('RUN ${i + 1} of $runs');
+    final bool result = await runDriverTest();
+    if (!result) {
+      totalFailed++;
+      io.stderr.writeln('FAIL');
+    } else {
+      io.stderr.writeln('PASS');
+    }
+  }
+
+  io.stderr.writeln('PASSED: ${runs - totalFailed} / $runs');
+  if (totalFailed != 0) {
+    io.exitCode = 1;
+  }
+}
+
+void _printUsage() {
+  io.stdout.writeln('Usage: dart tool/deflake.dart lib/<path-to-main>.dart');
+  io.stdout.writeln(_argParser.usage);
+}
+
+Future<String> _collectStdOut(io.Process process) async {
+  final StringBuffer buffer = StringBuffer();
+  buffer.writeln('stdout:');
+  buffer.writeln(await utf8.decodeStream(process.stdout));
+  buffer.writeln('stderr:');
+  buffer.writeln(await utf8.decodeStream(process.stderr));
+  return buffer.toString();
+}

dev/tools/android_driver_extensions/lib/skia_gold.dart

@@ -46,6 +46,12 @@ Future<void> enableSkiaGoldComparator({String? namePrefix}) async {
       'Set it to use Skia Gold.',
     );
   }
+  if (namePrefix != null) {
+    assert(
+      !namePrefix.endsWith('.'),
+      'The namePrefix automatically has a suffix of ".", so remove the last character from "$namePrefix".',
+    );
+  }
   final io.Directory tmpDir = io.Directory.systemTemp.createTempSync('android_driver_test');
   final bool isPresubmit = io.Platform.environment.containsKey(_kGoldctlPresubmitKey);
   io.stderr.writeln(
@@ -126,12 +132,6 @@ final class _GoldenFileComparator extends GoldenFileComparator {
       'Golden files in the Flutter framework must end with the file extension '
       '.png.',
     );
-    return Uri.parse(
+    return Uri.parse(<String>[if (namePrefix != null) namePrefix!, golden.toString()].join('.'));
-      <String>[
-        if (namePrefix != null) namePrefix!,
-        baseDir.pathSegments[baseDir.pathSegments.length - 2],
-        golden.toString(),
-      ].join('.'),
-    );
   }
 }

dev/tools/android_driver_extensions/lib/src/goldens.dart

@@ -16,7 +16,16 @@ part of '../native_driver.dart';
 ///
 /// When this is `true`, [matchesGoldenFile] will always report a successful
 /// match, because the bytes being tested implicitly become the new golden.
-bool autoUpdateGoldenFiles = false;
+///
+/// Defaults to `true` if the environment variable `UPDATE_GOLDENS` is either
+/// `true` or `1` (case insensitive).
+bool autoUpdateGoldenFiles = () {
+  final String? updateGoldens = io.Platform.environment['UPDATE_GOLDENS'];
+  return switch (updateGoldens?.toLowerCase()) {
+    '1' || 'true' => true,
+    _ => false,
+  };
+}();
 
 /// Compares pixels against those of a golden image file.
 ///
@@ -94,7 +103,17 @@ final class NaiveLocalFileComparator extends GoldenFileComparator {
     try {
       goldenBytes = await goldenFile.readAsBytes();
     } on io.PathNotFoundException {
-      throw TestFailure('Golden file not found: ${goldenFile.path}');
+      throw TestFailure(
+        'Golden file not found: ${path.relative(goldenFile.path)}.\n'
+        '\n'
+        'For local development, you must establish a local baseline image before '
+        'running tests, otherwise the test will always fail. Use UPDATE_GOLDENS=1 '
+        'when running "flutter drive" to establish a baseline, and then subequent '
+        '"flutter drive" instances will be tested against that (local) golden.\n'
+        '\n'
+        'See the documentation at dev/tools/android_engine_test/README.md for '
+        'details.',
+      );
     }
 
     if (goldenBytes.length != imageBytes.length) {