Add basic support for golden image file testing (#17094)

* Add a `matchesGoldenFile()` async matcher that will match
  a finder's widget's rasterized image against a golden file.
* Add support for pluggable image comparison backends
* Add a default backend that does simplistic PNG byte
  comparison on locally stored golden files.
* Add support for `flutter test --update-goldens`, which will
  treat the rasterized image bytes produced during the test
  as the new golden bytes and update the golden file accordingly

Still TODO:

* Add support for the `flutter_test_config.dart` test config hook
* Utilize `flutter_test_config.dart` in `packages/flutter/test`
  to install a backend that retrieves golden files from a dedicated
  `flutter/goldens` repo

https://github.com/flutter/flutter/issues/16859

dev/tools/vitool/pubspec.yaml

@@ -19,7 +19,8 @@ dependencies:
   stack_trace: 1.9.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
 
 dev_dependencies:
-  test: 0.12.34
+  flutter_test:
+    sdk: flutter
 
   async: 2.0.6 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   barback: 0.15.2+15 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"

dev/tools/vitool/test/vitool_test.dart

@@ -5,8 +5,8 @@
 import 'dart:math';
 
 import 'package:collection/collection.dart';
+import 'package:flutter_test/flutter_test.dart';
 import 'package:vitool/vitool.dart';
-import 'package:test/test.dart';
 import 'package:path/path.dart' as path;
 
 const String kPackagePath = '..';

packages/flutter_test/lib/flutter_test.dart

@@ -11,6 +11,7 @@ export 'src/all_elements.dart';
 export 'src/binding.dart';
 export 'src/controller.dart';
 export 'src/finders.dart';
+export 'src/goldens.dart';
 export 'src/matchers.dart';
 export 'src/nonconst.dart';
 export 'src/platform.dart';

packages/flutter_test/lib/src/goldens.dart

@@ -0,0 +1,171 @@
+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:io';
+import 'dart:typed_data';
+
+import 'package:path/path.dart' as path;
+import 'package:test/test.dart' show TestFailure;
+
+/// Compares rasterized image bytes against a golden image file.
+///
+/// Instances of this comparator will be used as the backend for
+/// [matchesGoldenFile].
+///
+/// Instances of this comparator will be invoked by the test framework in the
+/// [TestWidgetsFlutterBinding.runAsync] zone and are thus not subject to the
+/// fake async constraints that are normally imposed on widget tests (i.e. the
+/// need or the ability to call [WidgetTester.pump] to advance the microtask
+/// queue).
+abstract class GoldenFileComparator {
+  /// Compares [imageBytes] against the golden file identified by [golden].
+  ///
+  /// The returned future completes with a boolean value that indicates whether
+  /// [imageBytes] matches the golden file's bytes within the tolerance defined
+  /// by the comparator.
+  ///
+  /// In the case of comparison mismatch, the comparator may choose to throw a
+  /// [TestFailure] if it wants to control the failure message.
+  ///
+  /// The method by which [golden] is located and by which its bytes are loaded
+  /// is left up to the implementation class. For instance, some implementations
+  /// may load files from the local file system, whereas others may load files
+  /// over the network or from a remote repository.
+  Future<bool> compare(Uint8List imageBytes, Uri golden);
+
+  /// Updates the golden file identified by [golden] with [imageBytes].
+  ///
+  /// This will be invoked in lieu of [compare] when [autoUpdateGoldenFiles]
+  /// is `true` (which gets set automatically by the test framework when the
+  /// user runs `flutter test --update-goldens`).
+  ///
+  /// The method by which [golden] is located and by which its bytes are written
+  /// is left up to the implementation class.
+  Future<void> update(Uri golden, Uint8List imageBytes);
+}
+
+/// Compares rasterized image bytes against a golden image file.
+///
+/// This comparator is used as the backend for [matchesGoldenFile].
+///
+/// The default comparator, [LocalFileComparator], will treat the golden key as
+/// a relative path from the test file's directory. It will then load the
+/// golden file's bytes from disk and perform a byte-for-byte comparison of the
+/// encoded PNGs, returning true only if there's an exact match.
+///
+/// Callers may choose to override the default comparator by setting this to a
+/// custom comparator during test set-up. For example, some projects may wish to
+/// install a more intelligent comparator that knows how to decode the PNG
+/// images to raw pixels and compare pixel vales, reporting specific differences
+/// between the images.
+GoldenFileComparator goldenFileComparator = const _UninitializedComparator();
+
+/// Whether golden files should be automatically updated during tests rather
+/// than compared to the image bytes recorded by the tests.
+///
+/// When this is `true`, [matchesGoldenFile] will always report a successful
+/// match, because the bytes being tested implicitly become the new golden.
+///
+/// The Flutter tool will automatically set this to `true` when the user runs
+/// `flutter test --update-goldens`, so callers should generally never have to
+/// explicitly modify this value.
+///
+/// See also:
+///
+///   * [goldenFileComparator]
+bool autoUpdateGoldenFiles = false;
+
+/// Placeholder to signal an unexpected error in the testing framework itself.
+///
+/// The test harness file that gets generated by the Flutter tool when the
+/// user runs `flutter test` is expected to set [goldenFileComparator] to
+/// a valid comparator. From there, the caller may choose to override it by
+/// setting the comparator during test initialization (e.g. in `setUpAll()`).
+/// But under no circumstances do we expect it to remain uninitialized.
+class _UninitializedComparator implements GoldenFileComparator {
+  const _UninitializedComparator();
+
+  @override
+  Future<bool> compare(Uint8List imageBytes, Uri golden) {
+    throw new StateError('goldenFileComparator has not been initialized');
+  }
+
+  @override
+  Future<void> update(Uri golden, Uint8List imageBytes) {
+    throw new StateError('goldenFileComparator has not been initialized');
+  }
+}
+
+/// The default [GoldenFileComparator] implementation.
+///
+/// This comparator loads golden files from the local file system, treating the
+/// golden key as a relative path from the test file's directory.
+///
+/// This comparator performs a very simplistic comparison, doing a byte-for-byte
+/// comparison of the encoded PNGs, returning true only if there's an exact
+/// match. This means it will fail the test if two PNGs represent the same
+/// pixels but are encoded differently.
+class LocalFileComparator implements GoldenFileComparator {
+  /// Creates a new [LocalFileComparator] for the specified [testFile].
+  ///
+  /// Golden file keys will be interpreted as file paths relative to the
+  /// directory in which [testFile] resides.
+  ///
+  /// The [testFile] URI must represent a file.
+  LocalFileComparator(Uri testFile)
+      : assert(testFile.scheme == 'file'),
+        basedir = new Uri.directory(_path.dirname(_path.fromUri(testFile)));
+
+  // Due to https://github.com/flutter/flutter/issues/17118, we need to
+  // explicitly set the path style.
+  static final path.Context _path = new path.Context(style: Platform.isWindows
+      ? path.Style.windows
+      : path.Style.posix);
+
+  /// The directory in which the test was loaded.
+  ///
+  /// Golden file keys will be interpreted as file paths relative to this
+  /// directory.
+  final Uri basedir;
+
+  @override
+  Future<bool> compare(Uint8List imageBytes, Uri golden) async {
+    final File goldenFile = _getFile(golden);
+    if (!goldenFile.existsSync()) {
+      throw new TestFailure('Could not be compared against non-existent file: "$golden"');
+    }
+    final List<int> goldenBytes = await goldenFile.readAsBytes();
+    return _areListsEqual(imageBytes, goldenBytes);
+  }
+
+  @override
+  Future<void> update(Uri golden, Uint8List imageBytes) async {
+    final File goldenFile = _getFile(golden);
+    await goldenFile.writeAsBytes(imageBytes, flush: true);
+  }
+
+  File _getFile(Uri golden) {
+    return new File(_path.join(_path.fromUri(basedir), golden.path));
+  }
+
+  static bool _areListsEqual<T>(List<T> list1, List<T> list2) {
+    if (identical(list1, list2)) {
+      return true;
+    }
+    if (list1 == null || list2 == null) {
+      return false;
+    }
+    final int length = list1.length;
+    if (length != list2.length) {
+      return false;
+    }
+    for (int i = 0; i < length; i++) {
+      if (list1[i] != list2[i]) {
+        return false;
+      }
+    }
+    return true;
+  }
+}

packages/flutter_test/lib/src/matchers.dart

@@ -2,15 +2,21 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import 'dart:async';
 import 'dart:math' as math;
+import 'dart:ui' as ui;
 
+import 'package:flutter/foundation.dart';
 import 'package:flutter/material.dart';
 import 'package:flutter/rendering.dart';
 import 'package:flutter/services.dart';
 import 'package:meta/meta.dart';
-import 'package:test/test.dart';
+import 'package:test/test.dart' hide TypeMatcher;
+import 'package:test/src/frontend/async_matcher.dart'; // ignore: implementation_imports
 
+import 'binding.dart';
 import 'finders.dart';
+import 'goldens.dart';
 
 /// Asserts that the [Finder] matches no widgets in the widget tree.
 ///
@@ -234,7 +240,28 @@ Matcher isMethodCall(String name, {@required dynamic arguments}) {
 /// the area you expect to paint in for [areaToCompare] to catch errors where
 /// the path draws outside the expected area.
 Matcher coversSameAreaAs(Path expectedPath, {@required Rect areaToCompare, int sampleSize = 20})
-  => new _CoversSameAreaAs(expectedPath, areaToCompare: areaToCompare, sampleSize: sampleSize); 
+  => new _CoversSameAreaAs(expectedPath, areaToCompare: areaToCompare, sampleSize: sampleSize);
+
+/// Asserts that a [Finder] matches exactly one widget whose rendered image
+/// matches the golden image file identified by [key].
+///
+/// [key] may be either a [Uri] or a [String] representation of a URI.
+///
+/// This is an asynchronous matcher, meaning that callers should use
+/// [expectLater] when using this matcher and await the future returned by
+/// [expectLater].
+///
+/// See also:
+///
+///  * [goldenFileComparator], which acts as the backend for this matcher.
+Matcher matchesGoldenFile(dynamic key) {
+  if (key is Uri) {
+    return new _MatchesGoldenFile(key);
+  } else if (key is String) {
+    return new _MatchesGoldenFile.forStringPath(key);
+  }
+  throw new ArgumentError('Unexpected type for golden file: ${key.runtimeType}');
+}
 
 class _FindsWidgetMatcher extends Matcher {
   const _FindsWidgetMatcher(this.min, this.max);
@@ -1183,3 +1210,51 @@ class _CoversSameAreaAs extends Matcher {
   Description describe(Description description) =>
     description.add('covers expected area and only expected area');
 }
+
+class _MatchesGoldenFile extends AsyncMatcher {
+  const _MatchesGoldenFile(this.key);
+
+  _MatchesGoldenFile.forStringPath(String path) : key = Uri.parse(path);
+
+  final Uri key;
+
+  @override
+  Future<String> matchAsync(covariant Finder finder) async {
+    final Iterable<Element> elements = finder.evaluate();
+    if (elements.isEmpty) {
+      return 'could not be rendered because no widget was found';
+    } else if (elements.length > 1) {
+      return 'matched too many widgets';
+    }
+    final Element element = elements.single;
+
+    RenderObject renderObject = element.renderObject;
+    while (!renderObject.isRepaintBoundary) {
+      renderObject = renderObject.parent;
+      assert(renderObject != null);
+    }
+    assert(!renderObject.debugNeedsPaint);
+    final OffsetLayer layer = renderObject.layer;
+    final Future<ui.Image> imageFuture = layer.toImage(element.size);
+
+    final TestWidgetsFlutterBinding binding = TestWidgetsFlutterBinding.ensureInitialized();
+    return binding.runAsync<String>(() async {
+      final ui.Image image = await imageFuture;
+      final ByteData bytes = await image.toByteData(format: ui.ImageByteFormat.png);
+      if (autoUpdateGoldenFiles) {
+        await goldenFileComparator.update(key, bytes.buffer.asUint8List());
+      } else {
+        try {
+          final bool success = await goldenFileComparator.compare(bytes.buffer.asUint8List(), key);
+          return success ? null : 'does not match';
+        } on TestFailure catch (ex) {
+          return ex.message;
+        }
+      }
+    });
+  }
+
+  @override
+  Description describe(Description description) =>
+      description.add('one widget whose rasterized image matches golden image "$key"');
+}

packages/flutter_test/pubspec.yaml

@@ -2,17 +2,20 @@ name: flutter_test
 dependencies:
   # To update these, use "flutter update-packages --force-upgrade".
 
+  flutter:
+    sdk: flutter
+
   # We depend on very specific internal implementation details of the
   # 'test' package, which change between versions, so when upgrading
   # this, make sure the tests are still running correctly.
   test: 0.12.34
 
+  # Used by golden file comparator
+  path: 1.5.1
+
   # We use FakeAsync and other testing utilities.
   quiver: 0.29.0+1
 
-  flutter:
-    sdk: flutter
-
   # We import stack_trace because the test packages uses it and we
   # need to be able to unmangle the stack traces that it passed to
   # stack_trace. See https://github.com/dart-lang/test/issues/590
@@ -47,7 +50,6 @@ dependencies:
   node_preamble: 1.4.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   package_config: 1.0.3 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   package_resolver: 1.0.2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
-  path: 1.5.1 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   plugin: 0.2.0+2 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   pool: 1.3.4 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   pub_semver: 1.3.6 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
@@ -67,4 +69,9 @@ dependencies:
   web_socket_channel: 1.0.7 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
   yaml: 2.1.13 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
 
-# PUBSPEC CHECKSUM: 1343
+dev_dependencies:
+  file: 5.0.0
+
+  intl: 0.15.5 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade"
+
+# PUBSPEC CHECKSUM: 67b7

packages/flutter_test/test/goldens_test.dart

@@ -0,0 +1,151 @@
+// Copyright 2018 The Chromium Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:async';
+import 'dart:io' as io;
+import 'dart:typed_data';
+
+import 'package:file/memory.dart';
+import 'package:test/test.dart' as test_package;
+import 'package:test/test.dart' hide test;
+
+import 'package:flutter_test/flutter_test.dart' show goldenFileComparator, LocalFileComparator;
+
+const List<int> _kExpectedBytes = const <int>[1, 2, 3];
+
+void main() {
+  MemoryFileSystem fs;
+
+  setUp(() {
+    final FileSystemStyle style = io.Platform.isWindows
+        ? FileSystemStyle.windows
+        : FileSystemStyle.posix;
+    fs = new MemoryFileSystem(style: style);
+  });
+
+  void test(String description, FutureOr<void> body()) {
+    test_package.test(description, () {
+      return io.IOOverrides.runZoned(
+        body,
+        createDirectory: (String path) => fs.directory(path),
+        createFile: (String path) => fs.file(path),
+        createLink: (String path) => fs.link(path),
+        getCurrentDirectory: () => fs.currentDirectory,
+        setCurrentDirectory: (String path) => fs.currentDirectory = path,
+        getSystemTempDirectory: () => fs.systemTempDirectory,
+        stat: (String path) => fs.stat(path),
+        statSync: (String path) => fs.statSync(path),
+        fseIdentical: (String p1, String p2) => fs.identical(p1, p2),
+        fseIdenticalSync: (String p1, String p2) => fs.identicalSync(p1, p2),
+        fseGetType: (String path, bool followLinks) => fs.type(path, followLinks: followLinks),
+        fseGetTypeSync: (String path, bool followLinks) => fs.typeSync(path, followLinks: followLinks),
+        fsWatch: (String a, int b, bool c) => throw new UnsupportedError('unsupported'),
+        fsWatchIsSupported: () => fs.isWatchSupported,
+      );
+    });
+  }
+
+  group('goldenFileComparator', () {
+    test('is initialized by test framework', () {
+      expect(goldenFileComparator, isNotNull);
+      expect(goldenFileComparator, const isInstanceOf<LocalFileComparator>());
+      final LocalFileComparator comparator = goldenFileComparator;
+      expect(comparator.basedir.path, contains('flutter_test'));
+    });
+  });
+
+  group('LocalFileComparator', () {
+    LocalFileComparator comparator;
+
+    setUp(() {
+      comparator = new LocalFileComparator(new Uri.file('/golden_test.dart'));
+    });
+
+    test('calculates basedir correctly', () {
+      expect(comparator.basedir, new Uri.file('/'));
+      comparator = new LocalFileComparator(new Uri.file('/foo/bar/golden_test.dart'));
+      expect(comparator.basedir, new Uri.file('/foo/bar/'));
+    });
+
+    group('compare', () {
+      Future<bool> doComparison([String golden = 'golden.png']) {
+        final Uri uri = new Uri.file(golden);
+        return comparator.compare(
+          new Uint8List.fromList(_kExpectedBytes),
+          uri,
+        );
+      }
+
+      group('succeeds', () {
+        test('when golden file is in same folder as test', () async {
+          fs.file('/golden.png').writeAsBytesSync(_kExpectedBytes);
+          final bool success = await doComparison();
+          expect(success, isTrue);
+        });
+
+        test('when golden file is in subfolder of test', () async {
+          fs.file('/sub/foo.png')
+            ..createSync(recursive: true)
+            ..writeAsBytesSync(_kExpectedBytes);
+          final bool success = await doComparison('sub/foo.png');
+          expect(success, isTrue);
+        });
+      });
+
+      group('fails', () {
+        test('when golden file does not exist', () async {
+          final Future<bool> comparison = doComparison();
+          expect(comparison, throwsA(const isInstanceOf<TestFailure>()));
+        });
+
+        test('when golden bytes are leading subset of image bytes', () async {
+          fs.file('/golden.png').writeAsBytesSync(<int>[1, 2]);
+          expect(await doComparison(), isFalse);
+        });
+
+        test('when golden bytes are leading superset of image bytes', () async {
+          fs.file('/golden.png').writeAsBytesSync(<int>[1, 2, 3, 4]);
+          expect(await doComparison(), isFalse);
+        });
+
+        test('when golden bytes are trailing subset of image bytes', () async {
+          fs.file('/golden.png').writeAsBytesSync(<int>[2, 3]);
+          expect(await doComparison(), isFalse);
+        });
+
+        test('when golden bytes are trailing superset of image bytes', () async {
+          fs.file('/golden.png').writeAsBytesSync(<int>[0, 1, 2, 3]);
+          expect(await doComparison(), isFalse);
+        });
+
+        test('when golden bytes are disjoint from image bytes', () async {
+          fs.file('/golden.png').writeAsBytesSync(<int>[4, 5, 6]);
+          expect(await doComparison(), isFalse);
+        });
+
+        test('when golden bytes are empty', () async {
+          fs.file('/golden.png').writeAsBytesSync(<int>[]);
+          expect(await doComparison(), isFalse);
+        });
+      });
+    });
+
+    group('update', () {
+      test('updates existing file', () async {
+        fs.file('/golden.png').writeAsBytesSync(_kExpectedBytes);
+        const List<int> newBytes = const <int>[11, 12, 13];
+        await comparator.update(new Uri.file('golden.png'), new Uint8List.fromList(newBytes));
+        expect(fs.file('/golden.png').readAsBytesSync(), newBytes);
+      });
+
+      test('creates non-existent file', () async {
+        expect(fs.file('/foo.png').existsSync(), isFalse);
+        const List<int> newBytes = const <int>[11, 12, 13];
+        await comparator.update(new Uri.file('foo.png'), new Uint8List.fromList(newBytes));
+        expect(fs.file('/foo.png').existsSync(), isTrue);
+        expect(fs.file('/foo.png').readAsBytesSync(), newBytes);
+      });
+    });
+  });
+}

packages/flutter_test/test/matchers_test.dart

@@ -2,8 +2,10 @@
 // Use of this source code is governed by a BSD-style license that can be
 // found in the LICENSE file.
 
+import 'dart:typed_data';
 import 'dart:ui';
 
+import 'package:flutter/widgets.dart';
 import 'package:flutter_test/flutter_test.dart';
 
 /// Class that makes it easy to mock common toStringDeep behavior.
@@ -288,4 +290,137 @@ void main() {
       );
     });
   });
+
+  group('matchesGoldenFile', () {
+    _FakeComparator comparator;
+
+    Widget boilerplate(Widget child) {
+      return new Directionality(
+        textDirection: TextDirection.ltr,
+        child: child,
+      );
+    }
+
+    setUp(() {
+      comparator = new _FakeComparator();
+      goldenFileComparator = comparator;
+    });
+
+    group('matches', () {
+      testWidgets('if comparator succeeds', (WidgetTester tester) async {
+        await tester.pumpWidget(boilerplate(const Text('hello')));
+        final Finder finder = find.byType(Text);
+        await expectLater(finder, matchesGoldenFile('foo.png'));
+        expect(comparator.invocation, _ComparatorInvocation.compare);
+        expect(comparator.imageBytes, hasLength(greaterThan(0)));
+        expect(comparator.golden, Uri.parse('foo.png'));
+      });
+    });
+
+    group('does not match', () {
+      testWidgets('if comparator returns false', (WidgetTester tester) async {
+        comparator.behavior = _ComparatorBehavior.returnFalse;
+        await tester.pumpWidget(boilerplate(const Text('hello')));
+        final Finder finder = find.byType(Text);
+        try {
+          await expectLater(finder, matchesGoldenFile('foo.png'));
+          fail('TestFailure expected but not thrown');
+        } on TestFailure catch (error) {
+          expect(comparator.invocation, _ComparatorInvocation.compare);
+          expect(error.message, contains('does not match'));
+        }
+      });
+
+      testWidgets('if comparator throws', (WidgetTester tester) async {
+        comparator.behavior = _ComparatorBehavior.throwTestFailure;
+        await tester.pumpWidget(boilerplate(const Text('hello')));
+        final Finder finder = find.byType(Text);
+        try {
+          await expectLater(finder, matchesGoldenFile('foo.png'));
+          fail('TestFailure expected but not thrown');
+        } on TestFailure catch (error) {
+          expect(comparator.invocation, _ComparatorInvocation.compare);
+          expect(error.message, contains('fake message'));
+        }
+      });
+
+      testWidgets('if finder finds no widgets', (WidgetTester tester) async {
+        await tester.pumpWidget(boilerplate(new Container()));
+        final Finder finder = find.byType(Text);
+        try {
+          await expectLater(finder, matchesGoldenFile('foo.png'));
+          fail('TestFailure expected but not thrown');
+        } on TestFailure catch (error) {
+          expect(comparator.invocation, isNull);
+          expect(error.message, contains('no widget was found'));
+        }
+      });
+
+      testWidgets('if finder finds multiple widgets', (WidgetTester tester) async {
+        await tester.pumpWidget(boilerplate(new Column(
+          children: const <Widget>[const Text('hello'), const Text('world')],
+        )));
+        final Finder finder = find.byType(Text);
+        try {
+          await expectLater(finder, matchesGoldenFile('foo.png'));
+          fail('TestFailure expected but not thrown');
+        } on TestFailure catch (error) {
+          expect(comparator.invocation, isNull);
+          expect(error.message, contains('too many widgets'));
+        }
+      });
+    });
+
+    testWidgets('calls update on comparator if autoUpdateGoldenFiles is true', (WidgetTester tester) async {
+      autoUpdateGoldenFiles = true;
+      await tester.pumpWidget(boilerplate(const Text('hello')));
+      final Finder finder = find.byType(Text);
+      await expectLater(finder, matchesGoldenFile('foo.png'));
+      expect(comparator.invocation, _ComparatorInvocation.update);
+      expect(comparator.imageBytes, hasLength(greaterThan(0)));
+      expect(comparator.golden, Uri.parse('foo.png'));
+    });
+  });
+}
+
+enum _ComparatorBehavior {
+  returnTrue,
+  returnFalse,
+  throwTestFailure,
+}
+
+enum _ComparatorInvocation {
+  compare,
+  update,
+}
+
+class _FakeComparator implements GoldenFileComparator {
+  _ComparatorBehavior behavior = _ComparatorBehavior.returnTrue;
+  _ComparatorInvocation invocation;
+  Uint8List imageBytes;
+  Uri golden;
+
+  @override
+  Future<bool> compare(Uint8List imageBytes, Uri golden) {
+    invocation = _ComparatorInvocation.compare;
+    this.imageBytes = imageBytes;
+    this.golden = golden;
+    switch (behavior) {
+      case _ComparatorBehavior.returnTrue:
+        return new Future<bool>.value(true);
+      case _ComparatorBehavior.returnFalse:
+        return new Future<bool>.value(false);
+      case _ComparatorBehavior.throwTestFailure:
+        throw new TestFailure('fake message');
+    }
+    return new Future<bool>.value(false);
+  }
+
+  @override
+  Future<void> update(Uri golden, Uint8List imageBytes) {
+    invocation = _ComparatorInvocation.update;
+    this.golden = golden;
+    this.imageBytes = imageBytes;
+    return new Future<void>.value();
+  }
 }

packages/flutter_tools/lib/src/commands/test.dart

@@ -78,6 +78,11 @@ class TestCommand extends FlutterCommand {
         hide: !verboseHelp,
         help: 'Track widget creation locations.\n'
               'This enables testing of features such as the widget inspector.',
+      )
+      ..addFlag('update-goldens',
+        negatable: false,
+        help: 'Whether matchesGoldenFile() calls within your test methods should\n'
+              'update the golden files rather than test for an existing match.',
       );
   }
 
@@ -220,6 +225,7 @@ class TestCommand extends FlutterCommand {
       machine: machine,
       previewDart2: argResults['preview-dart-2'],
       trackWidgetCreation: argResults['track-widget-creation'],
+      updateGoldens: argResults['update-goldens'],
     );
 
     if (collector != null) {

packages/flutter_tools/lib/src/test/flutter_platform.dart

@@ -63,6 +63,7 @@ void installHook({
   int port: 0,
   String precompiledDillPath,
   bool trackWidgetCreation: false,
+  bool updateGoldens: false,
   int observatoryPort,
   InternetAddressType serverType: InternetAddressType.IP_V4,
 }) {
@@ -81,6 +82,7 @@ void installHook({
       port: port,
       precompiledDillPath: precompiledDillPath,
       trackWidgetCreation: trackWidgetCreation,
+      updateGoldens: updateGoldens,
     ),
   );
 }
@@ -211,6 +213,7 @@ class _FlutterPlatform extends PlatformPlugin {
     this.port,
     this.precompiledDillPath,
     this.trackWidgetCreation,
+    this.updateGoldens,
   }) : assert(shellPath != null);
 
   final String shellPath;
@@ -224,6 +227,7 @@ class _FlutterPlatform extends PlatformPlugin {
   final int port;
   final String precompiledDillPath;
   final bool trackWidgetCreation;
+  final bool updateGoldens;
 
   _Compiler compiler;
 
@@ -568,7 +572,7 @@ class _FlutterPlatform extends PlatformPlugin {
     final File listenerFile = fs.file('${temporaryDirectory.path}/listener.dart');
     listenerFile.createSync();
     listenerFile.writeAsStringSync(_generateTestMain(
-      testUrl: fs.path.toUri(fs.path.absolute(testPath)).toString(),
+      testUrl: fs.path.toUri(fs.path.absolute(testPath)),
       encodedWebsocketUrl: Uri.encodeComponent(_getWebSocketUrl()),
     ));
     return listenerFile.path;
@@ -616,7 +620,7 @@ class _FlutterPlatform extends PlatformPlugin {
   }
 
   String _generateTestMain({
-    String testUrl,
+    Uri testUrl,
     String encodedWebsocketUrl,
   }) {
     return '''
@@ -628,6 +632,7 @@ import 'dart:io';  // ignore: dart_io_import
 // to add a dependency on package:test.
 import 'package:test/src/runner/plugin/remote_platform_helpers.dart';
 
+import 'package:flutter_test/flutter_test.dart';
 import 'package:stream_channel/stream_channel.dart';
 import 'package:test/src/runner/vm/catch_isolate_errors.dart';
 
@@ -639,6 +644,8 @@ void main() {
   String server = Uri.decodeComponent('$encodedWebsocketUrl:\$serverPort');
   StreamChannel channel = serializeSuite(() {
     catchIsolateErrors();
+    goldenFileComparator = new LocalFileComparator(Uri.parse('$testUrl'));
+    autoUpdateGoldenFiles = $updateGoldens;
     return test.main;
   });
   WebSocket.connect(server).then((WebSocket socket) {

packages/flutter_tools/lib/src/test/runner.dart

@@ -31,6 +31,7 @@ Future<int> runTests(
     bool machine: false,
     bool previewDart2: false,
     bool trackWidgetCreation: false,
+    bool updateGoldens: false,
     TestWatcher watcher,
     }) async {
   if (trackWidgetCreation && !previewDart2) {
@@ -87,6 +88,7 @@ Future<int> runTests(
     serverType: serverType,
     previewDart2: previewDart2,
     trackWidgetCreation: trackWidgetCreation,
+    updateGoldens: updateGoldens,
   );
 
   // Make the global packages path absolute.