github/flutter
7
0
Fork 0
mirror of https://github.com/flutter/flutter.git synced 2025-06-03 00:51:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Reland: Skia gold driver test (#49905)

Browse Source
This commit is contained in:
Emmanuel Garcia 2020-01-31 14:50:50 -08:00 committed by GitHub
parent 8061894f05
commit e03f439145
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 338 additions and 232 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
dev/devicelab/bin/tasks/flutter_driver_screenshot_test_fuchsia.dart
View File

@ -10,5 +10,5 @@ import 'package:flutter_devicelab/tasks/integration_tests.dart';
Future<void> main() async {
deviceOperatingSystem = DeviceOperatingSystem.fuchsia;
await task(createFlutterDriverScreenshotTest());
await task(createFlutterDriverScreenshotTest(useFlutterGold: true));
}

11
dev/devicelab/lib/tasks/integration_tests.dart
View File

@ -107,15 +107,22 @@ TaskFunction createAndroidSplashScreenKitchenSinkTest() {
);
}
TaskFunction createFlutterDriverScreenshotTest() {
/// Executes a driver test that takes a screenshot and compares it against a golden image.
/// If [useFlutterGold] is true, the golden image is served by Flutter Gold
/// (https://flutter-gold.skia.org/), otherwise the golden image is read from the disk.
TaskFunction createFlutterDriverScreenshotTest({
bool useFlutterGold = false,
}) {
return DriverTest(
'${flutterDirectory.path}/dev/integration_tests/flutter_driver_screenshot_test',
'lib/main.dart',
extraOptions: useFlutterGold ? const <String>[
'--driver', 'test_driver/flutter_gold_main_test.dart'
] : const <String>[]
);
}
class DriverTest {
DriverTest(
this.testDirectory,
this.testTarget, {

12
dev/integration_tests/flutter_driver_screenshot_test/README.md
View File

@ -4,7 +4,12 @@ This tests contains an app with a main page and sub pages.
The main page contains a list of buttons; each button leads to a designated sub page when tapped on.
Each sub page should displays some simple UIs to screenshot tested.
The flutter driver test runs the app and opens each page to take a screenshot. Then it compares the screenshot against a golden image stored in `test_driver/goldens/<some_test_page_name>/<device_model>.png`.
The flutter driver test runs the app and opens each page to take a screenshot.
Use `test_driver/flutter_gold_main_test.dart` to test against golden files stored on Flutter Gold.
Otherwise, use `main_test.dart` to test against golden files stored on `test_driver/goldens/<some_test_page_name>/<device_model>.png`.
Note that new binaries can't be checked in the Flutter repo, so use [Flutter Gold](https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package:flutter) instead.
# Add a new page to test
@ -16,6 +21,7 @@ The flutter driver test runs the app and opens each page to take a screenshot. T
An example of a `Page` subclass can be found in `lib/image_page.dart`
# Experiments
# Environments
The test currently only runs on device lab ["mac/ios"] which runs the app on iPhone 6s.
* Device Lab which runs the app on iPhone 6s.
* LUCI which runs the app on a Fuchsia NUC device.

38
dev/integration_tests/flutter_driver_screenshot_test/test_driver/flutter_gold_main_test.dart Normal file
View File

@ -0,0 +1,38 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'package:flutter_driver/flutter_driver.dart';
import 'package:test/test.dart' hide TypeMatcher, isInstanceOf;
import 'package:flutter_test/src/buffer_matcher.dart';
Future<void> main() async {
FlutterDriver driver;
String deviceModel;
setUpAll(() async {
driver = await FlutterDriver.connect();
deviceModel = await driver.requestData('device_model');
});
tearDownAll(() => driver.close());
test('A page with an image screenshot', () async {
final SerializableFinder imagePageListTile =
find.byValueKey('image_page');
await driver.waitFor(imagePageListTile);
await driver.tap(imagePageListTile);
await driver.waitFor(find.byValueKey('red_square_image'));
await driver.waitUntilNoTransientCallbacks();
// TODO(egarciad): This is currently a no-op on LUCI.
// https://github.com/flutter/flutter/issues/49837
await expectLater(
driver.screenshot(),
bufferMatchesGoldenFile('red_square_driver_screenshot__$deviceModel.png'),
);
await driver.tap(find.byTooltip('Back'));
});
}

92
packages/flutter_test/lib/src/_goldens_io.dart
View File

@ -4,11 +4,8 @@
import 'dart:async';
import 'dart:io';
import 'dart:math' as math;
import 'dart:typed_data';
import 'dart:ui';
import 'package:flutter/widgets.dart' show Element;
import 'package:image/image.dart';
import 'package:path/path.dart' as path;
// ignore: deprecated_member_use
@ -166,92 +163,3 @@ class LocalComparisonOutput {
));
}
}
/// Returns a [ComparisonResult] to describe the pixel differential of the
/// [test] and [master] image bytes provided.
ComparisonResult compareLists(List<int> test, List<int> master) {
if (identical(test, master))
return ComparisonResult(passed: true);
if (test == null || master == null || test.isEmpty || master.isEmpty) {
return ComparisonResult(
passed: false,
error: 'Pixel test failed, null image provided.',
);
}
final Image testImage = decodePng(test);
final Image masterImage = decodePng(master);
assert(testImage != null);
assert(masterImage != null);
final int width = testImage.width;
final int height = testImage.height;
if (width != masterImage.width || height != masterImage.height) {
return ComparisonResult(
passed: false,
error: 'Pixel test failed, image sizes do not match.\n'
'Master Image: ${masterImage.width} X ${masterImage.height}\n'
'Test Image: ${testImage.width} X ${testImage.height}',
);
}
int pixelDiffCount = 0;
final int totalPixels = width * height;
final Image invertedMaster = invert(Image.from(masterImage));
final Image invertedTest = invert(Image.from(testImage));
final Map<String, Image> diffs = <String, Image>{
'masterImage' : masterImage,
'testImage' : testImage,
'maskedDiff' : Image.from(testImage),
'isolatedDiff' : Image(width, height),
};
for (int x = 0; x < width; x++) {
for (int y =0; y < height; y++) {
final int testPixel = testImage.getPixel(x, y);
final int masterPixel = masterImage.getPixel(x, y);
final int diffPixel = (getRed(testPixel) - getRed(masterPixel)).abs()
+ (getGreen(testPixel) - getGreen(masterPixel)).abs()
+ (getBlue(testPixel) - getBlue(masterPixel)).abs()
+ (getAlpha(testPixel) - getAlpha(masterPixel)).abs();
if (diffPixel != 0 ) {
final int invertedMasterPixel = invertedMaster.getPixel(x, y);
final int invertedTestPixel = invertedTest.getPixel(x, y);
final int maskPixel = math.max(invertedMasterPixel, invertedTestPixel);
diffs['maskedDiff'].setPixel(x, y, maskPixel);
diffs['isolatedDiff'].setPixel(x, y, maskPixel);
pixelDiffCount++;
}
}
}
if (pixelDiffCount > 0) {
return ComparisonResult(
passed: false,
error: 'Pixel test failed, '
'${((pixelDiffCount/totalPixels) * 100).toStringAsFixed(2)}% '
'diff detected.',
diffs: diffs,
);
}
return ComparisonResult(passed: true);
}
/// An unsupported [WebGoldenComparator] that exists for API compatibility.
class DefaultWebGoldenComparator extends WebGoldenComparator {
@override
Future<bool> compare(Element element, Size size, Uri golden) {
throw UnsupportedError('DefaultWebGoldenComparator is only supported on the web.');
}
@override
Future<void> update(Uri golden, Element element, Size size) {
throw UnsupportedError('DefaultWebGoldenComparator is only supported on the web.');
}
}

130
packages/flutter_test/lib/src/_goldens_web.dart
View File

@ -9,6 +9,7 @@ import 'dart:ui';
import 'package:flutter/rendering.dart';
import 'package:flutter/widgets.dart';
import 'package:path/path.dart' as path;
// ignore: deprecated_member_use
import 'package:test_api/test_api.dart' as test_package show TestFailure;
@ -35,6 +36,115 @@ ComparisonResult compareLists(List<int> test, List<int> master) {
throw UnsupportedError('Golden testing is not supported on the web.');
}
/// Compares image pixels against a golden image file.
///
/// Instances of this comparator will be used as the backend for
/// [matchesGoldenFile] when tests are running on Flutter Web, and will usually
/// implemented by deferring the screenshot taking and image comparison to a
/// test server.
///
/// Instances of this comparator will be invoked by the test framework in the
/// [TestWidgetsFlutterBinding.runAsync] zone and are thus not subject to the
/// fake async constraints that are normally imposed on widget tests (i.e. the
/// need or the ability to call [WidgetTester.pump] to advance the microtask
/// queue). Prior to the invocation, the test framework will render only the
/// [Element] to be compared on the screen.
///
/// See also:
///
/// * [GoldenFileComparator] for the comparator to be used when the test is
/// not running in a web browser.
/// * [DefaultWebGoldenComparator] for the default [WebGoldenComparator]
/// implementation for `flutter test`.
/// * [matchesGoldenFile], the function from [flutter_test] that invokes the
/// comparator.
abstract class WebGoldenComparator {
/// Compares the rendered pixels of [element] of size [size] that is being
/// rendered on the top left of the screen against the golden file identified
/// by [golden].
///
/// The returned future completes with a boolean value that indicates whether
/// the pixels rendered on screen match the golden file's pixels.
///
/// In the case of comparison mismatch, the comparator may choose to throw a
/// [TestFailure] if it wants to control the failure message, often in the
/// form of a [ComparisonResult] that provides detailed information about the
/// mismatch.
///
/// The method by which [golden] is located and by which its bytes are loaded
/// is left up to the implementation class. For instance, some implementations
/// may load files from the local file system, whereas others may load files
/// over the network or from a remote repository.
Future<bool> compare(Element element, Size size, Uri golden);
/// Updates the golden file identified by [golden] with rendered pixels of
/// [element].
///
/// This will be invoked in lieu of [compare] when [autoUpdateGoldenFiles]
/// is `true` (which gets set automatically by the test framework when the
/// user runs `flutter test --update-goldens --platform=chrome`).
///
/// The method by which [golden] is located and by which its bytes are written
/// is left up to the implementation class.
Future<void> update(Uri golden, Element element, Size size);
/// Returns a new golden file [Uri] to incorporate any [version] number with
/// the [key].
///
/// The [version] is an optional int that can be used to differentiate
/// historical golden files.
///
/// Version numbers are used in golden file tests for package:flutter. You can
/// learn more about these tests [here](https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package:flutter).
Uri getTestUri(Uri key, int version) {
if (version == null)
return key;
final String keyString = key.toString();
final String extension = path.extension(keyString);
return Uri.parse(
keyString
.split(extension)
.join() + '.' + version.toString() + extension
);
}
}
/// Compares pixels against those of a golden image file.
///
/// This comparator is used as the backend for [matchesGoldenFile] when tests
/// are running in a web browser.
///
/// When using `flutter test --platform=chrome`, a comparator implemented by
/// [DefaultWebGoldenComparator] is used if no other comparator is specified. It
/// will send a request to the test server, which uses [goldenFileComparator]
/// for golden file compatison.
///
/// When using `flutter test --update-goldens`, the [DefaultWebGoldenComparator]
/// updates the files on disk to match the rendering.
///
/// When using `flutter run`, the default comparator
/// ([_TrivialWebGoldenComparator]) is used. It prints a message to the console
/// but otherwise does nothing. This allows tests to be developed visually on a
/// web browser.
///
/// Callers may choose to override the default comparator by setting this to a
/// custom comparator during test set-up (or using directory-level test
/// configuration). For example, some projects may wish to install a comparator
/// with tolerance levels for allowable differences.
///
/// See also:
///
/// * [flutter_test] for more information about how to configure tests at the
/// directory-level.
/// * [goldenFileComparator], the comparator used when tests are not running on
/// a web browser.
WebGoldenComparator get webGoldenComparator => _webGoldenComparator;
WebGoldenComparator _webGoldenComparator = const _TrivialWebGoldenComparator._();
set webGoldenComparator(WebGoldenComparator value) {
assert(value != null);
_webGoldenComparator = value;
}
/// The default [WebGoldenComparator] implementation for `flutter test`.
///
/// This comparator will send a request to the test server for golden comparison
@ -87,3 +197,23 @@ class DefaultWebGoldenComparator extends WebGoldenComparator {
await compare(element, size, golden);
}
}
class _TrivialWebGoldenComparator implements WebGoldenComparator {
const _TrivialWebGoldenComparator._();
@override
Future<bool> compare(Element element, Size size, Uri golden) {
print('Golden comparison requested for "$golden"; skipping...');
return Future<bool>.value(true);
}
@override
Future<void> update(Uri golden, Element element, Size size) {
throw StateError('webGoldenComparator has not been initialized');
}
@override
Uri getTestUri(Uri key, int version) {
return key;
}
}

1
packages/flutter_test/lib/src/_matchers_web.dart
View File

@ -11,6 +11,7 @@ import 'package:test_api/src/frontend/async_matcher.dart'; // ignore: implementa
// ignore: deprecated_member_use
import 'package:test_api/test_api.dart' hide TypeMatcher, isInstanceOf;
import '_goldens_web.dart';
import 'binding.dart';
import 'finders.dart';
import 'goldens.dart';

75
packages/flutter_test/lib/src/buffer_matcher.dart Normal file
View File

@ -0,0 +1,75 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:typed_data';
import 'package:test_api/src/frontend/async_matcher.dart'; // ignore: implementation_imports
// ignore: deprecated_member_use
import 'package:test_api/test_api.dart' show Description, TestFailure;
import 'goldens.dart';
/// Matcher created by [bufferMatchesGoldenFile].
class _BufferGoldenMatcher extends AsyncMatcher {
/// Creates an instance of [BufferGoldenMatcher]. Called by [bufferMatchesGoldenFile].
const _BufferGoldenMatcher(this.key, this.version);
/// The [key] to the golden image.
final Uri key;
/// The [version] of the golden image.
final int version;
@override
Future<String> matchAsync(dynamic item) async {
Uint8List buffer;
if (item is List<int>) {
buffer = Uint8List.fromList(item);
} else if (item is Future<List<int>>) {
buffer = Uint8List.fromList(await item);
} else {
throw 'Expected `List<int>` or `Future<List<int>>`, instead found: ${item.runtimeType}';
}
final Uri testNameUri = goldenFileComparator.getTestUri(key, version);
if (autoUpdateGoldenFiles) {
await goldenFileComparator.update(testNameUri, buffer);
return null;
}
try {
final bool success = await goldenFileComparator.compare(buffer, testNameUri);
return success ? null : 'does not match';
} on TestFailure catch (ex) {
return ex.message;
}
}
@override
Description describe(Description description) {
final Uri testNameUri = goldenFileComparator.getTestUri(key, version);
return description.add('Byte buffer matches golden image "$testNameUri"');
}
}
/// Asserts that a [Future<List<int>>], or [List<int] matches the
/// golden image file identified by [key], with an optional [version] number.
///
/// The [key] is the [String] representation of a URL.
///
/// The [version] is a number that can be used to differentiate historical
/// golden files. This parameter is optional.
///
/// {@tool snippet}
/// Sample invocations of [matchesGoldenFile].
///
/// ```dart
/// await expectLater(
/// const <int>[],
/// bufferMatchesGoldenFile('sample.png'),
/// );
/// ```
/// {@end-tool}
AsyncMatcher bufferMatchesGoldenFile(String key, {int version}) {
return _BufferGoldenMatcher(Uri.parse(key), version);
}

209
packages/flutter_test/lib/src/goldens.dart
View File

@ -3,12 +3,12 @@
// found in the LICENSE file.
import 'dart:async';
import 'dart:math' as math;
import 'dart:typed_data';
import 'package:flutter/foundation.dart';
import 'package:flutter/widgets.dart';
import 'package:meta/meta.dart';
import 'package:path/path.dart' as path;
import '_goldens_io.dart' if (dart.library.html) '_goldens_web.dart' as _goldens;
import 'package:image/image.dart';
/// Compares image pixels against a golden image file.
///
@ -98,7 +98,77 @@ abstract class GoldenFileComparator {
/// Returns a [ComparisonResult] to describe the pixel differential of the
/// [test] and [master] image bytes provided.
static ComparisonResult compareLists(List<int> test, List<int> master) {
return _goldens.compareLists(test, master);
if (identical(test, master))
return ComparisonResult(passed: true);
if (test == null || master == null || test.isEmpty || master.isEmpty) {
return ComparisonResult(
passed: false,
error: 'Pixel test failed, null image provided.',
);
}
final Image testImage = decodePng(test);
final Image masterImage = decodePng(master);
assert(testImage != null);
assert(masterImage != null);
final int width = testImage.width;
final int height = testImage.height;
if (width != masterImage.width || height != masterImage.height) {
return ComparisonResult(
passed: false,
error: 'Pixel test failed, image sizes do not match.\n'
'Master Image: ${masterImage.width} X ${masterImage.height}\n'
'Test Image: ${testImage.width} X ${testImage.height}',
);
}
int pixelDiffCount = 0;
final int totalPixels = width * height;
final Image invertedMaster = invert(Image.from(masterImage));
final Image invertedTest = invert(Image.from(testImage));
final Map<String, Image> diffs = <String, Image>{
'masterImage' : masterImage,
'testImage' : testImage,
'maskedDiff' : Image.from(testImage),
'isolatedDiff' : Image(width, height),
};
for (int x = 0; x < width; x++) {
for (int y = 0; y < height; y++) {
final int testPixel = testImage.getPixel(x, y);
final int masterPixel = masterImage.getPixel(x, y);
final int diffPixel = (getRed(testPixel) - getRed(masterPixel)).abs()
+ (getGreen(testPixel) - getGreen(masterPixel)).abs()
+ (getBlue(testPixel) - getBlue(masterPixel)).abs()
+ (getAlpha(testPixel) - getAlpha(masterPixel)).abs();
if (diffPixel != 0 ) {
final int invertedMasterPixel = invertedMaster.getPixel(x, y);
final int invertedTestPixel = invertedTest.getPixel(x, y);
final int maskPixel = math.max(invertedMasterPixel, invertedTestPixel);
diffs['maskedDiff'].setPixel(x, y, maskPixel);
diffs['isolatedDiff'].setPixel(x, y, maskPixel);
pixelDiffCount++;
}
}
}
if (pixelDiffCount > 0) {
return ComparisonResult(
passed: false,
error: 'Pixel test failed, '
'${((pixelDiffCount/totalPixels) * 100).toStringAsFixed(2)}% '
'diff detected.',
diffs: diffs,
);
}
return ComparisonResult(passed: true);
}
}
@ -135,115 +205,6 @@ set goldenFileComparator(GoldenFileComparator value) {
_goldenFileComparator = value;
}
/// Compares image pixels against a golden image file.
///
/// Instances of this comparator will be used as the backend for
/// [matchesGoldenFile] when tests are running on Flutter Web, and will usually
/// implemented by deferring the screenshot taking and image comparison to a
/// test server.
///
/// Instances of this comparator will be invoked by the test framework in the
/// [TestWidgetsFlutterBinding.runAsync] zone and are thus not subject to the
/// fake async constraints that are normally imposed on widget tests (i.e. the
/// need or the ability to call [WidgetTester.pump] to advance the microtask
/// queue). Prior to the invocation, the test framework will render only the
/// [Element] to be compared on the screen.
///
/// See also:
///
/// * [GoldenFileComparator] for the comparator to be used when the test is
/// not running in a web browser.
/// * [DefaultWebGoldenComparator] for the default [WebGoldenComparator]
/// implementation for `flutter test`.
/// * [matchesGoldenFile], the function from [flutter_test] that invokes the
/// comparator.
abstract class WebGoldenComparator {
/// Compares the rendered pixels of [element] of size [size] that is being
/// rendered on the top left of the screen against the golden file identified
/// by [golden].
///
/// The returned future completes with a boolean value that indicates whether
/// the pixels rendered on screen match the golden file's pixels.
///
/// In the case of comparison mismatch, the comparator may choose to throw a
/// [TestFailure] if it wants to control the failure message, often in the
/// form of a [ComparisonResult] that provides detailed information about the
/// mismatch.
///
/// The method by which [golden] is located and by which its bytes are loaded
/// is left up to the implementation class. For instance, some implementations
/// may load files from the local file system, whereas others may load files
/// over the network or from a remote repository.
Future<bool> compare(Element element, Size size, Uri golden);
/// Updates the golden file identified by [golden] with rendered pixels of
/// [element].
///
/// This will be invoked in lieu of [compare] when [autoUpdateGoldenFiles]
/// is `true` (which gets set automatically by the test framework when the
/// user runs `flutter test --update-goldens --platform=chrome`).
///
/// The method by which [golden] is located and by which its bytes are written
/// is left up to the implementation class.
Future<void> update(Uri golden, Element element, Size size);
/// Returns a new golden file [Uri] to incorporate any [version] number with
/// the [key].
///
/// The [version] is an optional int that can be used to differentiate
/// historical golden files.
///
/// Version numbers are used in golden file tests for package:flutter. You can
/// learn more about these tests [here](https://github.com/flutter/flutter/wiki/Writing-a-golden-file-test-for-package:flutter).
Uri getTestUri(Uri key, int version) {
if (version == null)
return key;
final String keyString = key.toString();
final String extension = path.extension(keyString);
return Uri.parse(
keyString
.split(extension)
.join() + '.' + version.toString() + extension
);
}
}
/// Compares pixels against those of a golden image file.
///
/// This comparator is used as the backend for [matchesGoldenFile] when tests
/// are running in a web browser.
///
/// When using `flutter test --platform=chrome`, a comparator implemented by
/// [DefaultWebGoldenComparator] is used if no other comparator is specified. It
/// will send a request to the test server, which uses [goldenFileComparator]
/// for golden file compatison.
///
/// When using `flutter test --update-goldens`, the [DefaultWebGoldenComparator]
/// updates the files on disk to match the rendering.
///
/// When using `flutter run`, the default comparator
/// ([_TrivialWebGoldenComparator]) is used. It prints a message to the console
/// but otherwise does nothing. This allows tests to be developed visually on a
/// web browser.
///
/// Callers may choose to override the default comparator by setting this to a
/// custom comparator during test set-up (or using directory-level test
/// configuration). For example, some projects may wish to install a comparator
/// with tolerance levels for allowable differences.
///
/// See also:
///
/// * [flutter_test] for more information about how to configure tests at the
/// directory-level.
/// * [goldenFileComparator], the comparator used when tests are not running on
/// a web browser.
WebGoldenComparator get webGoldenComparator => _webGoldenComparator;
WebGoldenComparator _webGoldenComparator = const _TrivialWebGoldenComparator._();
set webGoldenComparator(WebGoldenComparator value) {
assert(value != null);
_webGoldenComparator = value;
}
/// Whether golden files should be automatically updated during tests rather
/// than compared to the image bytes recorded by the tests.
///
@ -280,7 +241,7 @@ class TrivialComparator implements GoldenFileComparator {
@override
Future<bool> compare(Uint8List imageBytes, Uri golden) {
debugPrint('Golden file comparison requested for "$golden"; skipping...');
print('Golden file comparison requested for "$golden"; skipping...');
return Future<bool>.value(true);
}
@ -295,26 +256,6 @@ class TrivialComparator implements GoldenFileComparator {
}
}
class _TrivialWebGoldenComparator implements WebGoldenComparator {
const _TrivialWebGoldenComparator._();
@override
Future<bool> compare(Element element, Size size, Uri golden) {
debugPrint('Golden comparison requested for "$golden"; skipping...');
return Future<bool>.value(true);
}
@override
Future<void> update(Uri golden, Element element, Size size) {
throw StateError('webGoldenComparator has not been initialized');
}
@override
Uri getTestUri(Uri key, int version) {
return key;
}
}
/// The result of a pixel comparison test.
///
/// The [ComparisonResult] will always indicate if a test has [passed]. The