github/flutter
7
0
Fork 0
mirror of https://github.com/flutter/flutter.git synced 2025-06-03 00:51:18 +00:00
Code Issues Packages Projects Releases Wiki Activity

Implement golden-file matching for integration_test on Android and iOS devices (#160484)

Browse Source 
Work towards https://github.com/flutter/flutter/issues/143299.
Work towards https://github.com/flutter/flutter/issues/160043.

---

This PR implements, end-to-end, support for `matchesGoldenFile` when (a)
running with `package:integration_test` (b) on a device, such as an
Android emulator, Android device, iOS simulator, or iOS device, where
the _runner_ of a test file does not have process and local-file system
access.

There are multiple parts to this PR; I could make it smaller than 1K
lines, but the bulk of that is tests, and it would mean landing PRs that
are incomplete and unused, which does not seem useful - so instead here
is a quick overview of the PR's contents - questions/feedback welcome,
and I am willing to break code out or land incremental refactors if
requested.

1. Augmented `flutter_platform.dart` (used for iOS and Android), similar
to
[`flutter_web_platform.dart`](1398dc7eec/packages/flutter_tools/lib/src/test/flutter_web_platform.dart (L117-L128)),
now creates and uses
[`test_golden_comparator.dart`](https://github.com/flutter/flutter/blob/master/packages/flutter_tools/lib/src/test/test_golden_comparator.dart)
to proxy calls (coming from the VM service protocol) for golden-file
updates and comparisons to a `flutter_tester` process. A full
explanation of how (or why) it works this way is too hard to include
here, but see https://github.com/flutter/flutter/pull/160215 for more
details.
1. Added `VmServiceProxyGoldenFileComparator`, which is a currently
unused (outside of a single e2e test) comparator that forwards calls to
`compare` and `update` to the VM service protocol (of which, the other
side of this is implemented above, in `flutter_platform.dart`. The idea
is that this comparator would be used automatically when running in an
integration test on a device that requires it (similar to how web works
today), but that is **not** wired up yet and requires additional work in
`flutter_tools`.
1. Added two unit tests (of both the client and server), and a full
e2e-test using it to run `matchesGoldenFile`.
This commit is contained in:
Matan Lurey 2024-12-28 11:48:20 -08:00 committed by GitHub
parent b15625ca92
commit 4cd0e33013
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
9 changed files with 958 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

20
packages/flutter_tools/bin/fuchsia_tester.dart
View File

@ -151,19 +151,19 @@ Future<void> run(List<String> args) async {
}
// TODO(dnfield): This should be injected.
final BuildInfo buildInfo = BuildInfo(
BuildMode.debug,
'',
treeShakeIcons: false,
packageConfigPath: globals.fs.path.normalize(
globals.fs.path.absolute(argResults[_kOptionPackages] as String),
),
);
exitCode = await const FlutterTestRunner().runTests(
const TestWrapper(),
tests.keys.map(Uri.file).toList(),
debuggingOptions: DebuggingOptions.enabled(
BuildInfo(
BuildMode.debug,
'',
treeShakeIcons: false,
packageConfigPath: globals.fs.path.normalize(
globals.fs.path.absolute(argResults[_kOptionPackages] as String),
),
),
),
debuggingOptions: DebuggingOptions.enabled(buildInfo),
buildInfo: buildInfo,
watcher: collector,
enableVmService: collector != null,
precompiledDillFiles: tests,

2
packages/flutter_tools/lib/src/device.dart
View File

@ -1008,7 +1008,7 @@ class DebuggingOptions {
startPaused = false,
dartFlags = '',
disableServiceAuthCodes = false,
enableDds = true,
enableDds = false,
cacheStartupProfile = false,
enableSoftwareRendering = false,
skiaDeterministicRendering = false,

99
packages/flutter_tools/lib/src/test/flutter_platform.dart
View File

@ -3,16 +3,20 @@
// found in the LICENSE file.
import 'dart:async';
import 'dart:typed_data';
import 'package:meta/meta.dart';
import 'package:package_config/package_config.dart';
import 'package:process/process.dart';
import 'package:stream_channel/stream_channel.dart';
import 'package:test_core/src/platform.dart'; // ignore: implementation_imports
import 'package:vm_service/vm_service.dart';
import '../base/async_guard.dart';
import '../base/common.dart';
import '../base/file_system.dart';
import '../base/io.dart';
import '../base/logger.dart';
import '../base/process.dart';
import '../build_info.dart';
import '../cache.dart';
@ -32,6 +36,7 @@ import 'integration_test_device.dart';
import 'test_compiler.dart';
import 'test_config.dart';
import 'test_device.dart';
import 'test_golden_comparator.dart';
import 'test_time_recorder.dart';
import 'watcher.dart';
@ -53,6 +58,10 @@ FlutterPlatform installHook({
TestWrapper testWrapper = const TestWrapper(),
required String shellPath,
required DebuggingOptions debuggingOptions,
required BuildInfo buildInfo,
required FileSystem fileSystem,
required Logger logger,
required ProcessManager processManager,
TestWatcher? watcher,
bool enableVmService = false,
bool machine = false,
@ -69,7 +78,6 @@ FlutterPlatform installHook({
String? integrationTestUserIdentifier,
TestTimeRecorder? testTimeRecorder,
TestCompilerNativeAssetsBuilder? nativeAssetsBuilder,
BuildInfo? buildInfo,
}) {
assert(
enableVmService ||
@ -101,6 +109,9 @@ FlutterPlatform installHook({
testTimeRecorder: testTimeRecorder,
nativeAssetsBuilder: nativeAssetsBuilder,
buildInfo: buildInfo,
fileSystem: fileSystem,
logger: logger,
processManager: processManager,
);
platformPluginRegistration(platform);
return platform;
@ -289,6 +300,10 @@ class FlutterPlatform extends PlatformPlugin {
FlutterPlatform({
required this.shellPath,
required this.debuggingOptions,
required this.buildInfo,
required this.logger,
required FileSystem fileSystem,
required ProcessManager processManager,
this.watcher,
this.enableVmService,
this.machine,
@ -304,9 +319,19 @@ class FlutterPlatform extends PlatformPlugin {
this.integrationTestUserIdentifier,
this.testTimeRecorder,
this.nativeAssetsBuilder,
this.buildInfo,
this.shutdownHooks,
});
}) {
_testGoldenComparator = TestGoldenComparator(
flutterTesterBinPath: shellPath,
compilerFactory:
() =>
compiler ??
TestCompiler(buildInfo, flutterProject, testTimeRecorder: testTimeRecorder),
fileSystem: fileSystem,
logger: logger,
processManager: processManager,
);
}
final String shellPath;
final DebuggingOptions debuggingOptions;
@ -323,7 +348,8 @@ class FlutterPlatform extends PlatformPlugin {
final String? icudtlPath;
final TestTimeRecorder? testTimeRecorder;
final TestCompilerNativeAssetsBuilder? nativeAssetsBuilder;
final BuildInfo? buildInfo;
final BuildInfo buildInfo;
final Logger logger;
final ShutdownHooks? shutdownHooks;
/// The device to run the test on for Integration Tests.
@ -332,6 +358,7 @@ class FlutterPlatform extends PlatformPlugin {
/// Tester; otherwise it will run as a Integration Test on this device.
final Device? integrationTestDevice;
bool get _isIntegrationTest => integrationTestDevice != null;
late final TestGoldenComparator _testGoldenComparator;
final String? integrationTestUserIdentifier;
@ -375,7 +402,9 @@ class FlutterPlatform extends PlatformPlugin {
return controller.suite;
}
StreamChannel<dynamic> loadChannel(String path, SuitePlatform platform) {
/// Used as an implementation detail for [load]ing a test suite.
@visibleForTesting
StreamChannel<Object?> loadChannel(String path, SuitePlatform platform) {
if (_testCount > 0) {
// Fail if there will be a port conflict.
if (debuggingOptions.hostVmServicePort != null) {
@ -481,15 +510,64 @@ class FlutterPlatform extends PlatformPlugin {
);
}
void _handleStartedDevice(Uri? uri, int testCount) {
void _handleStartedDevice({required Uri? uri, required int testCount, required String testPath}) {
if (uri != null) {
globals.printTrace('test $testCount: VM Service uri is available at $uri');
if (_isIntegrationTest) {
_listenToVmServiceForGoldens(uri: uri, testPath: testPath);
}
} else {
globals.printTrace('test $testCount: VM Service uri is not available');
}
watcher?.handleStartedDevice(uri);
}
static const String _kEventName = 'integration_test.VmServiceProxyGoldenFileComparator';
static const String _kExtension = 'ext.$_kEventName';
Future<void> _listenToVmServiceForGoldens({required Uri uri, required String testPath}) async {
final Uri goldensBaseUri = Uri.parse(testPath);
final FlutterVmService vmService = await connectToVmService(uri, logger: logger);
final IsolateRef testAppIsolate = await vmService.findExtensionIsolate(_kExtension);
await vmService.service.streamListen(_kEventName);
vmService.service.onEvent(_kEventName).listen((Event e) async {
if (!const <String>['compare', 'update'].contains(e.extensionKind)) {
throw StateError('Unexpected command: "${e.extensionKind}".');
}
final Map<String, Object?>? data = e.extensionData?.data;
if (data == null) {
throw StateError('Expected VM service data, but got null.');
}
final int id = data['id']! as int;
final Uri relativePath = Uri.parse(data['path']! as String);
final Uint8List bytes = base64.decode(data['bytes']! as String);
final Map<String, Object?> args;
if (e.extensionKind == 'update') {
switch (await _testGoldenComparator.update(goldensBaseUri, bytes, relativePath)) {
case TestGoldenUpdateDone():
args = <String, Object?>{'result': true};
case TestGoldenUpdateError(error: final String error):
args = <String, Object?>{'error': error};
}
} else {
switch (await _testGoldenComparator.compare(goldensBaseUri, bytes, relativePath)) {
case TestGoldenComparisonDone(matched: final bool matched):
args = <String, Object?>{'result': matched};
case TestGoldenComparisonError(error: final String error):
args = <String, Object?>{'error': error};
}
}
await vmService.callMethodWrapper(
_kExtension,
isolateId: testAppIsolate.id,
args: <String, Object?>{'id': id, ...args},
);
});
}
Future<_AsyncError?> _startTest(
String testPath,
StreamChannel<dynamic> testHarnessChannel,
@ -625,7 +703,11 @@ class FlutterPlatform extends PlatformPlugin {
// This future may depend on [_handleStartedDevice] having been called
remoteChannelCompleter.future,
testDevice.vmServiceUri.then<void>((Uri? processVmServiceUri) {
_handleStartedDevice(processVmServiceUri, ourTestCount);
_handleStartedDevice(
uri: processVmServiceUri,
testCount: ourTestCount,
testPath: testPath,
);
}),
],
// If [remoteChannelCompleter.future] errors, we may never get the
@ -727,8 +809,7 @@ class FlutterPlatform extends PlatformPlugin {
testUrl: testUrl,
testConfigFile: findTestConfigFile(globals.fs.file(testUrl), globals.logger),
// This MUST be a file URI.
packageConfigUri:
buildInfo != null ? globals.fs.path.toUri(buildInfo!.packageConfigPath) : null,
packageConfigUri: globals.fs.path.toUri(buildInfo.packageConfigPath),
host: host!,
updateGoldens: updateGoldens!,
flutterTestDep: packageConfig['flutter_test'] != null,

7
packages/flutter_tools/lib/src/test/runner.dart
View File

@ -63,7 +63,7 @@ abstract class FlutterTestRunner {
String? integrationTestUserIdentifier,
TestTimeRecorder? testTimeRecorder,
TestCompilerNativeAssetsBuilder? nativeAssetsBuilder,
BuildInfo? buildInfo,
required BuildInfo buildInfo,
});
/// Runs tests using the experimental strategy of spawning each test in a
@ -130,7 +130,7 @@ class _FlutterTestRunnerImpl implements FlutterTestRunner {
String? integrationTestUserIdentifier,
TestTimeRecorder? testTimeRecorder,
TestCompilerNativeAssetsBuilder? nativeAssetsBuilder,
BuildInfo? buildInfo,
required BuildInfo buildInfo,
}) async {
// Configure package:test to use the Flutter engine for child processes.
final String flutterTesterBinPath = globals.artifacts!.getArtifactPath(Artifact.flutterTester);
@ -236,6 +236,9 @@ class _FlutterTestRunnerImpl implements FlutterTestRunner {
testTimeRecorder: testTimeRecorder,
nativeAssetsBuilder: nativeAssetsBuilder,
buildInfo: buildInfo,
fileSystem: globals.fs,
logger: globals.logger,
processManager: globals.processManager,
);
try {

284
packages/flutter_tools/test/general.shard/flutter_platform_test.dart
View File

@ -2,8 +2,12 @@
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:async';
import 'dart:io' as io;
import 'package:file/memory.dart';
import 'package:flutter_tools/src/application_package.dart';
import 'package:flutter_tools/src/artifacts.dart';
import 'package:flutter_tools/src/base/file_system.dart';
import 'package:flutter_tools/src/base/io.dart';
import 'package:flutter_tools/src/base/logger.dart';
@ -14,8 +18,12 @@ import 'package:flutter_tools/src/flutter_manifest.dart';
import 'package:flutter_tools/src/globals.dart' as globals;
import 'package:flutter_tools/src/project.dart';
import 'package:flutter_tools/src/test/flutter_platform.dart';
import 'package:flutter_tools/src/test/test_compiler.dart';
import 'package:flutter_tools/src/vmservice.dart';
import 'package:stream_channel/stream_channel.dart';
import 'package:test/fake.dart';
import 'package:test_core/backend.dart';
import 'package:vm_service/src/vm_service.dart';
import '../src/common.dart';
import '../src/context.dart';
@ -44,6 +52,10 @@ void main() {
shellPath: '/',
debuggingOptions: DebuggingOptions.enabled(BuildInfo.debug, hostVmServicePort: 1234),
enableVmService: false,
buildInfo: BuildInfo.debug,
fileSystem: fileSystem,
processManager: FakeProcessManager.empty(),
logger: BufferLogger.test(),
);
flutterPlatform.loadChannel('test1.dart', fakeSuitePlatform);
@ -67,6 +79,10 @@ void main() {
shellPath: '/',
precompiledDillPath: 'example.dill',
enableVmService: false,
buildInfo: BuildInfo.debug,
fileSystem: fileSystem,
processManager: FakeProcessManager.empty(),
logger: BufferLogger.test(),
);
flutterPlatform.loadChannel('test1.dart', fakeSuitePlatform);
@ -93,6 +109,10 @@ void main() {
flutterProject: _FakeFlutterProject(),
host: InternetAddress.anyIPv4,
updateGoldens: false,
buildInfo: BuildInfo.debug,
fileSystem: fileSystem,
processManager: FakeProcessManager.empty(),
logger: BufferLogger.test(),
);
await expectLater(
@ -133,6 +153,10 @@ void main() {
host: InternetAddress.anyIPv4,
updateGoldens: false,
shutdownHooks: shutdownHooks,
buildInfo: BuildInfo.debug,
fileSystem: fileSystem,
processManager: FakeProcessManager.empty(),
logger: BufferLogger.test(),
);
await expectLater(
@ -156,6 +180,10 @@ void main() {
() => installHook(
shellPath: 'abc',
debuggingOptions: DebuggingOptions.enabled(BuildInfo.debug, startPaused: true),
buildInfo: BuildInfo.debug,
fileSystem: fileSystem,
processManager: FakeProcessManager.empty(),
logger: BufferLogger.test(),
),
throwsAssertionError,
);
@ -168,6 +196,10 @@ void main() {
startPaused: true,
hostVmServicePort: 123,
),
buildInfo: BuildInfo.debug,
fileSystem: fileSystem,
processManager: FakeProcessManager.empty(),
logger: BufferLogger.test(),
),
throwsAssertionError,
);
@ -193,6 +225,10 @@ void main() {
platformPluginRegistration: (FlutterPlatform platform) {
capturedPlatform = platform;
},
buildInfo: BuildInfo.debug,
fileSystem: fileSystem,
processManager: FakeProcessManager.empty(),
logger: BufferLogger.test(),
);
expect(identical(capturedPlatform, flutterPlatform), equals(true));
@ -247,6 +283,254 @@ void main() {
});
});
});
group('proxies goldenFileComparator using the VM service driver', () {
late SuitePlatform fakeSuitePlatform;
late MemoryFileSystem fileSystem;
late Artifacts artifacts;
late FlutterProject flutterProject;
late FakeProcessManager processManager;
late BufferLogger logger;
late TestCompiler testCompiler;
late _FakeFlutterVmService flutterVmService;
late Completer<void> testCompleter;
setUp(() {
fakeSuitePlatform = SuitePlatform(Runtime.vm);
fileSystem = MemoryFileSystem.test();
fileSystem.file('.dart_tool/package_config.json')
..createSync(recursive: true)
..writeAsStringSync('{"configVersion":2,"packages":[]}');
artifacts = Artifacts.test(fileSystem: fileSystem);
flutterProject = FlutterProject.fromDirectoryTest(fileSystem.systemTempDirectory);
testCompleter = Completer<void>();
processManager = FakeProcessManager.empty();
logger = BufferLogger.test();
testCompiler = _FakeTestCompiler();
flutterVmService = _FakeFlutterVmService();
});
tearDown(() {
printOnFailure(logger.errorText);
});
void addFlutterTesterDeviceExpectation() {
processManager.addCommand(
FakeCommand(
command: const <String>[
'flutter_tester',
'--disable-vm-service',
'--enable-checked-mode',
'--verify-entry-points',
'--enable-software-rendering',
'--skia-deterministic-rendering',
'--enable-dart-profiling',
'--non-interactive',
'--use-test-fonts',
'--disable-asset-fonts',
'--packages=.dart_tool/package_config.json',
'',
],
exitCode: -9,
completer: testCompleter,
),
);
}
testUsingContext(
'should not listen in a non-integration test',
() async {
addFlutterTesterDeviceExpectation();
const Device? notAnIntegrationTest = null;
final FlutterPlatform flutterPlatform = FlutterPlatform(
debuggingOptions: DebuggingOptions.disabled(BuildInfo.debug),
shellPath: 'flutter_tester',
enableVmService: false,
// ignore: avoid_redundant_argument_values
integrationTestDevice: notAnIntegrationTest,
flutterProject: flutterProject,
host: InternetAddress.anyIPv4,
updateGoldens: false,
buildInfo: BuildInfo.debug,
fileSystem: fileSystem,
processManager: processManager,
logger: BufferLogger.test(),
);
flutterPlatform.compiler = testCompiler;
// Simulate the test immediately completing.
testCompleter.complete();
final StreamChannel<Object?> channel = flutterPlatform.loadChannel(
'test1.dart',
fakeSuitePlatform,
);
// Without draining, the sink will never complete.
unawaited(channel.stream.drain<void>());
await expectLater(channel.sink.done, completes);
},
overrides: <Type, Generator>{
FileSystem: () => fileSystem,
ProcessManager: () => processManager,
Logger: () => logger,
VMServiceConnector: () => (_) => throw UnimplementedError(),
},
);
// This is not a complete test of all the possible cases supported by the
// golden-file integration, which is a complex multi-process implementation
// that lives across multiple packages and files.
//
// Instead, this is a unit-test based smoke test that the overall flow works
// as expected to give a quicker turn-around signal that either the test
// should be updated, or the process was broken; run an integration_test on
// an Android or iOS device or emulator/simulator that takes screenshots
// and compares them with matchesGoldenFile for a full e2e-test of the
// entire workflow.
testUsingContext(
'should listen in an integration test',
() async {
processManager.addCommand(
const FakeCommand(
command: <String>[
'flutter_tester',
'--disable-vm-service',
'--non-interactive',
'--packages=.dart_tool/package_config.json',
'',
],
stdout: '{"success": true}\n',
),
);
addFlutterTesterDeviceExpectation();
final FlutterPlatform flutterPlatform = FlutterPlatform(
debuggingOptions: DebuggingOptions.disabled(BuildInfo.debug),
shellPath: 'flutter_tester',
enableVmService: false,
flutterProject: flutterProject,
integrationTestDevice: _WorkingDevice(),
host: InternetAddress.anyIPv4,
updateGoldens: false,
buildInfo: BuildInfo.debug,
fileSystem: fileSystem,
processManager: processManager,
logger: BufferLogger.test(),
);
flutterPlatform.compiler = testCompiler;
final StreamChannel<Object?> channel = flutterPlatform.loadChannel(
'test1.dart',
fakeSuitePlatform,
);
// Responds to update events.
flutterVmService.service.onExtensionEventController.add(
Event(
extensionData: ExtensionData.parse(<String, Object?>{
'id': 1,
'path': 'foo',
'bytes': '',
}),
extensionKind: 'update',
),
);
// Wait for tiny async tasks to complete.
await pumpEventQueue();
await flutterVmService.service.onExtensionEventController.close();
final (String event, String? isolateId, Map<String, Object?>? data) =
flutterVmService.callMethodWrapperInvocation!;
expect(event, 'ext.integration_test.VmServiceProxyGoldenFileComparator');
expect(isolateId, null);
expect(data, <String, Object?>{'id': 1, 'result': true});
// Without draining, the sink will never complete.
unawaited(channel.stream.drain<void>());
// Allow the test to finish.
testCompleter.complete();
await expectLater(channel.sink.done, completes);
},
overrides: <Type, Generator>{
FileSystem: () => fileSystem,
ProcessManager: () => processManager,
Logger: () => logger,
VMServiceConnector:
() =>
(
Uri httpUri, {
ReloadSources? reloadSources,
Restart? restart,
CompileExpression? compileExpression,
GetSkSLMethod? getSkSLMethod,
FlutterProject? flutterProject,
PrintStructuredErrorLogMethod? printStructuredErrorLogMethod,
io.CompressionOptions? compression,
Device? device,
Logger? logger,
}) async => flutterVmService,
ApplicationPackageFactory: _FakeApplicationPackageFactory.new,
Artifacts: () => artifacts,
},
);
});
}
class _FakeFlutterVmService extends Fake implements FlutterVmService {
@override
Future<IsolateRef> findExtensionIsolate(String extensionName) async {
return IsolateRef();
}
@override
final _FakeVmService service = _FakeVmService();
(String, String?, Map<String, Object?>?)? callMethodWrapperInvocation;
@override
Future<Response?> callMethodWrapper(
String method, {
String? isolateId,
Map<String, Object?>? args,
}) async {
callMethodWrapperInvocation = (method, isolateId, args);
return Response();
}
}
class _FakeVmService extends Fake implements VmService {
String? lastStreamListenId;
@override
Future<Success> streamListen(String streamId) async {
lastStreamListenId = streamId;
return Success();
}
final StreamController<Event> onExtensionEventController = StreamController<Event>();
@override
Stream<Event> get onExtensionEvent => const Stream<Event>.empty();
@override
Stream<Event> onEvent(String streamId) {
return onExtensionEventController.stream;
}
@override
Future<void> get onDone => onExtensionEventController.done;
}
class _FakeTestCompiler extends Fake implements TestCompiler {
@override
Future<String?> compile(Uri mainDart) async => '';
}
class _UnstartableDevice extends Fake implements Device {

43
packages/integration_test/example/integration_test/matches_golden_test.dart Normal file
View File

@ -0,0 +1,43 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
@Tags(<String>['reduced-test-set'])
library;
import 'package:flutter/material.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:integration_test/integration_test.dart';
import 'package:integration_test_example/main.dart' as app;
/// To run:
///
/// ```sh
/// # Be in this directory
/// cd dev/packages/integration_test/example
///
/// flutter test integration_test/matches_golden_test.dart
/// ```
///
/// To run on a particular device, see `flutter -d`.
void main() {
IntegrationTestWidgetsFlutterBinding.ensureInitialized();
// TODO(matanlurey): Make this automatic as part of the bootstrap.
VmServiceProxyGoldenFileComparator.useIfRunningOnDevice();
testWidgets('can use matchesGoldenFile with integration_test', (WidgetTester tester) async {
// Build our app and trigger a frame.
app.main();
// TODO(matanlurey): Is this necessary?
await tester.pumpAndSettle();
// Take a screenshot.
await expectLater(
find.byType(MaterialApp),
matchesGoldenFile('integration_test_matches_golden_file.png'),
);
});
}

2
packages/integration_test/lib/integration_test.dart
View File

@ -24,6 +24,8 @@ import 'src/callback.dart' as driver_actions;
import 'src/channel.dart';
import 'src/extension.dart';
export 'src/vm_service_golden_client.dart';
const String _success = 'success';
/// Whether results should be reported to the native side over the method

226
packages/integration_test/lib/src/vm_service_golden_client.dart Normal file
View File

@ -0,0 +1,226 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
// Examples can assume:
// import 'dart:developer' as dev;
import 'dart:async';
import 'dart:convert';
import 'dart:developer' as dev;
import 'dart:io' as io;
import 'package:flutter/foundation.dart';
import 'package:flutter_test/flutter_test.dart';
import 'package:meta/meta.dart';
/// Compares image pixels against a golden image file on the host system.
///
/// This comparator will send a request, using the VM service protocol, to a
/// host script (i.e. the _driver_ script, running in a Dart VM on the host
/// desktop OS), which will then forward the comparison to a concrete
/// [GoldenFileComparator].
///
/// To use, run [useIfRunningOnDevice] in the `main()` of a test file or similar:
///
/// ```dart
/// import 'package:integration_test/integration_test.dart';
///
/// void main() {
/// VmServiceProxyGoldenFileComparator.useIfRunningOnDevice();
///
/// // Actual tests and such below.
/// }
/// ```
///
/// When either [compare] or [update] is called, the following event is sent
/// with [dev.postEvent]:
///
/// ```dart
/// dev.postEvent('compare' /* or 'update' */, <String, Object?>{
/// 'id': 1001, // a valid unique integer, often incrementing;
/// 'path': 'path/to/image.png', // golden key created by matchesGoldenFile;
/// 'bytes': '...base64encoded', // base64 encoded bytes representing the current image.
/// }, stream: 'integration_test.VmServiceProxyGoldenFileComparator');
/// ```
///
/// The comparator expects a response at the service extension
/// `ext.integration_test.VmServiceProxyGoldenFileComparator` that is either
/// of the following formats:
///
/// ```dart
/// <String, Object?>{
/// 'error': 'Description of why the operation failed'
/// }
/// ```
///
/// or:
///
/// ```dart
/// <String, Object?>{
/// 'result': true /* or possibly false, in the case of 'compare' calls */
/// }
/// ```
///
/// See also:
///
/// * [matchesGoldenFile], the function that invokes the comparator.
@experimental
final class VmServiceProxyGoldenFileComparator extends GoldenFileComparator {
VmServiceProxyGoldenFileComparator._() : _postEvent = dev.postEvent {
dev.registerExtension(_kServiceName, (_, Map<String, String> parameters) {
return handleEvent(parameters);
});
}
/// Creates an instance of [VmServiceProxyGoldenFileComparator] for internal testing.
///
/// @nodoc
@visibleForTesting
VmServiceProxyGoldenFileComparator.forTesting(this._postEvent);
static bool get _isRunningOnHost {
if (kIsWeb) {
return false;
}
return !io.Platform.isAndroid && !io.Platform.isIOS;
}
static void _assertNotRunningOnFuchsia() {
if (!kIsWeb && io.Platform.isFuchsia) {
throw UnsupportedError('Golden testing with integration_test does not support Fuchsia.');
}
}
/// Conditionally sets [goldenFileComparator] to [VmServiceProxyGoldenFileComparator].
///
/// If running on a non-mobile non-web platform (i.e. desktop), this method has no effect.
static void useIfRunningOnDevice() {
if (_isRunningOnHost) {
return;
}
_assertNotRunningOnFuchsia();
goldenFileComparator = _kInstance;
}
static final GoldenFileComparator _kInstance = VmServiceProxyGoldenFileComparator._();
static const String _kServiceName = 'ext.$_kEventName';
static const String _kEventName = 'integration_test.VmServiceProxyGoldenFileComparator';
final void Function(String, Map<Object?, Object?>, {String stream}) _postEvent;
/// Handles the received method and parameters as an incoming event.
///
/// Each event is treated as if it were received by the Dart developer
/// extension protocol; this method is public only to be able to write unit
/// tests that do not have to bring up and use a VM service.
///
/// @nodoc
@visibleForTesting
Future<dev.ServiceExtensionResponse> handleEvent(Map<String, String> parameters) async {
// Treat the method as the ID number of the pending request.
final String? methodIdString = parameters['id'];
if (methodIdString == null) {
return dev.ServiceExtensionResponse.error(
dev.ServiceExtensionResponse.extensionError,
'Required parameter "id" not present in response.',
);
}
final int? methodId = int.tryParse(methodIdString);
if (methodId == null) {
return dev.ServiceExtensionResponse.error(
dev.ServiceExtensionResponse.extensionError,
'Required parameter "id" not a valid integer: "$methodIdString".',
);
}
final Completer<_Result>? completer = _pendingRequests[methodId];
if (completer == null) {
return dev.ServiceExtensionResponse.error(
dev.ServiceExtensionResponse.extensionError,
'No pending request with method ID "$methodIdString".',
);
}
assert(!completer.isCompleted, 'Can never occur, as the completer should be removed');
final String? error = parameters['error'];
if (error != null) {
completer.complete(_Failure(error));
return dev.ServiceExtensionResponse.result('{}');
}
final String? result = parameters['result'];
if (result == null) {
return dev.ServiceExtensionResponse.error(
dev.ServiceExtensionResponse.invalidParams,
'Required parameter "result" not present in response.',
);
}
if (bool.tryParse(result) case final bool result) {
completer.complete(_Success(result));
return dev.ServiceExtensionResponse.result('{}');
} else {
return dev.ServiceExtensionResponse.error(
dev.ServiceExtensionResponse.invalidParams,
'Required parameter "result" not a valid boolean: "$result".',
);
}
}
int _nextId = 0;
final Map<int, Completer<_Result>> _pendingRequests = <int, Completer<_Result>>{};
Future<_Result> _postAndWait(
Uint8List imageBytes,
Uri golden, {
required String operation,
}) async {
final int nextId = ++_nextId;
assert(!_pendingRequests.containsKey(nextId));
final Completer<_Result> completer = Completer<_Result>();
_postEvent(operation, <String, Object?>{
'id': nextId,
'path': '$golden',
'bytes': base64.encode(imageBytes),
}, stream: _kEventName);
_pendingRequests[nextId] = completer;
completer.future.whenComplete(() {
_pendingRequests.remove(nextId);
});
return completer.future;
}
@override
Future<bool> compare(Uint8List imageBytes, Uri golden) async {
return switch (await _postAndWait(imageBytes, golden, operation: 'compare')) {
_Success(:final bool result) => result,
_Failure(:final String error) => Future<bool>.error(error),
};
}
@override
Future<void> update(Uri golden, Uint8List imageBytes) async {
final _Result result = await _postAndWait(imageBytes, golden, operation: 'update');
if (result is _Failure) {
return Future<void>.error(result.error);
}
}
}
// These wrapper classes allow us to use a Completer to indicate both a failed
// response and a successful response, without making a call of completeError
// within handleEvent, which is difficult or impossible to use correctly because
// of the semantics of error zones.
//
// Of course, this is a private implementation detail, others are welcome to try
// an alternative approach that might simplify the code above, but it's probably
// not worth it.
sealed class _Result {}
final class _Success implements _Result {
_Success(this.result);
final bool result;
}
final class _Failure implements _Result {
_Failure(this.error);
final String error;
}

297
packages/integration_test/test/vm_service_golden_client_test.dart Normal file
View File

@ -0,0 +1,297 @@
// Copyright 2014 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:convert';
import 'dart:developer' as dev;
import 'dart:typed_data';
import 'package:flutter_test/flutter_test.dart';
import 'package:integration_test/integration_test.dart' as integration_test;
void main() {
setUp(() {
// Ensure that we reset to a throwing comparator by default.
goldenFileComparator = const _NullGoldenFileComparator();
});
group('useIfRunningOnDevice', () {
test('is skipped on the web', () {
integration_test.VmServiceProxyGoldenFileComparator.useIfRunningOnDevice();
expect(goldenFileComparator, isInstanceOf<_NullGoldenFileComparator>());
}, testOn: 'js');
test('is skipped on desktop platforms', () {
integration_test.VmServiceProxyGoldenFileComparator.useIfRunningOnDevice();
expect(goldenFileComparator, isInstanceOf<_NullGoldenFileComparator>());
}, testOn: 'windows || mac-os || linux');
test('is set on mobile platforms', () {
integration_test.VmServiceProxyGoldenFileComparator.useIfRunningOnDevice();
expect(
goldenFileComparator,
isInstanceOf<integration_test.VmServiceProxyGoldenFileComparator>(),
);
}, testOn: 'ios || android');
});
group('handleEvent', () {
late integration_test.VmServiceProxyGoldenFileComparator goldenFileComparator;
setUp(() {
goldenFileComparator = integration_test.VmServiceProxyGoldenFileComparator.forTesting(
(String operation, Map<Object?, Object?> params, {String stream = ''}) {},
);
});
test('"id" must be provided', () async {
final dev.ServiceExtensionResponse response = await goldenFileComparator.handleEvent(
<String, String>{'result': 'true'},
);
expect(response.errorDetail, contains('Required parameter "id" not present in response'));
});
test('"id" must be an integer', () async {
final dev.ServiceExtensionResponse response = await goldenFileComparator.handleEvent(
<String, String>{'id': 'not-an-integer', 'result': 'true'},
);
expect(
response.errorDetail,
stringContainsInOrder(<String>[
'Required parameter "id" not a valid integer',
'not-an-integer',
]),
);
});
test('"id" must match a pending request (never occurred)', () async {
final dev.ServiceExtensionResponse response = await goldenFileComparator.handleEvent(
<String, String>{'id': '12345', 'result': 'true'},
);
expect(
response.errorDetail,
stringContainsInOrder(<String>['No pending request with method ID', '12345']),
);
});
test('"id" must match a pending request (already occurred)', () async {
// This is based on an implementation detail of knowing how IDs are generated.
const int nextId = 1;
goldenFileComparator.update(Uri(path: 'some-file'), Uint8List(0));
dev.ServiceExtensionResponse response;
response = await goldenFileComparator.handleEvent(<String, String>{
'id': '$nextId',
'result': 'true',
});
expect(response.errorDetail, isNull);
response = await goldenFileComparator.handleEvent(<String, String>{
'id': '$nextId',
'result': 'true',
});
expect(
response.errorDetail,
stringContainsInOrder(<String>['No pending request with method ID', '1']),
);
});
test('requests that contain "error" completes it as an error', () async {
// This is based on an implementation detail of knowing how IDs are generated.
const int nextId = 1;
expect(
goldenFileComparator.compare(Uint8List(0), Uri(path: 'some-file')),
throwsA(contains('We did a bad')),
);
final dev.ServiceExtensionResponse response = await goldenFileComparator.handleEvent(
<String, String>{'id': '$nextId', 'error': 'We did a bad'},
);
expect(response.errorDetail, isNull);
expect(response.result, '{}');
});
test('requests that do not contain "error" return an empty response', () async {
// This is based on an implementation detail of knowing how IDs are generated.
const int nextId = 1;
goldenFileComparator.update(Uri(path: 'some-file'), Uint8List(0));
final dev.ServiceExtensionResponse response = await goldenFileComparator.handleEvent(
<String, String>{'id': '$nextId', 'result': 'true'},
);
expect(response.errorDetail, isNull);
expect(response.result, '{}');
});
test('"result" must be provided if "error" is omitted', () async {
// This is based on an implementation detail of knowing how IDs are generated.
const int nextId = 1;
goldenFileComparator.update(Uri(path: 'some-file'), Uint8List(0));
final dev.ServiceExtensionResponse response = await goldenFileComparator.handleEvent(
<String, String>{'id': '$nextId'},
);
expect(response.errorDetail, contains('Required parameter "result" not present in response'));
});
test('"result" must be a boolean', () async {
// This is based on an implementation detail of knowing how IDs are generated.
const int nextId = 1;
goldenFileComparator.update(Uri(path: 'some-file'), Uint8List(0));
final dev.ServiceExtensionResponse response = await goldenFileComparator.handleEvent(
<String, String>{'id': '$nextId', 'result': 'not-a-boolean'},
);
expect(
response.errorDetail,
stringContainsInOrder(<String>[
'Required parameter "result" not a valid boolean',
'not-a-boolean',
]),
);
});
group('compare', () {
late integration_test.VmServiceProxyGoldenFileComparator goldenFileComparator;
late List<(String, Map<Object?, Object?>)> postedEvents;
setUp(() {
postedEvents = <(String, Map<Object?, Object?>)>[];
goldenFileComparator = integration_test.VmServiceProxyGoldenFileComparator.forTesting((
String operation,
Map<Object?, Object?> params, {
String stream = '',
}) {
postedEvents.add((operation, params));
});
});
test('posts an event and returns true', () async {
// This is based on an implementation detail of knowing how IDs are generated.
const int nextId = 1;
final Uint8List bytes = Uint8List.fromList(<int>[1, 2, 3, 4, 5]);
expect(goldenFileComparator.compare(bytes, Uri(path: 'golden-path')), completion(true));
await goldenFileComparator.handleEvent(<String, String>{'id': '$nextId', 'result': 'true'});
final (String event, Map<Object?, Object?> params) = postedEvents.single;
expect(event, 'compare');
expect(params, <Object?, Object?>{
'id': nextId,
'path': 'golden-path',
'bytes': base64.encode(bytes),
});
});
test('posts an event and returns false', () async {
// This is based on an implementation detail of knowing how IDs are generated.
const int nextId = 1;
final Uint8List bytes = Uint8List.fromList(<int>[1, 2, 3, 4, 5]);
expect(goldenFileComparator.compare(bytes, Uri(path: 'golden-path')), completion(false));
await goldenFileComparator.handleEvent(<String, String>{
'id': '$nextId',
'result': 'false',
});
final (String event, Map<Object?, Object?> params) = postedEvents.single;
expect(event, 'compare');
expect(params, <Object?, Object?>{
'id': nextId,
'path': 'golden-path',
'bytes': base64.encode(bytes),
});
});
test('posts an event and returns an error', () async {
// This is based on an implementation detail of knowing how IDs are generated.
const int nextId = 1;
final Uint8List bytes = Uint8List.fromList(<int>[1, 2, 3, 4, 5]);
expect(
goldenFileComparator.compare(bytes, Uri(path: 'golden-path')),
throwsA(contains('We did a bad')),
);
await goldenFileComparator.handleEvent(<String, String>{
'id': '$nextId',
'error': 'We did a bad',
});
final (String event, Map<Object?, Object?> params) = postedEvents.single;
expect(event, 'compare');
expect(params, <Object?, Object?>{
'id': nextId,
'path': 'golden-path',
'bytes': base64.encode(bytes),
});
});
});
group('update', () {
late integration_test.VmServiceProxyGoldenFileComparator goldenFileComparator;
late List<(String, Map<Object?, Object?>)> postedEvents;
setUp(() {
postedEvents = <(String, Map<Object?, Object?>)>[];
goldenFileComparator = integration_test.VmServiceProxyGoldenFileComparator.forTesting((
String operation,
Map<Object?, Object?> params, {
String stream = '',
}) {
postedEvents.add((operation, params));
});
});
test('posts an event and returns', () async {
// This is based on an implementation detail of knowing how IDs are generated.
const int nextId = 1;
final Uint8List bytes = Uint8List.fromList(<int>[1, 2, 3, 4, 5]);
expect(goldenFileComparator.update(Uri(path: 'golden-path'), bytes), completes);
await goldenFileComparator.handleEvent(<String, String>{'id': '$nextId', 'result': 'true'});
final (String event, Map<Object?, Object?> params) = postedEvents.single;
expect(event, 'update');
expect(params, <Object?, Object?>{
'id': nextId,
'path': 'golden-path',
'bytes': base64.encode(bytes),
});
});
test('posts an event and returns an error', () async {
// This is based on an implementation detail of knowing how IDs are generated.
const int nextId = 1;
final Uint8List bytes = Uint8List.fromList(<int>[1, 2, 3, 4, 5]);
expect(
goldenFileComparator.update(Uri(path: 'golden-path'), bytes),
throwsA(contains('We did a bad')),
);
await goldenFileComparator.handleEvent(<String, String>{
'id': '$nextId',
'error': 'We did a bad',
});
final (String event, Map<Object?, Object?> params) = postedEvents.single;
expect(event, 'update');
expect(params, <Object?, Object?>{
'id': nextId,
'path': 'golden-path',
'bytes': base64.encode(bytes),
});
});
});
});
}
final class _NullGoldenFileComparator with Fake implements GoldenFileComparator {
const _NullGoldenFileComparator();
}