// Copyright 2014 The Flutter Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. // @dart = 2.8 import 'dart:async'; import 'dart:convert'; import 'package:args/command_runner.dart'; import 'package:file/memory.dart'; import 'package:flutter_tools/src/base/common.dart'; import 'package:flutter_tools/src/base/file_system.dart'; import 'package:flutter_tools/src/build_info.dart'; import 'package:flutter_tools/src/cache.dart'; import 'package:flutter_tools/src/commands/test.dart'; import 'package:flutter_tools/src/device.dart'; import 'package:flutter_tools/src/project.dart'; import 'package:flutter_tools/src/runner/flutter_command.dart'; import 'package:flutter_tools/src/test/runner.dart'; import 'package:flutter_tools/src/test/test_wrapper.dart'; import 'package:flutter_tools/src/test/watcher.dart'; import 'package:meta/meta.dart'; import '../../src/common.dart'; import '../../src/context.dart'; import '../../src/fake_devices.dart'; import '../../src/test_flutter_command_runner.dart'; const String _pubspecContents = ''' dev_dependencies: flutter_test: sdk: flutter integration_test: sdk: flutter'''; final String _packageConfigContents = json.encode({ 'configVersion': 2, 'packages': >[ { 'name': 'test_api', 'rootUri': 'file:///path/to/pubcache/.pub-cache/hosted/pub.dartlang.org/test_api-0.2.19', 'packageUri': 'lib/', 'languageVersion': '2.12' }, { 'name': 'integration_test', 'rootUri': 'file:///path/to/flutter/packages/integration_test', 'packageUri': 'lib/', 'languageVersion': '2.12' }, ], 'generated': '2021-02-24T07:55:20.084834Z', 'generator': 'pub', 'generatorVersion': '2.13.0-68.0.dev' }); void main() { Cache.disableLocking(); MemoryFileSystem fs; setUp(() { fs = MemoryFileSystem.test(); fs.file('/package/pubspec.yaml').createSync(recursive: true); fs.file('/package/pubspec.yaml').writeAsStringSync(_pubspecContents); (fs.directory('/package/.dart_tool') .childFile('package_config.json') ..createSync(recursive: true)) .writeAsString(_packageConfigContents); fs.directory('/package/test').childFile('some_test.dart').createSync(recursive: true); fs.directory('/package/integration_test').childFile('some_integration_test.dart').createSync(recursive: true); fs.currentDirectory = '/package'; }); testUsingContext('Missing dependencies in pubspec', () async { // Clear the dependencies already added in [setUp]. fs.file('pubspec.yaml').writeAsStringSync(''); fs.directory('.dart_tool').childFile('package_config.json').writeAsStringSync(''); final FakePackageTest fakePackageTest = FakePackageTest(); final TestCommand testCommand = TestCommand(testWrapper: fakePackageTest); final CommandRunner commandRunner = createTestCommandRunner(testCommand); expect(() => commandRunner.run(const [ 'test', '--no-pub', ]), throwsToolExit()); }, overrides: { FileSystem: () => fs, ProcessManager: () => FakeProcessManager.any(), }); testUsingContext('Missing dependencies in pubspec for integration tests', () async { // Only use the flutter_test dependency, integration_test is deliberately // absent. fs.file('pubspec.yaml').writeAsStringSync(''' dev_dependencies: flutter_test: sdk: flutter '''); fs.directory('.dart_tool').childFile('package_config.json').writeAsStringSync(json.encode({ 'configVersion': 2, 'packages': >[ { 'name': 'test_api', 'rootUri': 'file:///path/to/pubcache/.pub-cache/hosted/pub.dartlang.org/test_api-0.2.19', 'packageUri': 'lib/', 'languageVersion': '2.12' }, ], 'generated': '2021-02-24T07:55:20.084834Z', 'generator': 'pub', 'generatorVersion': '2.13.0-68.0.dev' })); final FakePackageTest fakePackageTest = FakePackageTest(); final TestCommand testCommand = TestCommand(testWrapper: fakePackageTest); final CommandRunner commandRunner = createTestCommandRunner(testCommand); expect(() => commandRunner.run(const [ 'test', '--no-pub', 'integration_test' ]), throwsToolExit()); }, overrides: { FileSystem: () => fs, ProcessManager: () => FakeProcessManager.any(), }); testUsingContext('Pipes test-randomize-ordering-seed to package:test', () async { final FakePackageTest fakePackageTest = FakePackageTest(); final TestCommand testCommand = TestCommand(testWrapper: fakePackageTest); final CommandRunner commandRunner = createTestCommandRunner(testCommand); await commandRunner.run(const [ 'test', '--test-randomize-ordering-seed=random', '--no-pub', ]); expect( fakePackageTest.lastArgs, contains('--test-randomize-ordering-seed=random'), ); }, overrides: { FileSystem: () => fs, ProcessManager: () => FakeProcessManager.any(), Cache: () => Cache.test(processManager: FakeProcessManager.any()), }); group('shard-index and total-shards', () { testUsingContext('with the params they are Piped to package:test', () async { final FakePackageTest fakePackageTest = FakePackageTest(); final TestCommand testCommand = TestCommand(testWrapper: fakePackageTest); final CommandRunner commandRunner = createTestCommandRunner(testCommand); await commandRunner.run(const [ 'test', '--total-shards=1', '--shard-index=2', '--no-pub', ]); expect(fakePackageTest.lastArgs, contains('--total-shards=1')); expect(fakePackageTest.lastArgs, contains('--shard-index=2')); }, overrides: { FileSystem: () => fs, ProcessManager: () => FakeProcessManager.any(), Cache: () => Cache.test(processManager: FakeProcessManager.any()), }); testUsingContext('without the params they not Piped to package:test', () async { final FakePackageTest fakePackageTest = FakePackageTest(); final TestCommand testCommand = TestCommand(testWrapper: fakePackageTest); final CommandRunner commandRunner = createTestCommandRunner(testCommand); await commandRunner.run(const [ 'test', '--no-pub', ]); expect(fakePackageTest.lastArgs, isNot(contains('--total-shards'))); expect(fakePackageTest.lastArgs, isNot(contains('--shard-index'))); }, overrides: { FileSystem: () => fs, ProcessManager: () => FakeProcessManager.any(), Cache: () => Cache.test(processManager: FakeProcessManager.any()), }); }); testUsingContext('Supports coverage and machine', () async { final FakePackageTest fakePackageTest = FakePackageTest(); final TestCommand testCommand = TestCommand(testWrapper: fakePackageTest); final CommandRunner commandRunner = createTestCommandRunner(testCommand); expect(() => commandRunner.run(const [ 'test', '--no-pub', '--machine', '--coverage', '--', 'test/fake_test.dart', ]), throwsA(isA().having((ToolExit toolExit) => toolExit.message, 'message', isNull))); }, overrides: { FileSystem: () => fs, ProcessManager: () => FakeProcessManager.any(), Cache: () => Cache.test(processManager: FakeProcessManager.any()), }); testUsingContext('Pipes start-paused to package:test', () async { final FakePackageTest fakePackageTest = FakePackageTest(); final TestCommand testCommand = TestCommand(testWrapper: fakePackageTest); final CommandRunner commandRunner = createTestCommandRunner(testCommand); await commandRunner.run(const [ 'test', '--no-pub', '--start-paused', '--', 'test/fake_test.dart', ]); expect( fakePackageTest.lastArgs, contains('--pause-after-load'), ); }, overrides: { FileSystem: () => fs, ProcessManager: () => FakeProcessManager.any(), Cache: () => Cache.test(processManager: FakeProcessManager.any()), }); testUsingContext('Pipes run-skipped to package:test', () async { final FakePackageTest fakePackageTest = FakePackageTest(); final TestCommand testCommand = TestCommand(testWrapper: fakePackageTest); final CommandRunner commandRunner = createTestCommandRunner(testCommand); await commandRunner.run(const [ 'test', '--no-pub', '--run-skipped', '--', 'test/fake_test.dart', ]); expect( fakePackageTest.lastArgs, contains('--run-skipped'), ); }, overrides: { FileSystem: () => fs, ProcessManager: () => FakeProcessManager.any(), Cache: () => Cache.test(processManager: FakeProcessManager.any()), }); testUsingContext('Pipes enable-observatory', () async { final FakeFlutterTestRunner testRunner = FakeFlutterTestRunner(0); final TestCommand testCommand = TestCommand(testRunner: testRunner); final CommandRunner commandRunner = createTestCommandRunner(testCommand); await commandRunner.run(const [ 'test', '--no-pub', '--enable-vmservice', '--', 'test/fake_test.dart', ]); expect( testRunner.lastEnableObservatoryValue, true, ); await commandRunner.run(const [ 'test', '--no-pub', '--start-paused', '--no-enable-vmservice', '--', 'test/fake_test.dart', ]); expect( testRunner.lastEnableObservatoryValue, true, ); await commandRunner.run(const [ 'test', '--no-pub', '--', 'test/fake_test.dart', ]); expect( testRunner.lastEnableObservatoryValue, false, ); }, overrides: { FileSystem: () => fs, ProcessManager: () => FakeProcessManager.any(), Cache: () => Cache.test(processManager: FakeProcessManager.any()), }); testUsingContext('Pipes different args when running Integration Tests', () async { final FakePackageTest fakePackageTest = FakePackageTest(); final TestCommand testCommand = TestCommand(testWrapper: fakePackageTest); final CommandRunner commandRunner = createTestCommandRunner(testCommand); await commandRunner.run(const [ 'test', '--no-pub', 'integration_test', ]); expect(fakePackageTest.lastArgs, contains('--concurrency=1')); }, overrides: { FileSystem: () => fs, ProcessManager: () => FakeProcessManager.any(), DeviceManager: () => _FakeDeviceManager([ FakeDevice('ephemeral', 'ephemeral', type: PlatformType.android), ]), }); testUsingContext('Overrides concurrency when running Integration Tests', () async { final FakePackageTest fakePackageTest = FakePackageTest(); final TestCommand testCommand = TestCommand(testWrapper: fakePackageTest); final CommandRunner commandRunner = createTestCommandRunner(testCommand); await commandRunner.run(const [ 'test', '--no-pub', '--concurrency=100', 'integration_test', ]); expect(fakePackageTest.lastArgs, contains('--concurrency=1')); }, overrides: { FileSystem: () => fs, ProcessManager: () => FakeProcessManager.any(), DeviceManager: () => _FakeDeviceManager([ FakeDevice('ephemeral', 'ephemeral', type: PlatformType.android), ]), }); group('Detecting Integration Tests', () { testUsingContext('when integration_test is not passed', () async { final FakePackageTest fakePackageTest = FakePackageTest(); final TestCommand testCommand = TestCommand(testWrapper: fakePackageTest); final CommandRunner commandRunner = createTestCommandRunner(testCommand); await commandRunner.run(const [ 'test', '--no-pub', ]); expect(testCommand.isIntegrationTest, false); }, overrides: { FileSystem: () => fs, ProcessManager: () => FakeProcessManager.any(), DeviceManager: () => _FakeDeviceManager([ FakeDevice('ephemeral', 'ephemeral', type: PlatformType.android), ]), }); testUsingContext('when integration_test is passed', () async { final FakePackageTest fakePackageTest = FakePackageTest(); final TestCommand testCommand = TestCommand(testWrapper: fakePackageTest); final CommandRunner commandRunner = createTestCommandRunner(testCommand); await commandRunner.run(const [ 'test', '--no-pub', 'integration_test', ]); expect(testCommand.isIntegrationTest, true); }, overrides: { FileSystem: () => fs, ProcessManager: () => FakeProcessManager.any(), DeviceManager: () => _FakeDeviceManager([ FakeDevice('ephemeral', 'ephemeral', type: PlatformType.android), ]), }); testUsingContext('when relative path to integration test is passed', () async { final FakePackageTest fakePackageTest = FakePackageTest(); final TestCommand testCommand = TestCommand(testWrapper: fakePackageTest); final CommandRunner commandRunner = createTestCommandRunner(testCommand); await commandRunner.run(const [ 'test', '--no-pub', 'integration_test/some_integration_test.dart', ]); expect(testCommand.isIntegrationTest, true); }, overrides: { FileSystem: () => fs, ProcessManager: () => FakeProcessManager.any(), DeviceManager: () => _FakeDeviceManager([ FakeDevice('ephemeral', 'ephemeral', type: PlatformType.android), ]), }); testUsingContext('when absolute path to integration test is passed', () async { final FakePackageTest fakePackageTest = FakePackageTest(); final TestCommand testCommand = TestCommand(testWrapper: fakePackageTest); final CommandRunner commandRunner = createTestCommandRunner(testCommand); await commandRunner.run(const [ 'test', '--no-pub', '/package/integration_test/some_integration_test.dart', ]); expect(testCommand.isIntegrationTest, true); }, overrides: { FileSystem: () => fs, ProcessManager: () => FakeProcessManager.any(), DeviceManager: () => _FakeDeviceManager([ FakeDevice('ephemeral', 'ephemeral', type: PlatformType.android), ]), }); testUsingContext('when absolute unnormalized path to integration test is passed', () async { final FakePackageTest fakePackageTest = FakePackageTest(); final TestCommand testCommand = TestCommand(testWrapper: fakePackageTest); final CommandRunner commandRunner = createTestCommandRunner(testCommand); await commandRunner.run(const [ 'test', '--no-pub', '/package/../package/integration_test/some_integration_test.dart', ]); expect(testCommand.isIntegrationTest, true); }, overrides: { FileSystem: () => fs, ProcessManager: () => FakeProcessManager.any(), DeviceManager: () => _FakeDeviceManager([ FakeDevice('ephemeral', 'ephemeral', type: PlatformType.android), ]), }); testUsingContext('when both test and integration test are passed', () async { final FakeFlutterTestRunner testRunner = FakeFlutterTestRunner(0); final TestCommand testCommand = TestCommand(testRunner: testRunner); final CommandRunner commandRunner = createTestCommandRunner(testCommand); expect(() => commandRunner.run(const [ 'test', '--no-pub', 'test/some_test.dart', 'integration_test/some_integration_test.dart', ]), throwsToolExit()); }, overrides: { FileSystem: () => fs, ProcessManager: () => FakeProcessManager.any(), }); }); group('Required artifacts', () { testUsingContext('for default invocation', () async { final FakeFlutterTestRunner testRunner = FakeFlutterTestRunner(0); final TestCommand testCommand = TestCommand(testRunner: testRunner); final CommandRunner commandRunner = createTestCommandRunner(testCommand); await commandRunner.run(const [ 'test', '--no-pub', ]); expect(await testCommand.requiredArtifacts, isEmpty); }, overrides: { FileSystem: () => fs, ProcessManager: () => FakeProcessManager.any(), }); testUsingContext('when platform is chrome', () async { final FakeFlutterTestRunner testRunner = FakeFlutterTestRunner(0); final TestCommand testCommand = TestCommand(testRunner: testRunner); final CommandRunner commandRunner = createTestCommandRunner(testCommand); await commandRunner.run(const [ 'test', '--no-pub', '--platform=chrome' ]); expect(await testCommand.requiredArtifacts, [DevelopmentArtifact.web]); }, overrides: { FileSystem: () => fs, ProcessManager: () => FakeProcessManager.any(), }); testUsingContext('when running integration tests', () async { final FakeFlutterTestRunner testRunner = FakeFlutterTestRunner(0); final TestCommand testCommand = TestCommand(testRunner: testRunner); final CommandRunner commandRunner = createTestCommandRunner(testCommand); await commandRunner.run(const [ 'test', '--no-pub', 'integration_test', ]); expect(await testCommand.requiredArtifacts, [ DevelopmentArtifact.universal, DevelopmentArtifact.androidGenSnapshot, ]); }, overrides: { FileSystem: () => fs, ProcessManager: () => FakeProcessManager.any(), DeviceManager: () => _FakeDeviceManager([ FakeDevice('ephemeral', 'ephemeral', type: PlatformType.android), ]), }); }); testUsingContext('Integration tests when no devices are connected', () async { final FakeFlutterTestRunner testRunner = FakeFlutterTestRunner(0); final TestCommand testCommand = TestCommand(testRunner: testRunner); final CommandRunner commandRunner = createTestCommandRunner(testCommand); expect(() => commandRunner.run(const [ 'test', '--no-pub', 'integration_test', ]), throwsToolExit()); }, overrides: { FileSystem: () => fs, ProcessManager: () => FakeProcessManager.any(), DeviceManager: () => _FakeDeviceManager([]), }); // TODO(jiahaog): Remove this when web is supported. https://github.com/flutter/flutter/pull/74236 testUsingContext('Integration tests when only web devices are connected', () async { final FakeFlutterTestRunner testRunner = FakeFlutterTestRunner(0); final TestCommand testCommand = TestCommand(testRunner: testRunner); final CommandRunner commandRunner = createTestCommandRunner(testCommand); expect(() => commandRunner.run(const [ 'test', '--no-pub', 'integration_test', ]), throwsToolExit()); }, overrides: { FileSystem: () => fs, ProcessManager: () => FakeProcessManager.any(), DeviceManager: () => _FakeDeviceManager([ FakeDevice('ephemeral', 'ephemeral'), ]), }); testUsingContext('Integration tests set the correct dart-defines', () async { final FakeFlutterTestRunner testRunner = FakeFlutterTestRunner(0); final TestCommand testCommand = TestCommand(testRunner: testRunner); final CommandRunner commandRunner = createTestCommandRunner(testCommand); await commandRunner.run(const [ 'test', '--no-pub', 'integration_test', ]); expect( testRunner.lastDebuggingOptionsValue.buildInfo.dartDefines, contains('INTEGRATION_TEST_SHOULD_REPORT_RESULTS_TO_NATIVE=false'), ); }, overrides: { FileSystem: () => fs, ProcessManager: () => FakeProcessManager.any(), DeviceManager: () => _FakeDeviceManager([ FakeDevice('ephemeral', 'ephemeral', type: PlatformType.android), ]), }); testUsingContext('Builds the asset manifest by default', () async { final FakeFlutterTestRunner testRunner = FakeFlutterTestRunner(0); final TestCommand testCommand = TestCommand(testRunner: testRunner); final CommandRunner commandRunner = createTestCommandRunner(testCommand); await commandRunner.run(const [ 'test', '--no-pub', ]); final bool fileExists = await fs.isFile('build/unit_test_assets/AssetManifest.json'); expect(fileExists, true); }, overrides: { FileSystem: () => fs, ProcessManager: () => FakeProcessManager.any(), DeviceManager: () => _FakeDeviceManager([]), }); testUsingContext("Don't build the asset manifest if --no-test-assets if informed", () async { final FakeFlutterTestRunner testRunner = FakeFlutterTestRunner(0); final TestCommand testCommand = TestCommand(testRunner: testRunner); final CommandRunner commandRunner = createTestCommandRunner(testCommand); await commandRunner.run(const [ 'test', '--no-pub', '--no-test-assets', ]); final bool fileExists = await fs.isFile('build/unit_test_assets/AssetManifest.json'); expect(fileExists, false); }, overrides: { FileSystem: () => fs, ProcessManager: () => FakeProcessManager.any(), DeviceManager: () => _FakeDeviceManager([]), }); group('Fatal Logs', () { testUsingContext("doesn't fail when --fatal-warnings is set and no warning output", () async { final FakeFlutterTestRunner testRunner = FakeFlutterTestRunner(0); final TestCommand testCommand = TestCommand(testRunner: testRunner); final CommandRunner commandRunner = createTestCommandRunner(testCommand); try { await commandRunner.run(const [ 'test', '--no-pub', '--${FlutterOptions.kFatalWarnings}', ]); } on Exception { fail('Unexpected exception thrown'); } }, overrides: { FileSystem: () => fs, ProcessManager: () => FakeProcessManager.any(), }); testUsingContext('fails if --fatal-warnings specified and warnings emitted', () async { final FakeFlutterTestRunner testRunner = FakeFlutterTestRunner(0); final TestCommand testCommand = TestCommand(testRunner: testRunner); final CommandRunner commandRunner = createTestCommandRunner(testCommand); testLogger.printWarning('Warning: Mild annoyance, Will Robinson!'); expect(commandRunner.run(const [ 'test', '--no-pub', '--${FlutterOptions.kFatalWarnings}', ]), throwsToolExit(message: 'Logger received warning output during the run, and "--${FlutterOptions.kFatalWarnings}" is enabled.')); }, overrides: { FileSystem: () => fs, ProcessManager: () => FakeProcessManager.any(), }); testUsingContext('fails when --fatal-warnings is set and only errors emitted', () async { final FakeFlutterTestRunner testRunner = FakeFlutterTestRunner(0); final TestCommand testCommand = TestCommand(testRunner: testRunner); final CommandRunner commandRunner = createTestCommandRunner(testCommand); testLogger.printError('Error: Danger Will Robinson!'); expect(commandRunner.run(const [ 'test', '--no-pub', '--${FlutterOptions.kFatalWarnings}', ]), throwsToolExit(message: 'Logger received error output during the run, and "--${FlutterOptions.kFatalWarnings}" is enabled.')); }, overrides: { FileSystem: () => fs, ProcessManager: () => FakeProcessManager.any(), }); }); } class FakeFlutterTestRunner implements FlutterTestRunner { FakeFlutterTestRunner(this.exitCode); int exitCode; bool lastEnableObservatoryValue; DebuggingOptions lastDebuggingOptionsValue; @override Future runTests( TestWrapper testWrapper, List testFiles, { @required DebuggingOptions debuggingOptions, Directory workDir, List names = const [], List plainNames = const [], String tags, String excludeTags, bool enableObservatory = false, bool ipv6 = false, bool machine = false, String precompiledDillPath, Map precompiledDillFiles, BuildMode buildMode, bool trackWidgetCreation = false, bool updateGoldens = false, TestWatcher watcher, int concurrency, bool buildTestAssets = false, FlutterProject flutterProject, String icudtlPath, Directory coverageDirectory, bool web = false, String randomSeed, @override List extraFrontEndOptions, String reporter, String timeout, bool runSkipped = false, int shardIndex, int totalShards, Device integrationTestDevice, String integrationTestUserIdentifier, }) async { lastEnableObservatoryValue = enableObservatory; lastDebuggingOptionsValue = debuggingOptions; return exitCode; } } class FakePackageTest implements TestWrapper { List lastArgs; @override Future main(List args) async { lastArgs = args; } @override void registerPlatformPlugin( Iterable runtimes, FutureOr Function() platforms, ) {} } class _FakeDeviceManager extends DeviceManager { _FakeDeviceManager(this._connectedDevices); final List _connectedDevices; @override Future> getAllConnectedDevices() async => _connectedDevices; @override List get deviceDiscoverers => []; }