mirror of
https://github.com/flutter/flutter.git
synced 2025-06-03 00:51:18 +00:00
ff0ab6ba45
The previous expression `success || line.contains('--- TEST FAILED ---')` made it such that as soon as success is reported, the test would send "q" on every line of stdout, which is wrong. This fixes the condition such that "q" is sent once, only when `--- TEST SUCCEEDED ---` or `--- TEST FAILED ---` is printed.
771 lines
29 KiB
Dart
771 lines
29 KiB
Dart
// Copyright 2014 The Flutter Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
// found in the LICENSE file.
|
|
|
|
import 'dart:convert';
|
|
import 'dart:io' show Directory, File, FileSystemEntity, HttpClient, HttpClientRequest, HttpClientResponse, Platform, Process, RawSocket, SocketDirection, SocketException;
|
|
import 'dart:math' as math;
|
|
import 'package:file/local.dart';
|
|
import 'package:path/path.dart' as path;
|
|
|
|
import '../browser.dart';
|
|
import '../run_command.dart';
|
|
import '../service_worker_test.dart';
|
|
import '../utils.dart';
|
|
|
|
typedef ShardRunner = Future<void> Function();
|
|
|
|
class WebTestsSuite {
|
|
|
|
WebTestsSuite(this.flutterTestArgs);
|
|
|
|
/// Tests that we don't run on Web.
|
|
///
|
|
/// In general avoid adding new tests here. If a test cannot run on the web
|
|
/// because it fails at runtime, such as when a piece of functionality is not
|
|
/// implemented or not implementable on the web, prefer using `skip` in the
|
|
/// test code. Only add tests here that cannot be skipped using `skip`. For
|
|
/// example:
|
|
///
|
|
/// * Test code cannot be compiled because it uses Dart VM-specific
|
|
/// functionality. In this case `skip` doesn't help because the code cannot
|
|
/// reach the point where it can even run the skipping logic.
|
|
/// * Migrations. It is OK to put tests here that need to be temporarily
|
|
/// disabled in certain modes because of some migration or initial bringup.
|
|
///
|
|
/// The key in the map is the renderer type that the list applies to. The value
|
|
/// is the list of tests known to fail for that renderer.
|
|
//
|
|
// TODO(yjbanov): we're getting rid of this as part of https://github.com/flutter/flutter/projects/60
|
|
static const Map<String, List<String>> kWebTestFileKnownFailures = <String, List<String>>{
|
|
'html': <String>[
|
|
// These tests are not compilable on the web due to dependencies on
|
|
// VM-specific functionality.
|
|
'test/services/message_codecs_vm_test.dart',
|
|
'test/examples/sector_layout_test.dart',
|
|
],
|
|
'canvaskit': <String>[
|
|
// These tests are not compilable on the web due to dependencies on
|
|
// VM-specific functionality.
|
|
'test/services/message_codecs_vm_test.dart',
|
|
'test/examples/sector_layout_test.dart',
|
|
|
|
// These tests are broken and need to be fixed.
|
|
// TODO(yjbanov): https://github.com/flutter/flutter/issues/71604
|
|
'test/material/text_field_test.dart',
|
|
'test/widgets/performance_overlay_test.dart',
|
|
'test/widgets/html_element_view_test.dart',
|
|
'test/cupertino/scaffold_test.dart',
|
|
'test/rendering/platform_view_test.dart',
|
|
],
|
|
'skwasm': <String>[
|
|
// These tests are not compilable on the web due to dependencies on
|
|
// VM-specific functionality.
|
|
'test/services/message_codecs_vm_test.dart',
|
|
'test/examples/sector_layout_test.dart',
|
|
|
|
// These tests are broken and need to be fixed.
|
|
// TODO(jacksongardner): https://github.com/flutter/flutter/issues/71604
|
|
'test/material/text_field_test.dart',
|
|
'test/widgets/performance_overlay_test.dart',
|
|
],
|
|
};
|
|
|
|
/// The number of Cirrus jobs that run Web tests in parallel.
|
|
///
|
|
/// The default is 8 shards. Typically .cirrus.yml would define the
|
|
/// WEB_SHARD_COUNT environment variable rather than relying on the default.
|
|
///
|
|
/// WARNING: if you change this number, also change .cirrus.yml
|
|
/// and make sure it runs _all_ shards.
|
|
///
|
|
/// The last shard also runs the Web plugin tests.
|
|
int get webShardCount => Platform.environment.containsKey('WEB_SHARD_COUNT')
|
|
? int.parse(Platform.environment['WEB_SHARD_COUNT']!)
|
|
: 8;
|
|
|
|
|
|
static const List<String> _kAllBuildModes = <String>['debug', 'profile', 'release'];
|
|
|
|
final List<String> flutterTestArgs;
|
|
|
|
/// Coarse-grained integration tests running on the Web.
|
|
Future<void> webLongRunningTestsRunner() async {
|
|
|
|
final String engineVersionFile = path.join(flutterRoot, 'bin', 'internal', 'engine.version');
|
|
final String engineRealmFile = path.join(flutterRoot, 'bin', 'internal', 'engine.realm');
|
|
final String engineVersion = File(engineVersionFile).readAsStringSync().trim();
|
|
final String engineRealm = File(engineRealmFile).readAsStringSync().trim();
|
|
if (engineRealm.isNotEmpty) {
|
|
return;
|
|
}
|
|
final List<ShardRunner> tests = <ShardRunner>[
|
|
for (final String buildMode in _kAllBuildModes) ...<ShardRunner>[
|
|
() => _runFlutterDriverWebTest(
|
|
testAppDirectory: path.join('packages', 'integration_test', 'example'),
|
|
target: path.join('test_driver', 'failure.dart'),
|
|
buildMode: buildMode,
|
|
renderer: 'canvaskit',
|
|
// This test intentionally fails and prints stack traces in the browser
|
|
// logs. To avoid confusion, silence browser output.
|
|
silenceBrowserOutput: true,
|
|
),
|
|
() => _runFlutterDriverWebTest(
|
|
testAppDirectory: path.join('packages', 'integration_test', 'example'),
|
|
target: path.join('integration_test', 'example_test.dart'),
|
|
driver: path.join('test_driver', 'integration_test.dart'),
|
|
buildMode: buildMode,
|
|
renderer: 'canvaskit',
|
|
expectWriteResponseFile: true,
|
|
expectResponseFileContent: 'null',
|
|
),
|
|
() => _runFlutterDriverWebTest(
|
|
testAppDirectory: path.join('packages', 'integration_test', 'example'),
|
|
target: path.join('integration_test', 'extended_test.dart'),
|
|
driver: path.join('test_driver', 'extended_integration_test.dart'),
|
|
buildMode: buildMode,
|
|
renderer: 'canvaskit',
|
|
expectWriteResponseFile: true,
|
|
expectResponseFileContent: '''
|
|
{
|
|
"screenshots": [
|
|
{
|
|
"screenshotName": "platform_name",
|
|
"bytes": []
|
|
},
|
|
{
|
|
"screenshotName": "platform_name_2",
|
|
"bytes": []
|
|
}
|
|
]
|
|
}''',
|
|
),
|
|
],
|
|
|
|
// This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
|
|
() => _runWebE2eTest('profile_diagnostics_integration', buildMode: 'debug', renderer: 'html'),
|
|
() => _runWebE2eTest('profile_diagnostics_integration', buildMode: 'profile', renderer: 'canvaskit'),
|
|
() => _runWebE2eTest('profile_diagnostics_integration', buildMode: 'release', renderer: 'html'),
|
|
|
|
// This test is only known to work in debug mode.
|
|
() => _runWebE2eTest('scroll_wheel_integration', buildMode: 'debug', renderer: 'html'),
|
|
|
|
// This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
|
|
// These tests have been extremely flaky, so we are temporarily disabling them until we figure out how to make them more robust.
|
|
() => _runWebE2eTest('text_editing_integration', buildMode: 'debug', renderer: 'canvaskit'),
|
|
() => _runWebE2eTest('text_editing_integration', buildMode: 'profile', renderer: 'html'),
|
|
() => _runWebE2eTest('text_editing_integration', buildMode: 'release', renderer: 'html'),
|
|
|
|
// This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
|
|
() => _runWebE2eTest('url_strategy_integration', buildMode: 'debug', renderer: 'html'),
|
|
() => _runWebE2eTest('url_strategy_integration', buildMode: 'profile', renderer: 'canvaskit'),
|
|
() => _runWebE2eTest('url_strategy_integration', buildMode: 'release', renderer: 'html'),
|
|
|
|
// This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
|
|
() => _runWebE2eTest('capabilities_integration_canvaskit', buildMode: 'debug', renderer: 'auto'),
|
|
() => _runWebE2eTest('capabilities_integration_canvaskit', buildMode: 'profile', renderer: 'canvaskit'),
|
|
() => _runWebE2eTest('capabilities_integration_html', buildMode: 'release', renderer: 'html'),
|
|
|
|
// This test doesn't do anything interesting w.r.t. rendering, so we don't run the full build mode x renderer matrix.
|
|
// CacheWidth and CacheHeight are only currently supported in CanvasKit mode, so we don't run the test in HTML mode.
|
|
() => _runWebE2eTest('cache_width_cache_height_integration', buildMode: 'debug', renderer: 'auto'),
|
|
() => _runWebE2eTest('cache_width_cache_height_integration', buildMode: 'profile', renderer: 'canvaskit'),
|
|
|
|
() => _runWebTreeshakeTest(),
|
|
|
|
() => _runFlutterDriverWebTest(
|
|
testAppDirectory: path.join(flutterRoot, 'examples', 'hello_world'),
|
|
target: 'test_driver/smoke_web_engine.dart',
|
|
buildMode: 'profile',
|
|
renderer: 'auto',
|
|
),
|
|
() => _runGalleryE2eWebTest('debug'),
|
|
() => _runGalleryE2eWebTest('debug', canvasKit: true),
|
|
() => _runGalleryE2eWebTest('profile'),
|
|
() => _runGalleryE2eWebTest('profile', canvasKit: true),
|
|
() => _runGalleryE2eWebTest('release'),
|
|
() => _runGalleryE2eWebTest('release', canvasKit: true),
|
|
() => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withoutFlutterJs),
|
|
() => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withFlutterJs),
|
|
() => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withFlutterJsShort),
|
|
() => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withFlutterJsEntrypointLoadedEvent),
|
|
() => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withFlutterJsTrustedTypesOn),
|
|
() => runWebServiceWorkerTest(headless: true, testType: ServiceWorkerTestType.withFlutterJsNonceOn),
|
|
() => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withoutFlutterJs),
|
|
() => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withFlutterJs),
|
|
() => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withFlutterJsShort),
|
|
() => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withFlutterJsEntrypointLoadedEvent),
|
|
() => runWebServiceWorkerTestWithCachingResources(headless: true, testType: ServiceWorkerTestType.withFlutterJsTrustedTypesOn),
|
|
() => runWebServiceWorkerTestWithGeneratedEntrypoint(headless: true),
|
|
() => runWebServiceWorkerTestWithBlockedServiceWorkers(headless: true),
|
|
() => runWebServiceWorkerTestWithCustomServiceWorkerVersion(headless: true),
|
|
() => _runWebStackTraceTest('profile', 'lib/stack_trace.dart'),
|
|
() => _runWebStackTraceTest('release', 'lib/stack_trace.dart'),
|
|
() => _runWebStackTraceTest('profile', 'lib/framework_stack_trace.dart'),
|
|
() => _runWebStackTraceTest('release', 'lib/framework_stack_trace.dart'),
|
|
() => _runWebDebugTest('lib/stack_trace.dart'),
|
|
() => _runWebDebugTest('lib/framework_stack_trace.dart'),
|
|
() => _runWebDebugTest('lib/web_directory_loading.dart'),
|
|
() => _runWebDebugTest('lib/web_resources_cdn_test.dart',
|
|
additionalArguments: <String>[
|
|
'--dart-define=TEST_FLUTTER_ENGINE_VERSION=$engineVersion',
|
|
]),
|
|
() => _runWebDebugTest('test/test.dart'),
|
|
() => _runWebDebugTest('lib/null_safe_main.dart'),
|
|
() => _runWebDebugTest('lib/web_define_loading.dart',
|
|
additionalArguments: <String>[
|
|
'--dart-define=test.valueA=Example,A',
|
|
'--dart-define=test.valueB=Value',
|
|
]
|
|
),
|
|
() => _runWebReleaseTest('lib/web_define_loading.dart',
|
|
additionalArguments: <String>[
|
|
'--dart-define=test.valueA=Example,A',
|
|
'--dart-define=test.valueB=Value',
|
|
]
|
|
),
|
|
() => _runWebDebugTest('lib/sound_mode.dart'),
|
|
() => _runWebReleaseTest('lib/sound_mode.dart'),
|
|
() => _runFlutterWebTest(
|
|
'html',
|
|
path.join(flutterRoot, 'packages', 'integration_test'),
|
|
<String>['test/web_extension_test.dart'],
|
|
false,
|
|
),
|
|
() => _runFlutterWebTest(
|
|
'canvaskit',
|
|
path.join(flutterRoot, 'packages', 'integration_test'),
|
|
<String>['test/web_extension_test.dart'],
|
|
false,
|
|
),
|
|
() => _runFlutterWebTest(
|
|
'skwasm',
|
|
path.join(flutterRoot, 'packages', 'integration_test'),
|
|
<String>['test/web_extension_test.dart'],
|
|
true,
|
|
),
|
|
];
|
|
|
|
// Shuffling mixes fast tests with slow tests so shards take roughly the same
|
|
// amount of time to run.
|
|
tests.shuffle(math.Random(0));
|
|
|
|
await _ensureChromeDriverIsRunning();
|
|
await runShardRunnerIndexOfTotalSubshard(tests);
|
|
await _stopChromeDriver();
|
|
}
|
|
|
|
Future<void> runWebHtmlUnitTests() {
|
|
return _runWebUnitTests('html', false);
|
|
}
|
|
|
|
Future<void> runWebCanvasKitUnitTests() {
|
|
return _runWebUnitTests('canvaskit', false);
|
|
}
|
|
|
|
Future<void> runWebSkwasmUnitTests() {
|
|
return _runWebUnitTests('skwasm', true);
|
|
}
|
|
|
|
/// Runs one of the `dev/integration_tests/web_e2e_tests` tests.
|
|
Future<void> _runWebE2eTest(
|
|
String name, {
|
|
required String buildMode,
|
|
required String renderer,
|
|
}) async {
|
|
await _runFlutterDriverWebTest(
|
|
target: path.join('test_driver', '$name.dart'),
|
|
buildMode: buildMode,
|
|
renderer: renderer,
|
|
testAppDirectory: path.join(flutterRoot, 'dev', 'integration_tests', 'web_e2e_tests'),
|
|
);
|
|
}
|
|
|
|
Future<void> _runFlutterDriverWebTest({
|
|
required String target,
|
|
required String buildMode,
|
|
required String renderer,
|
|
required String testAppDirectory,
|
|
String? driver,
|
|
bool expectFailure = false,
|
|
bool silenceBrowserOutput = false,
|
|
bool expectWriteResponseFile = false,
|
|
String expectResponseFileContent = '',
|
|
}) async {
|
|
// TODO(yjbanov): this is temporarily disabled due to https://github.com/flutter/flutter/issues/147731
|
|
if (buildMode == 'debug' && renderer == 'canvaskit') {
|
|
print('SKIPPED: $target in debug CanvasKit mode due to https://github.com/flutter/flutter/issues/147731');
|
|
return;
|
|
}
|
|
printProgress('${green}Running integration tests $target in $buildMode mode.$reset');
|
|
await runCommand(
|
|
flutter,
|
|
<String>[ 'clean' ],
|
|
workingDirectory: testAppDirectory,
|
|
);
|
|
final String responseFile =
|
|
path.join(testAppDirectory, 'build', 'integration_response_data.json');
|
|
if (File(responseFile).existsSync()) {
|
|
File(responseFile).deleteSync();
|
|
}
|
|
await runCommand(
|
|
flutter,
|
|
<String>[
|
|
...flutterTestArgs,
|
|
'drive',
|
|
if (driver != null) '--driver=$driver',
|
|
'--target=$target',
|
|
'--browser-name=chrome',
|
|
'-d',
|
|
'web-server',
|
|
'--$buildMode',
|
|
'--web-renderer=$renderer',
|
|
],
|
|
expectNonZeroExit: expectFailure,
|
|
workingDirectory: testAppDirectory,
|
|
environment: <String, String>{
|
|
'FLUTTER_WEB': 'true',
|
|
},
|
|
removeLine: (String line) {
|
|
if (!silenceBrowserOutput) {
|
|
return false;
|
|
}
|
|
if (line.trim().startsWith('[INFO]')) {
|
|
return true;
|
|
}
|
|
return false;
|
|
},
|
|
);
|
|
if (expectWriteResponseFile) {
|
|
if (!File(responseFile).existsSync()) {
|
|
foundError(<String>[
|
|
'$bold${red}Command did not write the response file but expected response file written.$reset',
|
|
]);
|
|
} else {
|
|
final String response = File(responseFile).readAsStringSync();
|
|
if (response != expectResponseFileContent) {
|
|
foundError(<String>[
|
|
'$bold${red}Command write the response file with $response but expected response file with $expectResponseFileContent.$reset',
|
|
]);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Compiles a sample web app and checks that its JS doesn't contain certain
|
|
// debug code that we expect to be tree shaken out.
|
|
//
|
|
// The app is compiled in `--profile` mode to prevent the compiler from
|
|
// minifying the symbols.
|
|
Future<void> _runWebTreeshakeTest() async {
|
|
final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web_e2e_tests');
|
|
final String target = path.join('lib', 'treeshaking_main.dart');
|
|
await runCommand(
|
|
flutter,
|
|
<String>[ 'clean' ],
|
|
workingDirectory: testAppDirectory,
|
|
);
|
|
await runCommand(
|
|
flutter,
|
|
<String>[
|
|
'build',
|
|
'web',
|
|
'--target=$target',
|
|
'--profile',
|
|
],
|
|
workingDirectory: testAppDirectory,
|
|
environment: <String, String>{
|
|
'FLUTTER_WEB': 'true',
|
|
},
|
|
);
|
|
|
|
final File mainDartJs = File(path.join(testAppDirectory, 'build', 'web', 'main.dart.js'));
|
|
final String javaScript = mainDartJs.readAsStringSync();
|
|
|
|
// Check that we're not looking at minified JS. Otherwise this test would result in false positive.
|
|
expect(javaScript.contains('RootElement'), true);
|
|
|
|
const String word = 'debugFillProperties';
|
|
int count = 0;
|
|
int pos = javaScript.indexOf(word);
|
|
final int contentLength = javaScript.length;
|
|
while (pos != -1) {
|
|
count += 1;
|
|
pos += word.length;
|
|
if (pos >= contentLength || count > 100) {
|
|
break;
|
|
}
|
|
pos = javaScript.indexOf(word, pos);
|
|
}
|
|
|
|
// The following are classes from `timeline.dart` that should be treeshaken
|
|
// off unless the app (typically a benchmark) uses methods that need them.
|
|
expect(javaScript.contains('AggregatedTimedBlock'), false);
|
|
expect(javaScript.contains('AggregatedTimings'), false);
|
|
expect(javaScript.contains('_BlockBuffer'), false);
|
|
expect(javaScript.contains('_StringListChain'), false);
|
|
expect(javaScript.contains('_Float64ListChain'), false);
|
|
|
|
const int kMaxExpectedDebugFillProperties = 11;
|
|
if (count > kMaxExpectedDebugFillProperties) {
|
|
throw Exception(
|
|
'Too many occurrences of "$word" in compiled JavaScript.\n'
|
|
'Expected no more than $kMaxExpectedDebugFillProperties, but found $count.'
|
|
);
|
|
}
|
|
}
|
|
|
|
/// Exercises the old gallery in a browser for a long period of time, looking
|
|
/// for memory leaks and dangling pointers.
|
|
///
|
|
/// This is not a performance test.
|
|
///
|
|
/// If [canvasKit] is set to true, runs the test in CanvasKit mode.
|
|
///
|
|
/// The test is written using `package:integration_test` (despite the "e2e" in
|
|
/// the name, which is there for historic reasons).
|
|
Future<void> _runGalleryE2eWebTest(String buildMode, { bool canvasKit = false }) async {
|
|
// TODO(yjbanov): this is temporarily disabled due to https://github.com/flutter/flutter/issues/147731
|
|
if (buildMode == 'debug' && canvasKit) {
|
|
print('SKIPPED: Gallery e2e web test in debug CanvasKit mode due to https://github.com/flutter/flutter/issues/147731');
|
|
return;
|
|
}
|
|
printProgress('${green}Running flutter_gallery integration test in --$buildMode using ${canvasKit ? 'CanvasKit' : 'HTML'} renderer.$reset');
|
|
final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery');
|
|
await runCommand(
|
|
flutter,
|
|
<String>[ 'clean' ],
|
|
workingDirectory: testAppDirectory,
|
|
);
|
|
await runCommand(
|
|
flutter,
|
|
<String>[
|
|
...flutterTestArgs,
|
|
'drive',
|
|
if (canvasKit)
|
|
'--dart-define=FLUTTER_WEB_USE_SKIA=true',
|
|
if (!canvasKit)
|
|
'--dart-define=FLUTTER_WEB_USE_SKIA=false',
|
|
if (!canvasKit)
|
|
'--dart-define=FLUTTER_WEB_AUTO_DETECT=false',
|
|
'--driver=test_driver/transitions_perf_e2e_test.dart',
|
|
'--target=test_driver/transitions_perf_e2e.dart',
|
|
'--browser-name=chrome',
|
|
'-d',
|
|
'web-server',
|
|
'--$buildMode',
|
|
],
|
|
workingDirectory: testAppDirectory,
|
|
environment: <String, String>{
|
|
'FLUTTER_WEB': 'true',
|
|
},
|
|
);
|
|
}
|
|
|
|
Future<void> _runWebStackTraceTest(String buildMode, String entrypoint) async {
|
|
final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web');
|
|
final String appBuildDirectory = path.join(testAppDirectory, 'build', 'web');
|
|
|
|
// Build the app.
|
|
await runCommand(
|
|
flutter,
|
|
<String>[ 'clean' ],
|
|
workingDirectory: testAppDirectory,
|
|
);
|
|
await runCommand(
|
|
flutter,
|
|
<String>[
|
|
'build',
|
|
'web',
|
|
'--$buildMode',
|
|
'-t',
|
|
entrypoint,
|
|
],
|
|
workingDirectory: testAppDirectory,
|
|
environment: <String, String>{
|
|
'FLUTTER_WEB': 'true',
|
|
},
|
|
);
|
|
|
|
// Run the app.
|
|
final int serverPort = await findAvailablePortAndPossiblyCauseFlakyTests();
|
|
final int browserDebugPort = await findAvailablePortAndPossiblyCauseFlakyTests();
|
|
final String result = await evalTestAppInChrome(
|
|
appUrl: 'http://localhost:$serverPort/index.html',
|
|
appDirectory: appBuildDirectory,
|
|
serverPort: serverPort,
|
|
browserDebugPort: browserDebugPort,
|
|
);
|
|
|
|
if (!result.contains('--- TEST SUCCEEDED ---')) {
|
|
foundError(<String>[
|
|
result,
|
|
'${red}Web stack trace integration test failed.$reset',
|
|
]);
|
|
}
|
|
}
|
|
|
|
/// Debug mode is special because `flutter build web` doesn't build in debug mode.
|
|
///
|
|
/// Instead, we use `flutter run --debug` and sniff out the standard output.
|
|
Future<void> _runWebDebugTest(String target, {
|
|
List<String> additionalArguments = const<String>[],
|
|
}) async {
|
|
final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web');
|
|
bool success = false;
|
|
final Map<String, String> environment = <String, String>{
|
|
'FLUTTER_WEB': 'true',
|
|
};
|
|
adjustEnvironmentToEnableFlutterAsserts(environment);
|
|
final CommandResult result = await runCommand(
|
|
flutter,
|
|
<String>[
|
|
'run',
|
|
'--verbose',
|
|
'--debug',
|
|
'-d',
|
|
'chrome',
|
|
'--web-run-headless',
|
|
'--dart-define=FLUTTER_WEB_USE_SKIA=false',
|
|
'--dart-define=FLUTTER_WEB_AUTO_DETECT=false',
|
|
...additionalArguments,
|
|
'-t',
|
|
target,
|
|
],
|
|
outputMode: OutputMode.capture,
|
|
outputListener: (String line, Process process) {
|
|
bool shutdownFlutterTool = false;
|
|
if (line.contains('--- TEST SUCCEEDED ---')) {
|
|
success = true;
|
|
shutdownFlutterTool = true;
|
|
}
|
|
if (line.contains('--- TEST FAILED ---')) {
|
|
shutdownFlutterTool = true;
|
|
}
|
|
if (shutdownFlutterTool) {
|
|
process.stdin.add('q'.codeUnits);
|
|
}
|
|
},
|
|
workingDirectory: testAppDirectory,
|
|
environment: environment,
|
|
);
|
|
|
|
if (!success) {
|
|
foundError(<String>[
|
|
result.flattenedStdout!,
|
|
result.flattenedStderr!,
|
|
'${red}Web stack trace integration test failed.$reset',
|
|
]);
|
|
}
|
|
}
|
|
|
|
/// Run a web integration test in release mode.
|
|
Future<void> _runWebReleaseTest(String target, {
|
|
List<String> additionalArguments = const<String>[],
|
|
}) async {
|
|
final String testAppDirectory = path.join(flutterRoot, 'dev', 'integration_tests', 'web');
|
|
final String appBuildDirectory = path.join(testAppDirectory, 'build', 'web');
|
|
|
|
// Build the app.
|
|
await runCommand(
|
|
flutter,
|
|
<String>[ 'clean' ],
|
|
workingDirectory: testAppDirectory,
|
|
);
|
|
await runCommand(
|
|
flutter,
|
|
<String>[
|
|
...flutterTestArgs,
|
|
'build',
|
|
'web',
|
|
'--release',
|
|
...additionalArguments,
|
|
'-t',
|
|
target,
|
|
],
|
|
workingDirectory: testAppDirectory,
|
|
environment: <String, String>{
|
|
'FLUTTER_WEB': 'true',
|
|
},
|
|
);
|
|
|
|
// Run the app.
|
|
final int serverPort = await findAvailablePortAndPossiblyCauseFlakyTests();
|
|
final int browserDebugPort = await findAvailablePortAndPossiblyCauseFlakyTests();
|
|
final String result = await evalTestAppInChrome(
|
|
appUrl: 'http://localhost:$serverPort/index.html',
|
|
appDirectory: appBuildDirectory,
|
|
serverPort: serverPort,
|
|
browserDebugPort: browserDebugPort,
|
|
);
|
|
|
|
if (!result.contains('--- TEST SUCCEEDED ---')) {
|
|
foundError(<String>[
|
|
result,
|
|
'${red}Web release mode test failed.$reset',
|
|
]);
|
|
}
|
|
}
|
|
|
|
Future<void> _runWebUnitTests(String webRenderer, bool useWasm) async {
|
|
final Map<String, ShardRunner> subshards = <String, ShardRunner>{};
|
|
|
|
final Directory flutterPackageDirectory = Directory(path.join(flutterRoot, 'packages', 'flutter'));
|
|
final Directory flutterPackageTestDirectory = Directory(path.join(flutterPackageDirectory.path, 'test'));
|
|
|
|
final List<String> allTests = flutterPackageTestDirectory
|
|
.listSync()
|
|
.whereType<Directory>()
|
|
.expand((Directory directory) => directory
|
|
.listSync(recursive: true)
|
|
.where((FileSystemEntity entity) => entity.path.endsWith('_test.dart'))
|
|
)
|
|
.whereType<File>()
|
|
.map<String>((File file) => path.relative(file.path, from: flutterPackageDirectory.path))
|
|
.where((String filePath) => !kWebTestFileKnownFailures[webRenderer]!.contains(path.split(filePath).join('/')))
|
|
.toList()
|
|
// Finally we shuffle the list because we want the average cost per file to be uniformly
|
|
// distributed. If the list is not sorted then different shards and batches may have
|
|
// very different characteristics.
|
|
// We use a constant seed for repeatability.
|
|
..shuffle(math.Random(0));
|
|
|
|
assert(webShardCount >= 1);
|
|
final int testsPerShard = (allTests.length / webShardCount).ceil();
|
|
assert(testsPerShard * webShardCount >= allTests.length);
|
|
|
|
// This for loop computes all but the last shard.
|
|
for (int index = 0; index < webShardCount - 1; index += 1) {
|
|
subshards['$index'] = () => _runFlutterWebTest(
|
|
webRenderer,
|
|
flutterPackageDirectory.path,
|
|
allTests.sublist(
|
|
index * testsPerShard,
|
|
(index + 1) * testsPerShard,
|
|
),
|
|
useWasm,
|
|
);
|
|
}
|
|
|
|
// The last shard also runs the flutter_web_plugins tests.
|
|
//
|
|
// We make sure the last shard ends in _last so it's easier to catch mismatches
|
|
// between `.cirrus.yml` and `test.dart`.
|
|
subshards['${webShardCount - 1}_last'] = () async {
|
|
await _runFlutterWebTest(
|
|
webRenderer,
|
|
flutterPackageDirectory.path,
|
|
allTests.sublist(
|
|
(webShardCount - 1) * testsPerShard,
|
|
allTests.length,
|
|
),
|
|
useWasm,
|
|
);
|
|
await _runFlutterWebTest(
|
|
webRenderer,
|
|
path.join(flutterRoot, 'packages', 'flutter_web_plugins'),
|
|
<String>['test'],
|
|
useWasm,
|
|
);
|
|
await _runFlutterWebTest(
|
|
webRenderer,
|
|
path.join(flutterRoot, 'packages', 'flutter_driver'),
|
|
<String>[path.join('test', 'src', 'web_tests', 'web_extension_test.dart')],
|
|
useWasm,
|
|
);
|
|
};
|
|
|
|
await selectSubshard(subshards);
|
|
}
|
|
|
|
Future<void> _runFlutterWebTest(
|
|
String webRenderer,
|
|
String workingDirectory,
|
|
List<String> tests,
|
|
bool useWasm,
|
|
) async {
|
|
const LocalFileSystem fileSystem = LocalFileSystem();
|
|
final String suffix = DateTime.now().microsecondsSinceEpoch.toString();
|
|
final File metricFile = fileSystem.systemTempDirectory.childFile('metrics_$suffix.json');
|
|
await runCommand(
|
|
flutter,
|
|
<String>[
|
|
'test',
|
|
'--reporter=expanded',
|
|
'--file-reporter=json:${metricFile.path}',
|
|
'-v',
|
|
'--platform=chrome',
|
|
if (useWasm) '--wasm',
|
|
'--web-renderer=$webRenderer',
|
|
'--dart-define=DART_HHH_BOT=$runningInDartHHHBot',
|
|
...flutterTestArgs,
|
|
...tests,
|
|
],
|
|
workingDirectory: workingDirectory,
|
|
environment: <String, String>{
|
|
'FLUTTER_WEB': 'true',
|
|
},
|
|
);
|
|
// metriciFile is a transitional file that needs to be deleted once it is parsed.
|
|
// TODO(godofredoc): Ensure metricFile is parsed and aggregated before deleting.
|
|
// https://github.com/flutter/flutter/issues/146003
|
|
metricFile.deleteSync();
|
|
}
|
|
|
|
// The `chromedriver` process created by this test.
|
|
//
|
|
// If an existing chromedriver is already available on port 4444, the existing
|
|
// process is reused and this variable remains null.
|
|
Command? _chromeDriver;
|
|
|
|
Future<bool> _isChromeDriverRunning() async {
|
|
try {
|
|
final RawSocket socket = await RawSocket.connect('localhost', 4444);
|
|
socket.shutdown(SocketDirection.both);
|
|
await socket.close();
|
|
return true;
|
|
} on SocketException {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
Future<void> _stopChromeDriver() async {
|
|
if (_chromeDriver == null) {
|
|
return;
|
|
}
|
|
print('Stopping chromedriver');
|
|
_chromeDriver!.process.kill();
|
|
}
|
|
|
|
Future<void> _ensureChromeDriverIsRunning() async {
|
|
// If we cannot connect to ChromeDriver, assume it is not running. Launch it.
|
|
if (!await _isChromeDriverRunning()) {
|
|
printProgress('Starting chromedriver');
|
|
// Assume chromedriver is in the PATH.
|
|
_chromeDriver = await startCommand(
|
|
// TODO(ianh): this is the only remaining consumer of startCommand other than runCommand
|
|
// and it doesn't use most of startCommand's features; we could simplify this a lot by
|
|
// inlining the relevant parts of startCommand here.
|
|
'chromedriver',
|
|
<String>['--port=4444', '--log-level=INFO', '--enable-chrome-logs'],
|
|
);
|
|
while (!await _isChromeDriverRunning()) {
|
|
await Future<void>.delayed(const Duration(milliseconds: 100));
|
|
print('Waiting for chromedriver to start up.');
|
|
}
|
|
}
|
|
|
|
final HttpClient client = HttpClient();
|
|
final Uri chromeDriverUrl = Uri.parse('http://localhost:4444/status');
|
|
final HttpClientRequest request = await client.getUrl(chromeDriverUrl);
|
|
final HttpClientResponse response = await request.close();
|
|
final String responseString = await response.transform(utf8.decoder).join();
|
|
final Map<String, dynamic> webDriverStatus = json.decode(responseString) as Map<String, dynamic>;
|
|
client.close();
|
|
final bool webDriverReady = (webDriverStatus['value'] as Map<String, dynamic>)['ready'] as bool;
|
|
if (!webDriverReady) {
|
|
throw Exception('WebDriver not available.');
|
|
}
|
|
}
|
|
}
|