// Copyright 2014 The Flutter Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. import 'dart:io' show File, Platform; import 'package:path/path.dart' as path; import '../run_command.dart'; import '../utils.dart'; String get platformFolderName { if (Platform.isWindows) { return 'windows-x64'; } if (Platform.isMacOS) { return 'darwin-x64'; } if (Platform.isLinux) { return 'linux-x64'; } throw UnsupportedError('The platform ${Platform.operatingSystem} is not supported by this script.'); } Future testHarnessTestsRunner() async { printProgress('${green}Running test harness tests...$reset'); await _validateEngineHash(); // Verify that the tests actually return failure on failure and success on // success. final String automatedTests = path.join(flutterRoot, 'dev', 'automated_tests'); // We want to run these tests in parallel, because they each take some time // to run (e.g. compiling), so we don't want to run them in series, especially // on 20-core machines. However, we have a race condition, so for now... // Race condition issue: https://github.com/flutter/flutter/issues/90026 final List tests = [ () => runFlutterTest( automatedTests, script: path.join('test_smoke_test', 'pass_test.dart'), printOutput: false, ), () => runFlutterTest( automatedTests, script: path.join('test_smoke_test', 'fail_test.dart'), expectFailure: true, printOutput: false, ), () => runFlutterTest( automatedTests, script: path.join('test_smoke_test', 'pending_timer_fail_test.dart'), expectFailure: true, printOutput: false, outputChecker: (CommandResult result) { return result.flattenedStdout!.contains('failingPendingTimerTest') ? null : 'Failed to find the stack trace for the pending Timer.\n\n' 'stdout:\n${result.flattenedStdout}\n\n' 'stderr:\n${result.flattenedStderr}'; }, ), () => runFlutterTest( automatedTests, script: path.join('test_smoke_test', 'fail_test_on_exception_after_test.dart'), expectFailure: true, printOutput: false, outputChecker: (CommandResult result) { const String expectedError = '══╡ EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK ╞════════════════════════════════════════════════════\n' 'The following StateError was thrown running a test (but after the test had completed):\n' 'Bad state: Exception thrown after test completed.'; if (result.flattenedStdout!.contains(expectedError)) { return null; } return 'Failed to find expected output on stdout.\n\n' 'Expected output:\n$expectedError\n\n' 'Actual stdout:\n${result.flattenedStdout}\n\n' 'Actual stderr:\n${result.flattenedStderr}'; }, ), () => runFlutterTest( automatedTests, script: path.join('test_smoke_test', 'crash1_test.dart'), expectFailure: true, printOutput: false, ), () => runFlutterTest( automatedTests, script: path.join('test_smoke_test', 'crash2_test.dart'), expectFailure: true, printOutput: false, ), () => runFlutterTest( automatedTests, script: path.join('test_smoke_test', 'syntax_error_test.broken_dart'), expectFailure: true, printOutput: false, ), () => runFlutterTest( automatedTests, script: path.join('test_smoke_test', 'missing_import_test.broken_dart'), expectFailure: true, printOutput: false, ), () => runFlutterTest( automatedTests, script: path.join('test_smoke_test', 'disallow_error_reporter_modification_test.dart'), expectFailure: true, printOutput: false, ), ]; List testsToRun; // Run all tests unless sharding is explicitly specified. final String? shardName = Platform.environment[kShardKey]; if (shardName == kTestHarnessShardName) { testsToRun = selectIndexOfTotalSubshard(tests); } else { testsToRun = tests; } for (final ShardRunner test in testsToRun) { await test(); } // Verify that we correctly generated the version file. final String? versionError = await verifyVersion(File(path.join(flutterRoot, 'version'))); if (versionError != null) { foundError([versionError]); } } /// Verify the Flutter Engine is the revision in /// bin/cache/internal/engine.version. Future _validateEngineHash() async { final String flutterTester = path.join(flutterRoot, 'bin', 'cache', 'artifacts', 'engine', platformFolderName, 'flutter_tester$exe'); if (runningInDartHHHBot) { // The Dart HHH bots intentionally modify the local artifact cache // and then use this script to run Flutter's test suites. // Because the artifacts have been changed, this particular test will return // a false positive and should be skipped. print('${yellow}Skipping Flutter Engine Version Validation for swarming bot $luciBotId.'); return; } final String expectedVersion = File(engineVersionFile).readAsStringSync().trim(); final CommandResult result = await runCommand(flutterTester, ['--help'], outputMode: OutputMode.capture); if (result.flattenedStdout!.isNotEmpty) { foundError([ '${red}The stdout of `$flutterTester --help` was not empty:$reset', ...result.flattenedStdout!.split('\n').map((String line) => ' $gray┆$reset $line'), ]); } final String actualVersion; try { actualVersion = result.flattenedStderr!.split('\n').firstWhere((final String line) { return line.startsWith('Flutter Engine Version:'); }); } on StateError { foundError([ '${red}Could not find "Flutter Engine Version:" line in `${path.basename(flutterTester)} --help` stderr output:$reset', ...result.flattenedStderr!.split('\n').map((String line) => ' $gray┆$reset $line'), ]); return; } if (!actualVersion.contains(expectedVersion)) { foundError(['${red}Expected "Flutter Engine Version: $expectedVersion", but found "$actualVersion".$reset']); } }