diff --git a/dev/bots/analyze.dart b/dev/bots/analyze.dart index 9ef8cb8c343..7f47bf476eb 100644 --- a/dev/bots/analyze.dart +++ b/dev/bots/analyze.dart @@ -26,8 +26,6 @@ import 'custom_rules/render_box_intrinsics.dart'; import 'run_command.dart'; import 'utils.dart'; -final String flutterRoot = path.dirname(path.dirname(path.dirname(path.fromUri(Platform.script)))); -final String flutter = path.join(flutterRoot, 'bin', Platform.isWindows ? 'flutter.bat' : 'flutter'); final String flutterPackages = path.join(flutterRoot, 'packages'); final String flutterExamples = path.join(flutterRoot, 'examples'); diff --git a/dev/bots/suite_runners/run_framework_tests.dart b/dev/bots/suite_runners/run_framework_tests.dart new file mode 100644 index 00000000000..d14ecd1d5d3 --- /dev/null +++ b/dev/bots/suite_runners/run_framework_tests.dart @@ -0,0 +1,307 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:convert'; +import 'dart:io' show Directory, File, FileSystemEntity, Platform, Process; +import 'dart:typed_data'; + +import 'package:archive/archive.dart'; +import 'package:path/path.dart' as path; + +import '../run_command.dart'; +import '../utils.dart'; +import 'run_test_harness_tests.dart'; + +Future frameworkTestsRunner() async { + final List trackWidgetCreationAlternatives = ['--track-widget-creation', '--no-track-widget-creation']; + + Future runWidgets() async { + printProgress('${green}Running packages/flutter tests $reset for ${cyan}test/widgets/$reset'); + for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) { + await runFlutterTest( + path.join(flutterRoot, 'packages', 'flutter'), + options: [trackWidgetCreationOption], + tests: [ path.join('test', 'widgets') + path.separator ], + ); + } + // Try compiling code outside of the packages/flutter directory with and without --track-widget-creation + for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) { + await runFlutterTest( + path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery'), + options: [trackWidgetCreationOption], + fatalWarnings: false, // until we've migrated video_player + ); + } + // Run release mode tests (see packages/flutter/test_release/README.md) + await runFlutterTest( + path.join(flutterRoot, 'packages', 'flutter'), + options: ['--dart-define=dart.vm.product=true'], + tests: ['test_release${path.separator}'], + ); + // Run profile mode tests (see packages/flutter/test_profile/README.md) + await runFlutterTest( + path.join(flutterRoot, 'packages', 'flutter'), + options: ['--dart-define=dart.vm.product=false', '--dart-define=dart.vm.profile=true'], + tests: ['test_profile${path.separator}'], + ); + } + + Future runImpeller() async { + printProgress('${green}Running packages/flutter tests $reset in Impeller$reset'); + await runFlutterTest( + path.join(flutterRoot, 'packages', 'flutter'), + options: ['--enable-impeller'], + ); + } + + + Future runLibraries() async { + final List tests = Directory(path.join(flutterRoot, 'packages', 'flutter', 'test')) + .listSync(followLinks: false) + .whereType() + .where((Directory dir) => !dir.path.endsWith('widgets')) + .map((Directory dir) => path.join('test', path.basename(dir.path)) + path.separator) + .toList(); + printProgress('${green}Running packages/flutter tests$reset for $cyan${tests.join(", ")}$reset'); + for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) { + await runFlutterTest( + path.join(flutterRoot, 'packages', 'flutter'), + options: [trackWidgetCreationOption], + tests: tests, + ); + } + } + + Future runExampleTests() async { + await runCommand( + flutter, + ['config', '--enable-${Platform.operatingSystem}-desktop'], + workingDirectory: flutterRoot, + ); + await runCommand( + dart, + [path.join(flutterRoot, 'dev', 'tools', 'examples_smoke_test.dart')], + workingDirectory: path.join(flutterRoot, 'examples', 'api'), + ); + for (final FileSystemEntity entity in Directory(path.join(flutterRoot, 'examples')).listSync()) { + if (entity is! Directory || !Directory(path.join(entity.path, 'test')).existsSync()) { + continue; + } + await runFlutterTest(entity.path); + } + } + + Future runTracingTests() async { + final String tracingDirectory = path.join(flutterRoot, 'dev', 'tracing_tests'); + + // run the tests for debug mode + await runFlutterTest(tracingDirectory, options: ['--enable-vmservice']); + + Future> verifyTracingAppBuild({ + required String modeArgument, + required String sourceFile, + required Set allowed, + required Set disallowed, + }) async { + try { + await runCommand( + flutter, + [ + 'build', 'appbundle', '--$modeArgument', path.join('lib', sourceFile), + ], + workingDirectory: tracingDirectory, + ); + final Archive archive = ZipDecoder().decodeBytes(File(path.join(tracingDirectory, 'build', 'app', 'outputs', 'bundle', modeArgument, 'app-$modeArgument.aab')).readAsBytesSync()); + final ArchiveFile libapp = archive.findFile('base/lib/arm64-v8a/libapp.so')!; + final Uint8List libappBytes = libapp.content as Uint8List; // bytes decompressed here + final String libappStrings = utf8.decode(libappBytes, allowMalformed: true); + await runCommand(flutter, ['clean'], workingDirectory: tracingDirectory); + final List results = []; + for (final String pattern in allowed) { + if (!libappStrings.contains(pattern)) { + results.add('When building with --$modeArgument, expected to find "$pattern" in libapp.so but could not find it.'); + } + } + for (final String pattern in disallowed) { + if (libappStrings.contains(pattern)) { + results.add('When building with --$modeArgument, expected to not find "$pattern" in libapp.so but did find it.'); + } + } + return results; + } catch (error, stackTrace) { + return [ + error.toString(), + ...stackTrace.toString().trimRight().split('\n'), + ]; + } + } + + final List results = []; + results.addAll(await verifyTracingAppBuild( + modeArgument: 'profile', + sourceFile: 'control.dart', // this is the control, the other two below are the actual test + allowed: { + 'TIMELINE ARGUMENTS TEST CONTROL FILE', + 'toTimelineArguments used in non-debug build', // we call toTimelineArguments directly to check the message does exist + }, + disallowed: { + 'BUILT IN DEBUG MODE', 'BUILT IN RELEASE MODE', + }, + )); + results.addAll(await verifyTracingAppBuild( + modeArgument: 'profile', + sourceFile: 'test.dart', + allowed: { + 'BUILT IN PROFILE MODE', 'RenderTest.performResize called', // controls + 'BUILD', 'LAYOUT', 'PAINT', // we output these to the timeline in profile builds + // (LAYOUT and PAINT also exist because of NEEDS-LAYOUT and NEEDS-PAINT in RenderObject.toStringShort) + }, + disallowed: { + 'BUILT IN DEBUG MODE', 'BUILT IN RELEASE MODE', + 'TestWidget.debugFillProperties called', 'RenderTest.debugFillProperties called', // debug only + 'toTimelineArguments used in non-debug build', // entire function should get dropped by tree shaker + }, + )); + results.addAll(await verifyTracingAppBuild( + modeArgument: 'release', + sourceFile: 'test.dart', + allowed: { + 'BUILT IN RELEASE MODE', 'RenderTest.performResize called', // controls + }, + disallowed: { + 'BUILT IN DEBUG MODE', 'BUILT IN PROFILE MODE', + 'BUILD', 'LAYOUT', 'PAINT', // these are only used in Timeline.startSync calls that should not appear in release builds + 'TestWidget.debugFillProperties called', 'RenderTest.debugFillProperties called', // debug only + 'toTimelineArguments used in non-debug build', // not included in release builds + }, + )); + if (results.isNotEmpty) { + foundError(results); + } + } + + Future runFixTests(String package) async { + final List args = [ + 'fix', + '--compare-to-golden', + ]; + await runCommand( + dart, + args, + workingDirectory: path.join(flutterRoot, 'packages', package, 'test_fixes'), + ); + } + + Future runPrivateTests() async { + final List args = [ + 'run', + 'bin/test_private.dart', + ]; + final Map environment = { + 'FLUTTER_ROOT': flutterRoot, + if (Directory(pubCache).existsSync()) + 'PUB_CACHE': pubCache, + }; + adjustEnvironmentToEnableFlutterAsserts(environment); + await runCommand( + dart, + args, + workingDirectory: path.join(flutterRoot, 'packages', 'flutter', 'test_private'), + environment: environment, + ); + } + + // Tests that take longer than average to run. This is usually because they + // need to compile something large or make use of the analyzer for the test. + // These tests need to be platform agnostic as they are only run on a linux + // machine to save on execution time and cost. + Future runSlow() async { + printProgress('${green}Running slow package tests$reset for directories other than packages/flutter'); + await runTracingTests(); + await runFixTests('flutter'); + await runFixTests('flutter_test'); + await runFixTests('integration_test'); + await runFixTests('flutter_driver'); + await runPrivateTests(); + } + + Future runMisc() async { + printProgress('${green}Running package tests$reset for directories other than packages/flutter'); + await testHarnessTestsRunner(); + await runExampleTests(); + await runFlutterTest( + path.join(flutterRoot, 'dev', 'a11y_assessments'), + tests: [ 'test' ], + ); + await runDartTest(path.join(flutterRoot, 'dev', 'bots')); + await runDartTest(path.join(flutterRoot, 'dev', 'devicelab'), ensurePrecompiledTool: false); // See https://github.com/flutter/flutter/issues/86209 + await runDartTest(path.join(flutterRoot, 'dev', 'conductor', 'core'), forceSingleCore: true); + // TODO(gspencergoog): Remove the exception for fatalWarnings once https://github.com/flutter/flutter/issues/113782 has landed. + await runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing'), fatalWarnings: false); + await runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'ui')); + await runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests')); + await runFlutterTest(path.join(flutterRoot, 'dev', 'tools')); + await runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'vitool')); + await runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'gen_defaults')); + await runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'gen_keycodes')); + await runFlutterTest(path.join(flutterRoot, 'dev', 'benchmarks', 'test_apps', 'stocks')); + await runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'), tests: [path.join('test', 'src', 'real_tests')]); + await runFlutterTest(path.join(flutterRoot, 'packages', 'integration_test'), options: [ + '--enable-vmservice', + // Web-specific tests depend on Chromium, so they run as part of the web_long_running_tests shard. + '--exclude-tags=web', + ]); + // Run java unit tests for integration_test + // + // Generate Gradle wrapper if it doesn't exist. + Process.runSync( + flutter, + ['build', 'apk', '--config-only'], + workingDirectory: path.join(flutterRoot, 'packages', 'integration_test', 'example', 'android'), + ); + await runCommand( + path.join(flutterRoot, 'packages', 'integration_test', 'example', 'android', 'gradlew$bat'), + [ + ':integration_test:testDebugUnitTest', + '--tests', + 'dev.flutter.plugins.integration_test.FlutterDeviceScreenshotTest', + ], + workingDirectory: path.join(flutterRoot, 'packages', 'integration_test', 'example', 'android'), + ); + await runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_goldens')); + await runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_localizations')); + await runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test')); + await runFlutterTest(path.join(flutterRoot, 'packages', 'fuchsia_remote_debug_protocol')); + await runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'non_nullable')); + const String httpClientWarning = + 'Warning: At least one test in this suite creates an HttpClient. When running a test suite that uses\n' + 'TestWidgetsFlutterBinding, all HTTP requests will return status code 400, and no network request\n' + 'will actually be made. Any test expecting a real network connection and status code will fail.\n' + 'To test code that needs an HttpClient, provide your own HttpClient implementation to the code under\n' + 'test, so that your test can consistently provide a testable response to the code under test.'; + await runFlutterTest( + path.join(flutterRoot, 'packages', 'flutter_test'), + script: path.join('test', 'bindings_test_failure.dart'), + expectFailure: true, + printOutput: false, + outputChecker: (CommandResult result) { + final Iterable matches = httpClientWarning.allMatches(result.flattenedStdout!); + if (matches.isEmpty || matches.length > 1) { + return 'Failed to print warning about HttpClientUsage, or printed it too many times.\n\n' + 'stdout:\n${result.flattenedStdout}\n\n' + 'stderr:\n${result.flattenedStderr}'; + } + return null; + }, + ); + } + + await selectSubshard({ + 'widgets': runWidgets, + 'libraries': runLibraries, + 'slow': runSlow, + 'misc': runMisc, + 'impeller': runImpeller, + }); +} diff --git a/dev/bots/suite_runners/run_test_harness_tests.dart b/dev/bots/suite_runners/run_test_harness_tests.dart new file mode 100644 index 00000000000..9ebc2d006c9 --- /dev/null +++ b/dev/bots/suite_runners/run_test_harness_tests.dart @@ -0,0 +1,170 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:io' show File, Platform; + +import 'package:path/path.dart' as path; + +import '../run_command.dart'; +import '../utils.dart'; + +String get platformFolderName { + if (Platform.isWindows) { + return 'windows-x64'; + } + if (Platform.isMacOS) { + return 'darwin-x64'; + } + if (Platform.isLinux) { + return 'linux-x64'; + } + throw UnsupportedError('The platform ${Platform.operatingSystem} is not supported by this script.'); +} + +Future testHarnessTestsRunner() async { + + printProgress('${green}Running test harness tests...$reset'); + + await _validateEngineHash(); + + // Verify that the tests actually return failure on failure and success on + // success. + final String automatedTests = path.join(flutterRoot, 'dev', 'automated_tests'); + + // We want to run these tests in parallel, because they each take some time + // to run (e.g. compiling), so we don't want to run them in series, especially + // on 20-core machines. However, we have a race condition, so for now... + // Race condition issue: https://github.com/flutter/flutter/issues/90026 + final List tests = [ + () => runFlutterTest( + automatedTests, + script: path.join('test_smoke_test', 'pass_test.dart'), + printOutput: false, + ), + () => runFlutterTest( + automatedTests, + script: path.join('test_smoke_test', 'fail_test.dart'), + expectFailure: true, + printOutput: false, + ), + () => runFlutterTest( + automatedTests, + script: path.join('test_smoke_test', 'pending_timer_fail_test.dart'), + expectFailure: true, + printOutput: false, + outputChecker: (CommandResult result) { + return result.flattenedStdout!.contains('failingPendingTimerTest') + ? null + : 'Failed to find the stack trace for the pending Timer.\n\n' + 'stdout:\n${result.flattenedStdout}\n\n' + 'stderr:\n${result.flattenedStderr}'; + }, + ), + () => runFlutterTest( + automatedTests, + script: path.join('test_smoke_test', 'fail_test_on_exception_after_test.dart'), + expectFailure: true, + printOutput: false, + outputChecker: (CommandResult result) { + const String expectedError = '══╡ EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK ╞════════════════════════════════════════════════════\n' + 'The following StateError was thrown running a test (but after the test had completed):\n' + 'Bad state: Exception thrown after test completed.'; + if (result.flattenedStdout!.contains(expectedError)) { + return null; + } + return 'Failed to find expected output on stdout.\n\n' + 'Expected output:\n$expectedError\n\n' + 'Actual stdout:\n${result.flattenedStdout}\n\n' + 'Actual stderr:\n${result.flattenedStderr}'; + }, + ), + () => runFlutterTest( + automatedTests, + script: path.join('test_smoke_test', 'crash1_test.dart'), + expectFailure: true, + printOutput: false, + ), + () => runFlutterTest( + automatedTests, + script: path.join('test_smoke_test', 'crash2_test.dart'), + expectFailure: true, + printOutput: false, + ), + () => runFlutterTest( + automatedTests, + script: path.join('test_smoke_test', 'syntax_error_test.broken_dart'), + expectFailure: true, + printOutput: false, + ), + () => runFlutterTest( + automatedTests, + script: path.join('test_smoke_test', 'missing_import_test.broken_dart'), + expectFailure: true, + printOutput: false, + ), + () => runFlutterTest( + automatedTests, + script: path.join('test_smoke_test', 'disallow_error_reporter_modification_test.dart'), + expectFailure: true, + printOutput: false, + ), + ]; + + List testsToRun; + + // Run all tests unless sharding is explicitly specified. + final String? shardName = Platform.environment[kShardKey]; + if (shardName == kTestHarnessShardName) { + testsToRun = selectIndexOfTotalSubshard(tests); + } else { + testsToRun = tests; + } + for (final ShardRunner test in testsToRun) { + await test(); + } + + // Verify that we correctly generated the version file. + final String? versionError = await verifyVersion(File(path.join(flutterRoot, 'version'))); + if (versionError != null) { + foundError([versionError]); + } +} + +/// Verify the Flutter Engine is the revision in +/// bin/cache/internal/engine.version. +Future _validateEngineHash() async { + final String flutterTester = path.join(flutterRoot, 'bin', 'cache', 'artifacts', 'engine', platformFolderName, 'flutter_tester$exe'); + + if (runningInDartHHHBot) { + // The Dart HHH bots intentionally modify the local artifact cache + // and then use this script to run Flutter's test suites. + // Because the artifacts have been changed, this particular test will return + // a false positive and should be skipped. + print('${yellow}Skipping Flutter Engine Version Validation for swarming bot $luciBotId.'); + return; + } + final String expectedVersion = File(engineVersionFile).readAsStringSync().trim(); + final CommandResult result = await runCommand(flutterTester, ['--help'], outputMode: OutputMode.capture); + if (result.flattenedStdout!.isNotEmpty) { + foundError([ + '${red}The stdout of `$flutterTester --help` was not empty:$reset', + ...result.flattenedStdout!.split('\n').map((String line) => ' $gray┆$reset $line'), + ]); + } + final String actualVersion; + try { + actualVersion = result.flattenedStderr!.split('\n').firstWhere((final String line) { + return line.startsWith('Flutter Engine Version:'); + }); + } on StateError { + foundError([ + '${red}Could not find "Flutter Engine Version:" line in `${path.basename(flutterTester)} --help` stderr output:$reset', + ...result.flattenedStderr!.split('\n').map((String line) => ' $gray┆$reset $line'), + ]); + return; + } + if (!actualVersion.contains(expectedVersion)) { + foundError(['${red}Expected "Flutter Engine Version: $expectedVersion", but found "$actualVersion".$reset']); + } +} diff --git a/dev/bots/test.dart b/dev/bots/test.dart index 3d185103dab..86312350ea2 100644 --- a/dev/bots/test.dart +++ b/dev/bots/test.dart @@ -53,9 +53,7 @@ import 'dart:core' hide print; import 'dart:io' as system show exit; import 'dart:io' hide exit; import 'dart:math' as math; -import 'dart:typed_data'; -import 'package:archive/archive.dart'; import 'package:path/path.dart' as path; import 'run_command.dart'; @@ -66,35 +64,21 @@ import 'suite_runners/run_customer_testing_tests.dart'; import 'suite_runners/run_docs_tests.dart'; import 'suite_runners/run_flutter_packages_tests.dart'; import 'suite_runners/run_framework_coverage_tests.dart'; +import 'suite_runners/run_framework_tests.dart'; import 'suite_runners/run_fuchsia_precache.dart'; import 'suite_runners/run_realm_checker_tests.dart'; import 'suite_runners/run_skp_generator_tests.dart'; +import 'suite_runners/run_test_harness_tests.dart'; import 'suite_runners/run_verify_binaries_codesigned_tests.dart'; import 'suite_runners/run_web_tests.dart'; import 'utils.dart'; typedef ShardRunner = Future Function(); -String get platformFolderName { - if (Platform.isWindows) { - return 'windows-x64'; - } - if (Platform.isMacOS) { - return 'darwin-x64'; - } - if (Platform.isLinux) { - return 'linux-x64'; - } - throw UnsupportedError('The platform ${Platform.operatingSystem} is not supported by this script.'); -} -final String flutterTester = path.join(flutterRoot, 'bin', 'cache', 'artifacts', 'engine', platformFolderName, 'flutter_tester$exe'); - /// Environment variables to override the local engine when running `pub test`, /// if such flags are provided to `test.dart`. final Map localEngineEnv = {}; -const String kTestHarnessShardName = 'test_harness_tests'; - const String CIRRUS_TASK_NAME = 'CIRRUS_TASK_NAME'; /// When you call this, you can pass additional arguments to pass custom @@ -142,7 +126,7 @@ Future main(List args) async { 'add_to_app_life_cycle_tests': addToAppLifeCycleRunner, 'build_tests': _runBuildTests, 'framework_coverage': frameworkCoverageRunner, - 'framework_tests': _runFrameworkTests, + 'framework_tests': frameworkTestsRunner, 'tool_tests': _runToolTests, 'web_tool_tests': _runWebToolTests, 'tool_integration_tests': _runIntegrationToolTests, @@ -164,7 +148,7 @@ Future main(List args) async { 'fuchsia_precache': fuchsiaPrecacheRunner, 'docs': docsRunner, 'verify_binaries_codesigned': verifyCodesignedTestRunner, - kTestHarnessShardName: _runTestHarnessTests, // Used for testing this script; also run as part of SHARD=framework_tests, SUBSHARD=misc. + kTestHarnessShardName: testHarnessTestsRunner, // Used for testing this script; also run as part of SHARD=framework_tests, SUBSHARD=misc. }); } catch (error, stackTrace) { foundError([ @@ -182,150 +166,6 @@ Future main(List args) async { reportSuccessAndExit('${bold}Test successful.$reset'); } -/// Verify the Flutter Engine is the revision in -/// bin/cache/internal/engine.version. -Future _validateEngineHash() async { - if (runningInDartHHHBot) { - // The Dart HHH bots intentionally modify the local artifact cache - // and then use this script to run Flutter's test suites. - // Because the artifacts have been changed, this particular test will return - // a false positive and should be skipped. - print('${yellow}Skipping Flutter Engine Version Validation for swarming bot $luciBotId.'); - return; - } - final String expectedVersion = File(engineVersionFile).readAsStringSync().trim(); - final CommandResult result = await runCommand(flutterTester, ['--help'], outputMode: OutputMode.capture); - if (result.flattenedStdout!.isNotEmpty) { - foundError([ - '${red}The stdout of `$flutterTester --help` was not empty:$reset', - ...result.flattenedStdout!.split('\n').map((String line) => ' $gray┆$reset $line'), - ]); - } - final String actualVersion; - try { - actualVersion = result.flattenedStderr!.split('\n').firstWhere((final String line) { - return line.startsWith('Flutter Engine Version:'); - }); - } on StateError { - foundError([ - '${red}Could not find "Flutter Engine Version:" line in `${path.basename(flutterTester)} --help` stderr output:$reset', - ...result.flattenedStderr!.split('\n').map((String line) => ' $gray┆$reset $line'), - ]); - return; - } - if (!actualVersion.contains(expectedVersion)) { - foundError(['${red}Expected "Flutter Engine Version: $expectedVersion", but found "$actualVersion".$reset']); - } -} - -Future _runTestHarnessTests() async { - printProgress('${green}Running test harness tests...$reset'); - - await _validateEngineHash(); - - // Verify that the tests actually return failure on failure and success on - // success. - final String automatedTests = path.join(flutterRoot, 'dev', 'automated_tests'); - - // We want to run these tests in parallel, because they each take some time - // to run (e.g. compiling), so we don't want to run them in series, especially - // on 20-core machines. However, we have a race condition, so for now... - // Race condition issue: https://github.com/flutter/flutter/issues/90026 - final List tests = [ - () => runFlutterTest( - automatedTests, - script: path.join('test_smoke_test', 'pass_test.dart'), - printOutput: false, - ), - () => runFlutterTest( - automatedTests, - script: path.join('test_smoke_test', 'fail_test.dart'), - expectFailure: true, - printOutput: false, - ), - () => runFlutterTest( - automatedTests, - script: path.join('test_smoke_test', 'pending_timer_fail_test.dart'), - expectFailure: true, - printOutput: false, - outputChecker: (CommandResult result) { - return result.flattenedStdout!.contains('failingPendingTimerTest') - ? null - : 'Failed to find the stack trace for the pending Timer.\n\n' - 'stdout:\n${result.flattenedStdout}\n\n' - 'stderr:\n${result.flattenedStderr}'; - }, - ), - () => runFlutterTest( - automatedTests, - script: path.join('test_smoke_test', 'fail_test_on_exception_after_test.dart'), - expectFailure: true, - printOutput: false, - outputChecker: (CommandResult result) { - const String expectedError = '══╡ EXCEPTION CAUGHT BY FLUTTER TEST FRAMEWORK ╞════════════════════════════════════════════════════\n' - 'The following StateError was thrown running a test (but after the test had completed):\n' - 'Bad state: Exception thrown after test completed.'; - if (result.flattenedStdout!.contains(expectedError)) { - return null; - } - return 'Failed to find expected output on stdout.\n\n' - 'Expected output:\n$expectedError\n\n' - 'Actual stdout:\n${result.flattenedStdout}\n\n' - 'Actual stderr:\n${result.flattenedStderr}'; - }, - ), - () => runFlutterTest( - automatedTests, - script: path.join('test_smoke_test', 'crash1_test.dart'), - expectFailure: true, - printOutput: false, - ), - () => runFlutterTest( - automatedTests, - script: path.join('test_smoke_test', 'crash2_test.dart'), - expectFailure: true, - printOutput: false, - ), - () => runFlutterTest( - automatedTests, - script: path.join('test_smoke_test', 'syntax_error_test.broken_dart'), - expectFailure: true, - printOutput: false, - ), - () => runFlutterTest( - automatedTests, - script: path.join('test_smoke_test', 'missing_import_test.broken_dart'), - expectFailure: true, - printOutput: false, - ), - () => runFlutterTest( - automatedTests, - script: path.join('test_smoke_test', 'disallow_error_reporter_modification_test.dart'), - expectFailure: true, - printOutput: false, - ), - ]; - - List testsToRun; - - // Run all tests unless sharding is explicitly specified. - final String? shardName = Platform.environment[kShardKey]; - if (shardName == kTestHarnessShardName) { - testsToRun = selectIndexOfTotalSubshard(tests); - } else { - testsToRun = tests; - } - for (final ShardRunner test in testsToRun) { - await test(); - } - - // Verify that we correctly generated the version file. - final String? versionError = await verifyVersion(File(path.join(flutterRoot, 'version'))); - if (versionError != null) { - foundError([versionError]); - } -} - final String _toolsPath = path.join(flutterRoot, 'packages', 'flutter_tools'); Future _runGeneralToolTests() async { @@ -685,325 +525,3 @@ Future _flutterBuildDart2js(String relativePathToApplication, String targe }, ); } - -Future _runFrameworkTests() async { - final List trackWidgetCreationAlternatives = ['--track-widget-creation', '--no-track-widget-creation']; - - Future runWidgets() async { - printProgress('${green}Running packages/flutter tests $reset for ${cyan}test/widgets/$reset'); - for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) { - await runFlutterTest( - path.join(flutterRoot, 'packages', 'flutter'), - options: [trackWidgetCreationOption], - tests: [ path.join('test', 'widgets') + path.separator ], - ); - } - // Try compiling code outside of the packages/flutter directory with and without --track-widget-creation - for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) { - await runFlutterTest( - path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery'), - options: [trackWidgetCreationOption], - fatalWarnings: false, // until we've migrated video_player - ); - } - // Run release mode tests (see packages/flutter/test_release/README.md) - await runFlutterTest( - path.join(flutterRoot, 'packages', 'flutter'), - options: ['--dart-define=dart.vm.product=true'], - tests: ['test_release${path.separator}'], - ); - // Run profile mode tests (see packages/flutter/test_profile/README.md) - await runFlutterTest( - path.join(flutterRoot, 'packages', 'flutter'), - options: ['--dart-define=dart.vm.product=false', '--dart-define=dart.vm.profile=true'], - tests: ['test_profile${path.separator}'], - ); - } - - Future runImpeller() async { - printProgress('${green}Running packages/flutter tests $reset in Impeller$reset'); - await runFlutterTest( - path.join(flutterRoot, 'packages', 'flutter'), - options: ['--enable-impeller'], - ); - } - - - Future runLibraries() async { - final List tests = Directory(path.join(flutterRoot, 'packages', 'flutter', 'test')) - .listSync(followLinks: false) - .whereType() - .where((Directory dir) => !dir.path.endsWith('widgets')) - .map((Directory dir) => path.join('test', path.basename(dir.path)) + path.separator) - .toList(); - printProgress('${green}Running packages/flutter tests$reset for $cyan${tests.join(", ")}$reset'); - for (final String trackWidgetCreationOption in trackWidgetCreationAlternatives) { - await runFlutterTest( - path.join(flutterRoot, 'packages', 'flutter'), - options: [trackWidgetCreationOption], - tests: tests, - ); - } - } - - Future runExampleTests() async { - await runCommand( - flutter, - ['config', '--enable-${Platform.operatingSystem}-desktop'], - workingDirectory: flutterRoot, - ); - await runCommand( - dart, - [path.join(flutterRoot, 'dev', 'tools', 'examples_smoke_test.dart')], - workingDirectory: path.join(flutterRoot, 'examples', 'api'), - ); - for (final FileSystemEntity entity in Directory(path.join(flutterRoot, 'examples')).listSync()) { - if (entity is! Directory || !Directory(path.join(entity.path, 'test')).existsSync()) { - continue; - } - await runFlutterTest(entity.path); - } - } - - Future runTracingTests() async { - final String tracingDirectory = path.join(flutterRoot, 'dev', 'tracing_tests'); - - // run the tests for debug mode - await runFlutterTest(tracingDirectory, options: ['--enable-vmservice']); - - Future> verifyTracingAppBuild({ - required String modeArgument, - required String sourceFile, - required Set allowed, - required Set disallowed, - }) async { - try { - await runCommand( - flutter, - [ - 'build', 'appbundle', '--$modeArgument', path.join('lib', sourceFile), - ], - workingDirectory: tracingDirectory, - ); - final Archive archive = ZipDecoder().decodeBytes(File(path.join(tracingDirectory, 'build', 'app', 'outputs', 'bundle', modeArgument, 'app-$modeArgument.aab')).readAsBytesSync()); - final ArchiveFile libapp = archive.findFile('base/lib/arm64-v8a/libapp.so')!; - final Uint8List libappBytes = libapp.content as Uint8List; // bytes decompressed here - final String libappStrings = utf8.decode(libappBytes, allowMalformed: true); - await runCommand(flutter, ['clean'], workingDirectory: tracingDirectory); - return [ - for (final String pattern in allowed) - if (!libappStrings.contains(pattern)) - 'When building with --$modeArgument, expected to find "$pattern" in libapp.so but could not find it.', - for (final String pattern in disallowed) - if (libappStrings.contains(pattern)) - 'When building with --$modeArgument, expected to not find "$pattern" in libapp.so but did find it.', - ]; - } catch (error, stackTrace) { - return [ - error.toString(), - ...stackTrace.toString().trimRight().split('\n'), - ]; - } - } - - final List results = []; - results.addAll(await verifyTracingAppBuild( - modeArgument: 'profile', - sourceFile: 'control.dart', // this is the control, the other two below are the actual test - allowed: { - 'TIMELINE ARGUMENTS TEST CONTROL FILE', - 'toTimelineArguments used in non-debug build', // we call toTimelineArguments directly to check the message does exist - }, - disallowed: { - 'BUILT IN DEBUG MODE', 'BUILT IN RELEASE MODE', - }, - )); - results.addAll(await verifyTracingAppBuild( - modeArgument: 'profile', - sourceFile: 'test.dart', - allowed: { - 'BUILT IN PROFILE MODE', 'RenderTest.performResize called', // controls - 'BUILD', 'LAYOUT', 'PAINT', // we output these to the timeline in profile builds - // (LAYOUT and PAINT also exist because of NEEDS-LAYOUT and NEEDS-PAINT in RenderObject.toStringShort) - }, - disallowed: { - 'BUILT IN DEBUG MODE', 'BUILT IN RELEASE MODE', - 'TestWidget.debugFillProperties called', 'RenderTest.debugFillProperties called', // debug only - 'toTimelineArguments used in non-debug build', // entire function should get dropped by tree shaker - }, - )); - results.addAll(await verifyTracingAppBuild( - modeArgument: 'release', - sourceFile: 'test.dart', - allowed: { - 'BUILT IN RELEASE MODE', 'RenderTest.performResize called', // controls - }, - disallowed: { - 'BUILT IN DEBUG MODE', 'BUILT IN PROFILE MODE', - 'BUILD', 'LAYOUT', 'PAINT', // these are only used in Timeline.startSync calls that should not appear in release builds - 'TestWidget.debugFillProperties called', 'RenderTest.debugFillProperties called', // debug only - 'toTimelineArguments used in non-debug build', // not included in release builds - }, - )); - if (results.isNotEmpty) { - foundError(results); - } - } - - Future runFixTests(String package) async { - final List args = [ - 'fix', - '--compare-to-golden', - ]; - await runCommand( - dart, - args, - workingDirectory: path.join(flutterRoot, 'packages', package, 'test_fixes'), - ); - } - - Future runPrivateTests() async { - final List args = [ - 'run', - 'bin/test_private.dart', - ]; - final Map environment = { - 'FLUTTER_ROOT': flutterRoot, - if (Directory(pubCache).existsSync()) - 'PUB_CACHE': pubCache, - }; - adjustEnvironmentToEnableFlutterAsserts(environment); - await runCommand( - dart, - args, - workingDirectory: path.join(flutterRoot, 'packages', 'flutter', 'test_private'), - environment: environment, - ); - } - - // Tests that take longer than average to run. This is usually because they - // need to compile something large or make use of the analyzer for the test. - // These tests need to be platform agnostic as they are only run on a linux - // machine to save on execution time and cost. - Future runSlow() async { - printProgress('${green}Running slow package tests$reset for directories other than packages/flutter'); - await runTracingTests(); - await runFixTests('flutter'); - await runFixTests('flutter_test'); - await runFixTests('integration_test'); - await runFixTests('flutter_driver'); - await runPrivateTests(); - } - - Future runMisc() async { - printProgress('${green}Running package tests$reset for directories other than packages/flutter'); - await _runTestHarnessTests(); - await runExampleTests(); - await runFlutterTest( - path.join(flutterRoot, 'dev', 'a11y_assessments'), - tests: [ 'test' ], - ); - await runDartTest(path.join(flutterRoot, 'dev', 'bots')); - await runDartTest(path.join(flutterRoot, 'dev', 'devicelab'), ensurePrecompiledTool: false); // See https://github.com/flutter/flutter/issues/86209 - await runDartTest(path.join(flutterRoot, 'dev', 'conductor', 'core'), forceSingleCore: true); - // TODO(gspencergoog): Remove the exception for fatalWarnings once https://github.com/flutter/flutter/issues/113782 has landed. - await runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing'), fatalWarnings: false); - await runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'ui')); - await runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests')); - await runFlutterTest(path.join(flutterRoot, 'dev', 'tools')); - await runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'vitool')); - await runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'gen_defaults')); - await runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'gen_keycodes')); - await runFlutterTest(path.join(flutterRoot, 'dev', 'benchmarks', 'test_apps', 'stocks')); - await runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'), tests: [path.join('test', 'src', 'real_tests')]); - await runFlutterTest(path.join(flutterRoot, 'packages', 'integration_test'), options: [ - '--enable-vmservice', - // Web-specific tests depend on Chromium, so they run as part of the web_long_running_tests shard. - '--exclude-tags=web', - ]); - // Run java unit tests for integration_test - // - // Generate Gradle wrapper if it doesn't exist. - Process.runSync( - flutter, - ['build', 'apk', '--config-only'], - workingDirectory: path.join(flutterRoot, 'packages', 'integration_test', 'example', 'android'), - ); - await runCommand( - path.join(flutterRoot, 'packages', 'integration_test', 'example', 'android', 'gradlew$bat'), - [ - ':integration_test:testDebugUnitTest', - '--tests', - 'dev.flutter.plugins.integration_test.FlutterDeviceScreenshotTest', - ], - workingDirectory: path.join(flutterRoot, 'packages', 'integration_test', 'example', 'android'), - ); - await runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_goldens')); - await runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_localizations')); - await runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test')); - await runFlutterTest(path.join(flutterRoot, 'packages', 'fuchsia_remote_debug_protocol')); - await runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'non_nullable')); - const String httpClientWarning = - 'Warning: At least one test in this suite creates an HttpClient. When running a test suite that uses\n' - 'TestWidgetsFlutterBinding, all HTTP requests will return status code 400, and no network request\n' - 'will actually be made. Any test expecting a real network connection and status code will fail.\n' - 'To test code that needs an HttpClient, provide your own HttpClient implementation to the code under\n' - 'test, so that your test can consistently provide a testable response to the code under test.'; - await runFlutterTest( - path.join(flutterRoot, 'packages', 'flutter_test'), - script: path.join('test', 'bindings_test_failure.dart'), - expectFailure: true, - printOutput: false, - outputChecker: (CommandResult result) { - final Iterable matches = httpClientWarning.allMatches(result.flattenedStdout!); - if (matches.isEmpty || matches.length > 1) { - return 'Failed to print warning about HttpClientUsage, or printed it too many times.\n\n' - 'stdout:\n${result.flattenedStdout}\n\n' - 'stderr:\n${result.flattenedStderr}'; - } - return null; - }, - ); - } - - await selectSubshard({ - 'widgets': runWidgets, - 'libraries': runLibraries, - 'slow': runSlow, - 'misc': runMisc, - 'impeller': runImpeller, - }); -} - -/// This will force the next run of the Flutter tool (if it uses the provided -/// environment) to have asserts enabled, by setting an environment variable. -void adjustEnvironmentToEnableFlutterAsserts(Map environment) { - // If an existing env variable exists append to it, but only if - // it doesn't appear to already include enable-asserts. - String toolsArgs = Platform.environment['FLUTTER_TOOL_ARGS'] ?? ''; - if (!toolsArgs.contains('--enable-asserts')) { - toolsArgs += ' --enable-asserts'; - } - environment['FLUTTER_TOOL_ARGS'] = toolsArgs.trim(); -} - -/// Checks the given file's contents to determine if they match the allowed -/// pattern for version strings. -/// -/// Returns null if the contents are good. Returns a string if they are bad. -/// The string is an error message. -Future verifyVersion(File file) async { - final RegExp pattern = RegExp( - r'^(\d+)\.(\d+)\.(\d+)((-\d+\.\d+)?\.pre(\.\d+)?)?$'); - if (!file.existsSync()) { - return 'The version logic failed to create the Flutter version file.'; - } - final String version = await file.readAsString(); - if (version == '0.0.0-unknown') { - return 'The version logic failed to determine the Flutter version.'; - } - if (!version.contains(pattern)) { - return 'The version logic generated an invalid version string: "$version".'; - } - return null; -} diff --git a/dev/bots/test/test_test.dart b/dev/bots/test/test_test.dart index 72f13408047..e620de8f3ae 100644 --- a/dev/bots/test/test_test.dart +++ b/dev/bots/test/test_test.dart @@ -9,9 +9,8 @@ import 'package:file/memory.dart'; import 'package:path/path.dart' as path; import 'package:process/process.dart'; -import '../analyze.dart'; import '../suite_runners/run_flutter_packages_tests.dart'; -import '../test.dart'; +import '../utils.dart'; import 'common.dart'; /// Fails a test if the exit code of `result` is not the expected value. This diff --git a/dev/bots/utils.dart b/dev/bots/utils.dart index 7306ced5185..61b523ed54b 100644 --- a/dev/bots/utils.dart +++ b/dev/bots/utils.dart @@ -59,6 +59,7 @@ final bool runningInDartHHHBot = const String kShardKey = 'SHARD'; const String kSubshardKey = 'SUBSHARD'; +const String kTestHarnessShardName = 'test_harness_tests'; const String CIRRUS_TASK_NAME = 'CIRRUS_TASK_NAME'; /// Environment variables to override the local engine when running `pub test`, @@ -600,3 +601,24 @@ Future _runFromList(Map items, String key, String nam await items[item]!(); } } + +/// Checks the given file's contents to determine if they match the allowed +/// pattern for version strings. +/// +/// Returns null if the contents are good. Returns a string if they are bad. +/// The string is an error message. +Future verifyVersion(File file) async { + final RegExp pattern = RegExp( + r'^(\d+)\.(\d+)\.(\d+)((-\d+\.\d+)?\.pre(\.\d+)?)?$'); + if (!file.existsSync()) { + return 'The version logic failed to create the Flutter version file.'; + } + final String version = await file.readAsString(); + if (version == '0.0.0-unknown') { + return 'The version logic failed to determine the Flutter version.'; + } + if (!version.contains(pattern)) { + return 'The version logic generated an invalid version string: "$version".'; + } + return null; +}