Add --dry-run to dev/bots/test.dart. (#158956)

Closes https://github.com/flutter/flutter/issues/158884.

/cc @reidbaker.

Example output:

```sh
% SHARD=tool_integration_tests SUBSHARD=6_6 dart dev/bots/test.dart --dry-run 
▌13:59:18▐ STARTING ANALYSIS
▌13:59:18▐ --dry-run enabled. Tests will not actually be executed.
▌13:59:18▐ SHARD=tool_integration_tests
▌13:59:18▐ SUBSHARD=6_6
▌13:59:18▐ |> bin/flutter: 
  --version
▌13:59:18▐ |> bin/cache/dart-sdk/bin/dart (packages/flutter_tools): 
  run
  test
  --reporter=expanded
  --file-reporter=json:/var/folders/qw/qw_3qd1x4kz5w975jhdq4k58007b7h/T/metrics_1731621558619861.json
  --test-randomize-ordering-seed=20241114
  -j1
  test/integration.shard/shader_compiler_test.dart
  test/integration.shard/overall_experience_test.dart
  test/integration.shard/expression_evaluation_test.dart
  test/integration.shard/isolated/native_assets_without_cbuild_assemble_test.dart
  test/integration.shard/isolated/native_assets_test.dart
  test/integration.shard/isolated/native_assets_agp_version_test.dart
  test/integration.shard/coverage_collection_test.dart
  test/integration.shard/devtools_uri_test.dart
  test/integration.shard/deprecated_gradle_settings_test.dart
  test/integration.shard/batch_entrypoint_test.dart
  test/integration.shard/bash_entrypoint_test.dart
  test/integration.shard/background_isolate_test.dart
```

dev/bots/run_command.dart

@@ -162,6 +162,15 @@ Future<CommandResult> runCommand(String executable, List<String> arguments, {
 }) async {
   final String commandDescription = '${path.relative(executable, from: workingDirectory)} ${arguments.join(' ')}';
   final String relativeWorkingDir = workingDirectory ?? path.relative(io.Directory.current.path);
+  if (dryRun) {
+    printProgress(_prettyPrintRunCommand(executable, arguments, workingDirectory));
+    return CommandResult._(
+      0,
+      Duration.zero,
+      '$executable ${arguments.join(' ')}',
+      'Simulated execution due to --dry-run',
+    );
+  }
 
   final Command command = await startCommand(executable, arguments,
     workingDirectory: workingDirectory,
@@ -205,6 +214,23 @@ Future<CommandResult> runCommand(String executable, List<String> arguments, {
   return result;
 }
 
+final String _flutterRoot = path.dirname(path.dirname(path.dirname(path.fromUri(io.Platform.script))));
+
+String _prettyPrintRunCommand(String executable, List<String> arguments, String? workingDirectory) {
+  final StringBuffer output = StringBuffer();
+
+  // Print CWD relative to the root.
+  output.write('|> ');
+  output.write(path.relative(executable, from: _flutterRoot));
+  if (workingDirectory != null) {
+    output.write(' (${path.relative(workingDirectory, from: _flutterRoot)})');
+  }
+  output.writeln(': ');
+  output.writeAll(arguments.map((String a) => '  $a'), '\n');
+
+  return output.toString();
+}
+
 /// Specifies what to do with the command output from [runCommand] and [startCommand].
 enum OutputMode {
   /// Forwards standard output and standard error streams to the test process'

dev/bots/suite_runners/run_web_tests.dart

@@ -708,7 +708,9 @@ class WebTestsSuite {
     // metriciFile is a transitional file that needs to be deleted once it is parsed.
     // TODO(godofredoc): Ensure metricFile is parsed and aggregated before deleting.
     // https://github.com/flutter/flutter/issues/146003
-    metricFile.deleteSync();
+    if (!dryRun) {
+      metricFile.deleteSync();
+    }
   }
 
   // The `chromedriver` process created by this test.

dev/bots/test.dart

@@ -4,14 +4,12 @@
 
 // Runs the tests for the flutter/flutter repository.
 //
-//
 // By default, test output is filtered and only errors are shown. (If a
 // particular test takes longer than _quietTimeout in utils.dart, the output is
 // shown then also, in case something has hung.)
 //
 //  --verbose stops the output cleanup and just outputs everything verbatim.
 //
-//
 // By default, errors are non-fatal; all tests are executed and the output
 // ends with a summary of the errors that were detected.
 //
@@ -19,10 +17,10 @@
 //
 //  --abort-on-error causes the script to exit immediately when hitting an error.
 //
-//
 // By default, all tests are run. However, the tests support being split by
-// shard and subshard. (Inspect the code to see what shards and subshards are
-// supported.)
+// shard and subshard. Inspect the code to see what shards and subshards are
+// supported, or run with `--dry-run` to get a list of tests that _would_ have
+// been executed.
 //
 // If the CIRRUS_TASK_NAME environment variable exists, it is used to determine
 // the shard and sub-shard, by parsing it in the form shard-subshard-platform,
@@ -43,7 +41,6 @@
 //
 //  --test-randomize-ordering-seed=<n> sets the shuffle seed for reproducing runs.
 //
-//
 // All other arguments are treated as arguments to pass to the flutter tool when
 // running tests.
 
@@ -96,6 +93,7 @@ const String CIRRUS_TASK_NAME = 'CIRRUS_TASK_NAME';
 Future<void> main(List<String> args) async {
   try {
     printProgress('STARTING ANALYSIS');
+    bool dryRunArgSet = false;
     for (final String arg in args) {
       if (arg.startsWith('--local-engine=')) {
         localEngineEnv['FLUTTER_LOCAL_ENGINE'] = arg.substring('--local-engine='.length);
@@ -116,10 +114,16 @@ Future<void> main(List<String> args) async {
         onError = () {
           system.exit(1);
         };
+      } else if (arg == '--dry-run') {
+        dryRunArgSet = true;
+        printProgress('--dry-run enabled. Tests will not actually be executed.');
       } else {
         flutterTestArgs.add(arg);
       }
     }
+    if (dryRunArgSet) {
+      enableDryRun();
+    }
     if (Platform.environment.containsKey(CIRRUS_TASK_NAME)) {
       printProgress('Running task: ${Platform.environment[CIRRUS_TASK_NAME]}');
     }
@@ -541,6 +545,9 @@ Future<void> _flutterBuild(
 }
 
 bool _allTargetsCached(File performanceFile) {
+  if (dryRun) {
+    return true;
+  }
   final Map<String, Object?> data = json.decode(performanceFile.readAsStringSync())
     as Map<String, Object?>;
   final List<Map<String, Object?>> targets = (data['targets']! as List<Object?>)

dev/bots/test/test_test.dart

@@ -154,6 +154,15 @@ void main() {
       expectExitCode(result, 255);
       expect(result.stdout, contains('Invalid subshard name'));
     });
+
+    test('--dry-run prints every test that would run', () async {
+      final ProcessResult result = await runScript(
+        <String, String> {},
+        <String>['--dry-run'],
+      );
+      expectExitCode(result, 0);
+      expect(result.stdout, contains('|> bin/flutter'));
+    }, testOn: 'posix');
   });
 
   test('selectTestsForSubShard distributes tests amongst subshards correctly', () async {

dev/bots/utils.dart

@@ -68,9 +68,27 @@ const String CIRRUS_TASK_NAME = 'CIRRUS_TASK_NAME';
 final Map<String,String> localEngineEnv = <String, String>{};
 
 /// The arguments to pass to `flutter test` (typically the local engine
-/// configuration) -- prefilled with the arguments passed to test.dart.
+/// configuration) -- prefilled with  the arguments passed to test.dart.
 final List<String> flutterTestArgs = <String>[];
 
+/// Whether execution should be simulated for debugging purposes.
+///
+/// When `true`, calls to [runCommand] print to [io.stdout] instead of running
+/// the process. This is useful for determing what an invocation of `test.dart`
+/// _might_ due if not invoked with `--dry-run`, or otherwise determine what the
+/// different test shards and sub-shards are configured as.
+bool get dryRun => _dryRun ?? false;
+
+/// Switches [dryRun] to `true`.
+///
+/// Expected to be called at most once during execution of a process.
+void enableDryRun() {
+  if (_dryRun != null) {
+    throw StateError('Should only be called at most once');
+  }
+  _dryRun = true;
+}
+bool? _dryRun;
 
 const int kESC = 0x1B;
 const int kOpenSquareBracket = 0x5B;
@@ -135,6 +153,10 @@ final List<String> _pendingLogs = <String>[];
 Timer? _hideTimer; // When this is null, the output is verbose.
 
 void foundError(List<String> messages) {
+  if (dryRun) {
+    printProgress(messages.join('\n'));
+    return;
+  }
   assert(messages.isNotEmpty);
   // Make the error message easy to notice in the logs by
   // wrapping it in a red box.
@@ -413,6 +435,10 @@ Future<void> runDartTest(String workingDirectory, {
     removeLine: useBuildRunner ? (String line) => line.startsWith('[INFO]') : null,
   );
 
+  if (dryRun) {
+    return;
+  }
+
   final TestFileReporterResults test = TestFileReporterResults.fromFile(metricFile); // --file-reporter name
   final File info = fileSystem.file(path.join(flutterRoot, 'error.log'));
   info.writeAsStringSync(json.encode(test.errors));
@@ -508,7 +534,9 @@ Future<void> runFlutterTest(String workingDirectory, {
   // metriciFile is a transitional file that needs to be deleted once it is parsed.
   // TODO(godofredoc): Ensure metricFile is parsed and aggregated before deleting.
   // https://github.com/flutter/flutter/issues/146003
-  metricFile.deleteSync();
+  if (!dryRun) {
+    metricFile.deleteSync();
+  }
 
   if (outputChecker != null) {
     final String? message = outputChecker(result);
@@ -619,27 +647,33 @@ List<T> selectIndexOfTotalSubshard<T>(List<T> tests, {String subshardKey = kSubs
 }
 
 Future<void> _runFromList(Map<String, ShardRunner> items, String key, String name, int positionInTaskName) async {
-  String? item = Platform.environment[key];
-  if (item == null && Platform.environment.containsKey(CIRRUS_TASK_NAME)) {
-    final List<String> parts = Platform.environment[CIRRUS_TASK_NAME]!.split('-');
-    assert(positionInTaskName < parts.length);
-    item = parts[positionInTaskName];
-  }
-  if (item == null) {
-    for (final String currentItem in items.keys) {
-      printProgress('$bold$key=$currentItem$reset');
-      await items[currentItem]!();
+  try {
+    String? item = Platform.environment[key];
+    if (item == null && Platform.environment.containsKey(CIRRUS_TASK_NAME)) {
+      final List<String> parts = Platform.environment[CIRRUS_TASK_NAME]!.split('-');
+      assert(positionInTaskName < parts.length);
+      item = parts[positionInTaskName];
     }
-  } else {
-    printProgress('$bold$key=$item$reset');
-    if (!items.containsKey(item)) {
-      foundError(<String>[
-        '${red}Invalid $name: $item$reset',
-        'The available ${name}s are: ${items.keys.join(", ")}',
-      ]);
-      return;
+    if (item == null) {
+      for (final String currentItem in items.keys) {
+        printProgress('$bold$key=$currentItem$reset');
+        await items[currentItem]!();
+      }
+    } else {
+      printProgress('$bold$key=$item$reset');
+      if (!items.containsKey(item)) {
+        foundError(<String>[
+          '${red}Invalid $name: $item$reset',
+          'The available ${name}s are: ${items.keys.join(", ")}',
+        ]);
+        return;
+      }
+      await items[item]!();
+    }
+  } catch (_) {
+    if (!dryRun) {
+      rethrow;
     }
-    await items[item]!();
   }
 }