Add flutter startup benchmark (#111658)

Add flutter startup benchmark for Linux, Windows and MacOs.

Via guide on https://github.com/flutter/flutter/pull/111461#issuecomment-1247113905

.ci.yaml

@@ -4445,3 +4445,33 @@ targets:
       tags: >
         ["devicelab", "hostonly"]
       task_name: windows_startup_test
+
+  - name: Windows flutter_tool_startup__windows
+    bringup: true
+    recipe: devicelab/devicelab_drone
+    presubmit: false
+    timeout: 60
+    properties:
+      tags: >
+        ["devicelab", "hostonly"]
+      task_name: flutter_tool_startup__windows
+
+  - name: Linux flutter_tool_startup__linux
+    bringup: true
+    recipe: devicelab/devicelab_drone
+    presubmit: false
+    timeout: 60
+    properties:
+      tags: >
+        ["devicelab", "hostonly"]
+      task_name: flutter_tool_startup__linux
+
+  - name: Mac flutter_tool_startup__macos
+    bringup: true
+    recipe: devicelab/devicelab_drone
+    presubmit: false
+    timeout: 60
+    properties:
+      tags: >
+        ["devicelab", "hostonly"]
+      task_name: flutter_tool_startup__macos

TESTOWNERS

@@ -220,6 +220,9 @@
 /dev/devicelab/bin/tasks/flutter_gallery_win_desktop__compile.dart @yaakovschectman @flutter/desktop
 /dev/devicelab/bin/tasks/flutter_gallery_win_desktop__start_up.dart @yaakovschectman @flutter/desktop
 /dev/devicelab/bin/tasks/flutter_view_macos__start_up.dart @a-wallen @flutter/desktop
+/dev/devicelab/bin/tasks/flutter_tool_startup__windows.dart @jensjoha @flutter/tool
+/dev/devicelab/bin/tasks/flutter_tool_startup__linux.dart @jensjoha @flutter/tool
+/dev/devicelab/bin/tasks/flutter_tool_startup__macos.dart @jensjoha @flutter/tool
 /dev/devicelab/bin/tasks/flutter_view_win_desktop__start_up.dart @yaakovschectman @flutter/desktop
 /dev/devicelab/bin/tasks/gradle_desugar_classes_test.dart @zanderso @flutter/tool
 /dev/devicelab/bin/tasks/gradle_non_android_plugin_test.dart @stuartmorgan @flutter/plugin

dev/devicelab/bin/tasks/flutter_tool_startup__linux.dart

@@ -0,0 +1,10 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/tasks/flutter_tool_startup.dart';
+
+Future<void> main() async {
+  await task(flutterToolStartupBenchmarkTask);
+}

dev/devicelab/bin/tasks/flutter_tool_startup__macos.dart

@@ -0,0 +1,10 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/tasks/flutter_tool_startup.dart';
+
+Future<void> main() async {
+  await task(flutterToolStartupBenchmarkTask);
+}

dev/devicelab/bin/tasks/flutter_tool_startup__windows.dart

@@ -0,0 +1,10 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'package:flutter_devicelab/framework/framework.dart';
+import 'package:flutter_devicelab/tasks/flutter_tool_startup.dart';
+
+Future<void> main() async {
+  await task(flutterToolStartupBenchmarkTask);
+}

dev/devicelab/lib/tasks/flutter_tool_startup.dart

@@ -0,0 +1,170 @@
+// Copyright 2014 The Flutter Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style license that can be
+// found in the LICENSE file.
+
+import 'dart:io';
+
+import 'package:path/path.dart' as path;
+
+import '../framework/task_result.dart';
+import '../framework/utils.dart';
+
+/// Run each benchmark this many times and compute average, min, max.
+const int _kRunsPerBenchmark = 10;
+
+Future<TaskResult> flutterToolStartupBenchmarkTask() async {
+  final Directory projectParentDirectory =
+      Directory.systemTemp.createTempSync('flutter_tool_startup_benchmark');
+  final Directory projectDirectory =
+      dir(path.join(projectParentDirectory.path, 'benchmark'));
+  await inDirectory<void>(flutterDirectory, () async {
+    await flutter('update-packages');
+    await flutter('create', options: <String>[projectDirectory.path]);
+    // Remove 'test' directory so we don't time the actual testing, but only the launching of the flutter tool
+    rmTree(dir(path.join(projectDirectory.path, 'test')));
+  });
+
+  final Map<String, dynamic> data = <String, dynamic>{
+    // `flutter test` in dir with no `test` folder.
+    ...(await _Benchmark(
+      projectDirectory,
+      'test startup',
+      'test',
+    ).run())
+        .asMap('flutter_tool_startup_test'),
+
+    // `flutter test -d foo_device` in dir woth no `test` folder.
+    ...(await _Benchmark(
+      projectDirectory,
+      'test startup with specified device',
+      'test',
+      options: <String>['-d', 'foo_device'],
+    ).run())
+        .asMap('flutter_tool_startup_test_with_specified_device'),
+
+    // `flutter test -v` where no android sdk will be found (at least currently).
+    ...(await _Benchmark(
+      projectDirectory,
+      'test startup no android sdk',
+      'test',
+      options: <String>['-v'],
+      environment: <String, String>{
+        'ANDROID_HOME': 'dummy value',
+        'ANDROID_SDK_ROOT': 'dummy value',
+        'PATH': pathWithoutWhereHits(<String>['adb', 'aapt']),
+      },
+    ).run())
+        .asMap('flutter_tool_startup_test_no_android_sdk'),
+
+    // `flutter -h`.
+    ...(await _Benchmark(
+      projectDirectory,
+      'help startup',
+      '-h',
+    ).run())
+        .asMap('flutter_tool_startup_help'),
+  };
+
+  // Cleanup.
+  rmTree(projectParentDirectory);
+
+  return TaskResult.success(data, benchmarkScoreKeys: data.keys.toList());
+}
+
+String pathWithoutWhereHits(List<String> whats) {
+  final String pathEnvironment = Platform.environment['PATH'] ?? '';
+  List<String> paths;
+  if (Platform.isWindows) {
+    paths = pathEnvironment.split(';');
+  } else {
+    paths = pathEnvironment.split(':');
+  }
+  // This isn't great but will probably work for our purposes.
+  final List<String> extensions = <String>['', '.exe', '.bat', '.com'];
+
+  final List<String> notFound = <String>[];
+  for (final String path in paths) {
+    bool found = false;
+    for (final String extension in extensions) {
+      for (final String what in whats) {
+        final File f = File('$path${Platform.pathSeparator}$what$extension');
+        if (f.existsSync()) {
+          found = true;
+          break;
+        }
+      }
+      if (found) {
+        break;
+      }
+    }
+    if (!found) {
+      notFound.add(path);
+    }
+  }
+
+  if (Platform.isWindows) {
+    return notFound.join(';');
+  } else {
+    return notFound.join(':');
+  }
+}
+
+class _BenchmarkResult {
+  const _BenchmarkResult(this.mean, this.min, this.max);
+
+  final int mean; // Milliseconds
+
+  final int min; // Milliseconds
+
+  final int max; // Milliseconds
+
+  Map<String, dynamic> asMap(String name) {
+    return <String, dynamic>{
+      name: mean,
+      '${name}_minimum': min,
+      '${name}_maximum': max,
+    };
+  }
+}
+
+class _Benchmark {
+  _Benchmark(this.directory, this.title, this.command,
+      {this.options = const <String>[], this.environment});
+
+  final Directory directory;
+
+  final String title;
+
+  final String command;
+
+  final List<String> options;
+
+  final Map<String, String>? environment;
+
+  Future<int> execute(int iteration, int targetIterations) async {
+    section('Benchmark $title - ${iteration + 1} / $targetIterations');
+    final Stopwatch stopwatch = Stopwatch();
+    await inDirectory<void>(directory, () async {
+      stopwatch.start();
+      // canFail is set to true, as e.g. `flutter test` in a dir with no `test`
+      // directory sets a non-zero return value.
+      await flutter(command,
+          options: options, canFail: true, environment: environment);
+      stopwatch.stop();
+    });
+    return stopwatch.elapsedMilliseconds;
+  }
+
+  /// Runs `benchmark` several times and reports the results.
+  Future<_BenchmarkResult> run() async {
+    final List<int> results = <int>[];
+    int sum = 0;
+    for (int i = 0; i < _kRunsPerBenchmark; i++) {
+      final int thisRuntime = await execute(i, _kRunsPerBenchmark);
+      results.add(thisRuntime);
+      sum += thisRuntime;
+    }
+    results.sort();
+    return _BenchmarkResult(sum ~/ results.length, results.first, results.last);
+  }
+}