// Copyright 2014 The Flutter Authors. All rights reserved. // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. import 'dart:async'; import 'dart:ui'; import 'package:flutter/scheduler.dart'; import 'package:flutter_test/flutter_test.dart'; import 'package:flutter/widgets.dart'; import 'package:macrobenchmarks/common.dart'; import 'package:e2e/e2e.dart'; import 'package:macrobenchmarks/main.dart' as app; /// The maximum amount of time considered safe to spend for a frame's build /// phase. Anything past that is in the danger of missing the frame as 60FPS. /// /// Changing this doesn't re-evaluate existing summary. Duration kBuildBudget = const Duration(milliseconds: 16); // TODO(CareF): Automatically calculate the refresh budget (#61958) typedef ControlCallback = Future Function(WidgetController controller); void macroPerfTestE2E( String testName, String routeName, { Duration pageDelay, Duration duration = const Duration(seconds: 3), Duration timeout = const Duration(seconds: 30), ControlCallback body, ControlCallback setup, }) { assert(() { debugPrint(kDebugWarning); return true; }()); final WidgetsBinding _binding = E2EWidgetsFlutterBinding.ensureInitialized(); assert(_binding is E2EWidgetsFlutterBinding); final E2EWidgetsFlutterBinding binding = _binding as E2EWidgetsFlutterBinding; binding.framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.benchmarkLive; testWidgets(testName, (WidgetTester tester) async { assert(tester.binding == binding); app.main(); await tester.pumpAndSettle(); // The slight initial delay avoids starting the timing during a // period of increased load on the device. Without this delay, the // benchmark has greater noise. // See: https://github.com/flutter/flutter/issues/19434 await tester.binding.delayed(const Duration(microseconds: 250)); final Finder scrollable = find.byKey(const ValueKey(kScrollableName)); expect(scrollable, findsOneWidget); final Finder button = find.byKey(ValueKey(routeName), skipOffstage: false); await tester.ensureVisible(button); expect(button, findsOneWidget); await tester.pumpAndSettle(); await tester.tap(button); // Cannot be pumpAndSettle because some tests have inifite animation. await tester.pump(const Duration(milliseconds: 20)); if (pageDelay != null) { // Wait for the page to load await tester.binding.delayed(pageDelay); } if (setup != null) { await setup(tester); } await watchPerformance(binding, () async { final Future durationFuture = tester.binding.delayed(duration); if (body != null) { await body(tester); } await durationFuture; }); }, semanticsEnabled: false, timeout: Timeout(timeout)); } Future watchPerformance( E2EWidgetsFlutterBinding binding, Future action(), ) async { final List frameTimings = []; final TimingsCallback watcher = frameTimings.addAll; binding.addTimingsCallback(watcher); await action(); binding.removeTimingsCallback(watcher); // TODO(CareF): determine if it's running on firebase and report metric online final FrameTimingSummarizer frameTimes = FrameTimingSummarizer(frameTimings); binding.reportData = {'performance': frameTimes.summary}; } /// This class and summarizes a list of [FrameTiming] for the performance /// metrics. class FrameTimingSummarizer { factory FrameTimingSummarizer(List data) { assert(data != null); assert(data.isNotEmpty); final List frameBuildTime = List.unmodifiable( data.map((FrameTiming datum) => datum.buildDuration), ); final List frameBuildTimeSorted = List.from(frameBuildTime)..sort(); final List frameRasterizerTime = List.unmodifiable( data.map((FrameTiming datum) => datum.rasterDuration), ); final List frameRasterizerTimeSorted = List.from(frameRasterizerTime)..sort(); final Duration Function(Duration, Duration) add = (Duration a, Duration b) => a + b; return FrameTimingSummarizer._( frameBuildTime: frameBuildTime, frameRasterizerTime: frameRasterizerTime, // This avarage calculation is microsecond precision, which is fine // because typical values of these times are milliseconds. averageFrameBuildTime: frameBuildTime.reduce(add) ~/ data.length, p90FrameBuildTime: _findPercentile(frameBuildTimeSorted, 0.90), p99FrameBuildTime: _findPercentile(frameBuildTimeSorted, 0.99), worstFrameBuildTime: frameBuildTimeSorted.last, missedFrameBuildBudget: _countExceed(frameBuildTimeSorted, kBuildBudget), averageFrameRasterizerTime: frameRasterizerTime.reduce(add) ~/ data.length, p90FrameRasterizerTime: _findPercentile(frameRasterizerTimeSorted, 0.90), p99FrameRasterizerTime: _findPercentile(frameRasterizerTimeSorted, 0.90), worstFrameRasterizerTime: frameRasterizerTimeSorted.last, missedFrameRasterizerBudget: _countExceed(frameRasterizerTimeSorted, kBuildBudget), ); } const FrameTimingSummarizer._({ @required this.frameBuildTime, @required this.frameRasterizerTime, @required this.averageFrameBuildTime, @required this.p90FrameBuildTime, @required this.p99FrameBuildTime, @required this.worstFrameBuildTime, @required this.missedFrameBuildBudget, @required this.averageFrameRasterizerTime, @required this.p90FrameRasterizerTime, @required this.p99FrameRasterizerTime, @required this.worstFrameRasterizerTime, @required this.missedFrameRasterizerBudget }); /// List of frame build time in microseconds final List frameBuildTime; /// List of frame rasterizer time in microseconds final List frameRasterizerTime; /// The average value of [frameBuildTime] in milliseconds. final Duration averageFrameBuildTime; /// The 90-th percentile value of [frameBuildTime] in milliseconds final Duration p90FrameBuildTime; /// The 99-th percentile value of [frameBuildTime] in milliseconds final Duration p99FrameBuildTime; /// The largest value of [frameBuildTime] in milliseconds final Duration worstFrameBuildTime; /// Number of items in [frameBuildTime] that's greater than [kBuildBudget] final int missedFrameBuildBudget; /// The average value of [frameRasterizerTime] in milliseconds. final Duration averageFrameRasterizerTime; /// The 90-th percentile value of [frameRasterizerTime] in milliseconds. final Duration p90FrameRasterizerTime; /// The 99-th percentile value of [frameRasterizerTime] in milliseconds. final Duration p99FrameRasterizerTime; /// The largest value of [frameRasterizerTime] in milliseconds. final Duration worstFrameRasterizerTime; /// Number of items in [frameRasterizerTime] that's greater than [kBuildBudget] final int missedFrameRasterizerBudget; Map get summary => { 'average_frame_build_time_millis': averageFrameBuildTime.inMicroseconds / 1E3, '90th_percentile_frame_build_time_millis': p90FrameBuildTime.inMicroseconds / 1E3, '99th_percentile_frame_build_time_millis': p99FrameBuildTime.inMicroseconds / 1E3, 'worst_frame_build_time_millis': worstFrameBuildTime.inMicroseconds / 1E3, 'missed_frame_build_budget_count': missedFrameBuildBudget, 'average_frame_rasterizer_time_millis': averageFrameRasterizerTime.inMicroseconds / 1E3, '90th_percentile_frame_rasterizer_time_millis': p90FrameRasterizerTime.inMicroseconds / 1E3, '99th_percentile_frame_rasterizer_time_millis': p99FrameRasterizerTime.inMicroseconds / 1E3, 'worst_frame_rasterizer_time_millis': worstFrameRasterizerTime.inMicroseconds / 1E3, 'missed_frame_rasterizer_budget_count': missedFrameRasterizerBudget, 'frame_count': frameBuildTime.length, 'frame_build_times': frameBuildTime .map((Duration datum) => datum.inMicroseconds).toList(), 'frame_rasterizer_times': frameRasterizerTime .map((Duration datum) => datum.inMicroseconds).toList(), }; } // The following helper functions require data sorted // return the 100*p-th percentile of the data T _findPercentile(List data, double p) { assert(p >= 0 && p <= 1); return data[((data.length - 1) * p).round()]; } // return the number of items in data that > threshold int _countExceed>(List data, T threshold) { return data.length - data.indexWhere((T datum) => datum.compareTo(threshold) > 0); }