diff --git a/dev/benchmarks/macrobenchmarks/lib/common.dart b/dev/benchmarks/macrobenchmarks/lib/common.dart index 8e49f7aad03..9174d2102dc 100644 --- a/dev/benchmarks/macrobenchmarks/lib/common.dart +++ b/dev/benchmarks/macrobenchmarks/lib/common.dart @@ -16,5 +16,6 @@ const String kFadingChildAnimationRouteName = '/fading_child_animation'; const String kImageFilteredTransformAnimationRouteName = '/imagefiltered_transform_animation'; const String kMultiWidgetConstructionRouteName = '/multi_widget_construction'; const String kHeavyGridViewRouteName = '/heavy_gridview'; +const String kSimpleScrollRouteName = '/simple_scroll'; const String kScrollableName = '/macrobenchmark_listview'; diff --git a/dev/benchmarks/macrobenchmarks/lib/main.dart b/dev/benchmarks/macrobenchmarks/lib/main.dart index 7c2a91fc2d6..61dd0f0bcc9 100644 --- a/dev/benchmarks/macrobenchmarks/lib/main.dart +++ b/dev/benchmarks/macrobenchmarks/lib/main.dart @@ -17,6 +17,7 @@ import 'src/filtered_child_animation.dart'; import 'src/multi_widget_construction.dart'; import 'src/post_backdrop_filter.dart'; import 'src/simple_animation.dart'; +import 'src/simple_scroll.dart'; import 'src/text.dart'; const String kMacrobenchmarks = 'Macrobenchmarks'; @@ -47,6 +48,7 @@ class MacrobenchmarksApp extends StatelessWidget { kImageFilteredTransformAnimationRouteName: (BuildContext context) => const FilteredChildAnimationPage(FilterType.rotateFilter), kMultiWidgetConstructionRouteName: (BuildContext context) => const MultiWidgetConstructTable(10, 20), kHeavyGridViewRouteName: (BuildContext context) => HeavyGridViewPage(), + kSimpleScrollRouteName: (BuildContext context) => SimpleScroll(), }, ); } diff --git a/dev/benchmarks/macrobenchmarks/lib/src/simple_scroll.dart b/dev/benchmarks/macrobenchmarks/lib/src/simple_scroll.dart new file mode 100644 index 00000000000..e4b82502e29 --- /dev/null +++ b/dev/benchmarks/macrobenchmarks/lib/src/simple_scroll.dart @@ -0,0 +1,17 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:flutter/material.dart'; + +class SimpleScroll extends StatelessWidget { + @override + Widget build(BuildContext context) { + return ListView( + children: [ + for (int n = 0; n < 200; n += 1) + Container(height: 40.0, child: Text('$n')), + ], + ); + } +} diff --git a/dev/benchmarks/macrobenchmarks/pubspec.yaml b/dev/benchmarks/macrobenchmarks/pubspec.yaml index e5e17e75dde..4043574c1e7 100644 --- a/dev/benchmarks/macrobenchmarks/pubspec.yaml +++ b/dev/benchmarks/macrobenchmarks/pubspec.yaml @@ -86,6 +86,7 @@ dev_dependencies: flutter_test: sdk: flutter test: 1.15.3 + e2e: 0.6.1 _fe_analyzer_shared: 5.0.0 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" analyzer: 0.39.13 # THIS LINE IS AUTOGENERATED - TO UPDATE USE "flutter update-packages --force-upgrade" @@ -291,4 +292,4 @@ flutter: fonts: - asset: packages/flutter_gallery_assets/fonts/GalleryIcons.ttf -# PUBSPEC CHECKSUM: 0d76 +# PUBSPEC CHECKSUM: 36c1 diff --git a/dev/benchmarks/macrobenchmarks/test/frame_policy.dart b/dev/benchmarks/macrobenchmarks/test/frame_policy.dart new file mode 100644 index 00000000000..93e4a0ab92d --- /dev/null +++ b/dev/benchmarks/macrobenchmarks/test/frame_policy.dart @@ -0,0 +1,105 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'package:flutter/foundation.dart'; +import 'package:flutter/material.dart'; +import 'package:flutter/gestures.dart'; +import 'package:flutter/scheduler.dart'; +import 'package:flutter_test/flutter_test.dart'; +import 'package:e2e/e2e.dart'; + +import 'package:macrobenchmarks/src/simple_scroll.dart'; + +void main() { + final E2EWidgetsFlutterBinding binding = + E2EWidgetsFlutterBinding.ensureInitialized() as E2EWidgetsFlutterBinding; + testWidgets( + 'Frame Counter and Input Delay for benchmarkLive', + (WidgetTester tester) async { + await tester.pumpWidget(MaterialApp(home: Scaffold(body: SimpleScroll()))); + await tester.pumpAndSettle(); + final Offset location = tester.getCenter(find.byType(ListView)); + int frameCount = 0; + final FrameCallback frameCounter = (Duration elapsed) { + frameCount += 1; + }; + tester.binding.addPersistentFrameCallback(frameCounter); + + const int timeInSecond = 1; + const Duration totalTime = Duration(seconds: timeInSecond); + const int moveEventNumber = timeInSecond * 120; // 120Hz + const Offset movePerRun = Offset(0.0, -200.0 / moveEventNumber); + final List records = [ + PointerEventRecord(Duration.zero, [ + PointerAddedEvent( + timeStamp: Duration.zero, + position: location, + ), + PointerDownEvent( + timeStamp: Duration.zero, + position: location, + pointer: 1, + ), + ]), + ...[ + for (int t=0; t < moveEventNumber; t++) + PointerEventRecord(totalTime * (t / moveEventNumber), [ + PointerMoveEvent( + timeStamp: totalTime * (t / moveEventNumber), + position: location + movePerRun * t.toDouble(), + pointer: 1, + delta: movePerRun, + ) + ]) + ], + PointerEventRecord(totalTime, [ + PointerUpEvent( + // Deviate a little from integer number of frames to reduce flakiness + timeStamp: totalTime - const Duration(milliseconds: 1), + position: location + movePerRun * moveEventNumber.toDouble(), + pointer: 1, + ) + ]) + ]; + + binding.framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.benchmarkLive; + List delays = await tester.handlePointerEventRecord(records); + await tester.pumpAndSettle(); + binding.reportData = { + 'benchmarkLive': _summarizeResult(frameCount, delays), + }; + await tester.idle(); + await tester.binding.delayed(const Duration(milliseconds: 250)); + + binding.framePolicy = LiveTestWidgetsFlutterBindingFramePolicy.fullyLive; + frameCount = 0; + delays = await tester.handlePointerEventRecord(records); + await tester.pumpAndSettle(); + binding.reportData['fullyLive'] = _summarizeResult(frameCount, delays); + await tester.idle(); + }, + ); +} + +Map _summarizeResult( + final int frameCount, + final List delays, +) { + assert(delays.length > 1); + final List delayedInMicro = delays.map( + (Duration delay) => delay.inMicroseconds, + ).toList(); + final List delayedInMicroSorted = List.from(delayedInMicro)..sort(); + final int index90th = (delayedInMicroSorted.length * 0.90).round(); + final int percentile90th = delayedInMicroSorted[index90th]; + final int sum = delayedInMicroSorted.reduce((int a, int b) => a + b); + final double averageDelay = sum.toDouble() / delayedInMicroSorted.length; + return { + 'frame_count': frameCount, + 'average_delay_millis': averageDelay / 1E3, + '90th_percentile_delay_millis': percentile90th / 1E3, + if (kDebugMode) + 'delaysInMicro': delayedInMicro, + }; +} diff --git a/dev/benchmarks/macrobenchmarks/test_driver/frame_policy_test.dart b/dev/benchmarks/macrobenchmarks/test_driver/frame_policy_test.dart new file mode 100644 index 00000000000..c941ac6b8d1 --- /dev/null +++ b/dev/benchmarks/macrobenchmarks/test_driver/frame_policy_test.dart @@ -0,0 +1,54 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; +import 'dart:convert'; +import 'dart:io'; + +import 'package:e2e/common.dart' as e2e; +import 'package:flutter_driver/flutter_driver.dart'; + +import 'package:path/path.dart' as path; + +Future main() async { + const Duration timeout = Duration(minutes: 1); + const String testName = 'frame_policy'; + + final FlutterDriver driver = await FlutterDriver.connect(); + String jsonResult; + jsonResult = await driver.requestData(null, timeout: timeout); + final e2e.Response response = e2e.Response.fromJson(jsonResult); + await driver.close(); + final Map benchmarkLiveResult = + response.data['benchmarkLive'] as Map; + final Map fullyLiveResult = + response.data['fullyLive'] as Map; + + if (response.allTestsPassed) { + if(benchmarkLiveResult['frame_count'] as int < 10 + || fullyLiveResult['frame_count'] as int < 10) { + print('Failure Details:\nNot Enough frames collected:' + 'benchmarkLive ${benchmarkLiveResult['frameCount']},' + '${fullyLiveResult['frameCount']}.'); + exit(1); + } + print('All tests passed.'); + const String destinationDirectory = 'build'; + await fs.directory(destinationDirectory).create(recursive: true); + final File file = fs.file(path.join( + destinationDirectory, + '${testName}_event_delay.json' + )); + await file.writeAsString(const JsonEncoder.withIndent(' ').convert( + { + 'benchmarkLive': benchmarkLiveResult, + 'fullyLive': fullyLiveResult, + }, + )); + exit(0); + } else { + print('Failure Details:\n${response.formattedFailureDetails}'); + exit(1); + } +} diff --git a/dev/devicelab/bin/tasks/frame_policy_delay_test_android.dart b/dev/devicelab/bin/tasks/frame_policy_delay_test_android.dart new file mode 100644 index 00000000000..094a4b70c0c --- /dev/null +++ b/dev/devicelab/bin/tasks/frame_policy_delay_test_android.dart @@ -0,0 +1,14 @@ +// Copyright 2014 The Flutter Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/framework/adb.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/tasks/perf_tests.dart'; + +Future main() async { + deviceOperatingSystem = DeviceOperatingSystem.android; + await task(createFramePolicyIntegrationTest()); +} diff --git a/dev/devicelab/lib/tasks/perf_tests.dart b/dev/devicelab/lib/tasks/perf_tests.dart index 9088e849fea..a9b343d3377 100644 --- a/dev/devicelab/lib/tasks/perf_tests.dart +++ b/dev/devicelab/lib/tasks/perf_tests.dart @@ -262,6 +262,49 @@ TaskFunction createsMultiWidgetConstructPerfTest() { ).run; } +TaskFunction createFramePolicyIntegrationTest() { + final String testDirectory = + '${flutterDirectory.path}/dev/benchmarks/macrobenchmarks'; + const String testTarget = 'test/frame_policy.dart'; + return () { + return inDirectory(testDirectory, () async { + final Device device = await devices.workingDevice; + await device.unlock(); + final String deviceId = device.deviceId; + await flutter('packages', options: ['get']); + + await flutter('drive', options: [ + '-v', + '--verbose-system-logs', + '--profile', + '-t', testTarget, + '-d', + deviceId, + ]); + final Map data = json.decode( + file('$testDirectory/build/frame_policy_event_delay.json').readAsStringSync(), + ) as Map; + final Map fullLiveData = data['fullyLive'] as Map; + final Map benchmarkLiveData = data['benchmarkLive'] as Map; + final Map dataFormated = { + 'average_delay_fullyLive_millis': + fullLiveData['average_delay_millis'], + 'average_delay_benchmarkLive_millis': + benchmarkLiveData['average_delay_millis'], + '90th_percentile_delay_fullyLive_millis': + fullLiveData['90th_percentile_delay_millis'], + '90th_percentile_delay_benchmarkLive_millis': + benchmarkLiveData['90th_percentile_delay_millis'], + }; + + return TaskResult.success( + dataFormated, + benchmarkScoreKeys: dataFormated.keys.toList(), + ); + }); + }; +} + /// Measure application startup performance. class StartupTest { const StartupTest(this.testDirectory, { this.reportMetrics = true }); diff --git a/dev/devicelab/manifest.yaml b/dev/devicelab/manifest.yaml index 4764d529c30..4cad3655b49 100644 --- a/dev/devicelab/manifest.yaml +++ b/dev/devicelab/manifest.yaml @@ -167,6 +167,13 @@ tasks: stage: devicelab required_agent_capabilities: ["linux/android"] + frame_policy_delay_test_android: + description: > + Tests the effect of LiveTestWidgetsFlutterBindingFramePolicy.benchmarkLive + stage: devicelab + required_agent_capabilities: ["linux/android"] + flaky: true + picture_cache_perf__timeline_summary: description: > Measures the runtime performance of raster caching many pictures on Android. diff --git a/packages/flutter_test/lib/src/binding.dart b/packages/flutter_test/lib/src/binding.dart index a47fdf1728f..500bc28d1ec 100644 --- a/packages/flutter_test/lib/src/binding.dart +++ b/packages/flutter_test/lib/src/binding.dart @@ -1264,6 +1264,21 @@ enum LiveTestWidgetsFlutterBindingFramePolicy { /// on the [SchedulerBinding.hasScheduledFrame] property to determine when the /// application has "settled". benchmark, + + /// Ignore any request from pump but respect other requests to schedule a + /// frame. + /// + /// This is used for running the test on a device, where scheduling of new + /// frames respects what the engine and the device needed. + /// + /// Compared to `fullyLive` this policy ignores the frame requests from pump + /// of the test code so that the frame scheduling respects the situation of + /// that for the real environment, and avoids waiting for the new frame beyond + /// the expected time. + /// + /// Compared to `benchmark` this policy can be used for capturing the + /// animation frames requested by the framework. + benchmarkLive, } /// A variant of [TestWidgetsFlutterBinding] for executing tests in @@ -1398,6 +1413,7 @@ class LiveTestWidgetsFlutterBinding extends TestWidgetsFlutterBinding { assert(_doDrawThisFrame == null); if (_expectingFrame || (framePolicy == LiveTestWidgetsFlutterBindingFramePolicy.fullyLive) || + (framePolicy == LiveTestWidgetsFlutterBindingFramePolicy.benchmarkLive) || (framePolicy == LiveTestWidgetsFlutterBindingFramePolicy.benchmark) || (framePolicy == LiveTestWidgetsFlutterBindingFramePolicy.fadePointers && _viewNeedsPaint)) { _doDrawThisFrame = true; @@ -1489,6 +1505,10 @@ class LiveTestWidgetsFlutterBinding extends TestWidgetsFlutterBinding { assert(inTest); assert(!_expectingFrame); assert(_pendingFrame == null); + if (framePolicy == LiveTestWidgetsFlutterBindingFramePolicy.benchmarkLive) { + // Ignore all pumps and just wait. + return delayed(duration ?? Duration.zero); + } return TestAsyncUtils.guard(() { if (duration != null) { Timer(duration, () { diff --git a/packages/flutter_test/lib/src/controller.dart b/packages/flutter_test/lib/src/controller.dart index 88e3f883807..759c946e647 100644 --- a/packages/flutter_test/lib/src/controller.dart +++ b/packages/flutter_test/lib/src/controller.dart @@ -406,9 +406,17 @@ abstract class WidgetController { /// The [PointerEventRecord.timeDelay] is used as the time delay of the events /// injection relative to the starting point of the method call. /// - /// Returns a list of the difference between [PointerEventRecord.timeDelay] - /// and the real delay time when the [PointerEventRecord.events] are processed. - /// The closer these values are to zero the more faithful it is to the + /// Returns a list of the difference between the real delay time when the + /// [PointerEventRecord.events] are processed and + /// [PointerEventRecord.timeDelay]. + /// - For [AutomatedTestWidgetsFlutterBinding] where the clock is fake, the + /// return value should be exact zeros. + /// - For [LiveTestWidgetsFlutterBinding], the values are typically small + /// positives, meaning the event happens a little later than the set time, + /// but a very small portion may have a tiny negatvie value for about tens of + /// microseconds. This is due to the nature of [Future.delayed]. + /// + /// The closer the return values are to zero the more faithful it is to the /// `records`. /// /// See [PointerEventRecord]. diff --git a/packages/flutter_test/lib/src/widget_tester.dart b/packages/flutter_test/lib/src/widget_tester.dart index 7b6f810cd94..a28f7e66dbd 100644 --- a/packages/flutter_test/lib/src/widget_tester.dart +++ b/packages/flutter_test/lib/src/widget_tester.dart @@ -480,7 +480,7 @@ class WidgetTester extends WidgetController implements HitTestDispatcher, Ticker final Duration timeDiff = record.timeDelay - now.difference(startTime); if (timeDiff.isNegative) { // Flush all past events - handleTimeStampDiff.add(timeDiff); + handleTimeStampDiff.add(-timeDiff); for (final PointerEvent event in record.events) { _handlePointerEvent(event, hitTestHistory); } @@ -490,7 +490,7 @@ class WidgetTester extends WidgetController implements HitTestDispatcher, Ticker await binding.pump(); await binding.delayed(timeDiff); handleTimeStampDiff.add( - record.timeDelay - binding.clock.now().difference(startTime), + binding.clock.now().difference(startTime) - record.timeDelay, ); for (final PointerEvent event in record.events) { _handlePointerEvent(event, hitTestHistory);