From b5f9612cac82aac0d4aa6ef393ccdf2a727b792d Mon Sep 17 00:00:00 2001 From: Greg Spencer Date: Tue, 29 Jun 2021 13:46:13 -0700 Subject: [PATCH] Randomize Framework tests, opt out some tests that currently fail. (#85159) This turns on order shuffling for all tests that don't fail with it on, marking those tests that do fail with a tag so that they will be run without shuffling on. To determine which tests fail with it on, I ran all the tests 100 times with different random shuffle seeds, and then also ran it with the date seeds from today until the end of July, and tagged all of the test suites (files) that fail, with a seed that caused them to fail. --- dev/bots/run_command.dart | 32 +++- dev/bots/test.dart | 138 +++++++++++++----- packages/flutter/dart_test.yaml | 4 + .../animation/animation_controller_test.dart | 6 + .../test/cupertino/action_sheet_test.dart | 6 + .../test/cupertino/date_picker_test.dart | 6 + .../cupertino/nav_bar_transition_test.dart | 6 + .../test/cupertino/text_field_test.dart | 6 + .../foundation/service_extensions_test.dart | 6 + .../flutter/test/gestures/debug_test.dart | 6 + .../test/gestures/long_press_test.dart | 6 + packages/flutter/test/gestures/tap_test.dart | 6 + packages/flutter/test/material/app_test.dart | 6 + .../flutter/test/material/dropdown_test.dart | 6 + .../material/paginated_data_table_test.dart | 6 + .../material/progress_indicator_test.dart | 6 + .../flutter/test/material/scaffold_test.dart | 6 + .../flutter/test/material/scrollbar_test.dart | 6 + .../test/material/text_field_test.dart | 6 + .../test/painting/image_stream_test.dart | 6 + .../flutter/test/physics/newton_test.dart | 6 + .../flutter/test/rendering/editable_test.dart | 6 + .../flutter/test/rendering/viewport_test.dart | 6 + .../test/scheduler/scheduler_test.dart | 6 + .../flutter/test/scheduler/ticker_test.dart | 6 + .../test/services/platform_channel_test.dart | 6 + .../test/widgets/app_overrides_test.dart | 6 + .../flutter/test/widgets/binding_test.dart | 6 + packages/flutter/test/widgets/clip_test.dart | 6 + packages/flutter/test/widgets/debug_test.dart | 6 + .../test/widgets/dismissible_test.dart | 6 + .../flutter/test/widgets/draggable_test.dart | 6 + .../test/widgets/platform_view_test.dart | 6 + .../flutter/test/widgets/routes_test.dart | 6 + .../scroll_aware_image_provider_test.dart | 6 + .../test/widgets/widget_inspector_test.dart | 15 +- 36 files changed, 333 insertions(+), 48 deletions(-) create mode 100644 packages/flutter/dart_test.yaml diff --git a/dev/bots/run_command.dart b/dev/bots/run_command.dart index 5a5aaeb35e3..8e05a0d1405 100644 --- a/dev/bots/run_command.dart +++ b/dev/bots/run_command.dart @@ -22,6 +22,7 @@ Stream runAndGetStdout(String executable, List arguments, { String workingDirectory, Map environment, bool expectNonZeroExit = false, + bool expectNoTests = false, }) async* { final StreamController output = StreamController(); final Future command = runCommand( @@ -30,6 +31,7 @@ Stream runAndGetStdout(String executable, List arguments, { workingDirectory: workingDirectory, environment: environment, expectNonZeroExit: expectNonZeroExit, + expectNoTests: expectNoTests, // Capture the output so it's not printed to the console by default. outputMode: OutputMode.capture, outputListener: (String line, io.Process process) { @@ -113,7 +115,8 @@ Future startCommand(String executable, List arguments, { environment: environment, ); - Future>> savedStdout, savedStderr; + Future>> savedStdout = Future>>.value(>[]); + Future>> savedStderr = Future>>.value(>[]); final Stream> stdoutSource = process.stdout .transform(const Utf8Decoder()) .transform(const LineSplitter()) @@ -128,8 +131,14 @@ Future startCommand(String executable, List arguments, { .transform(const Utf8Encoder()); switch (outputMode) { case OutputMode.print: - stdoutSource.listen(io.stdout.add); - process.stderr.listen(io.stderr.add); + stdoutSource.listen((List output) { + io.stdout.add(output); + savedStdout.then((List> list) => list.add(output)); + }); + process.stderr.listen((List output) { + io.stderr.add(output); + savedStdout.then((List> list) => list.add(output)); + }); break; case OutputMode.capture: savedStdout = stdoutSource.toList(); @@ -158,6 +167,7 @@ Future runCommand(String executable, List arguments, { String workingDirectory, Map environment, bool expectNonZeroExit = false, + bool expectNoTests = false, int expectedExitCode, String failureMessage, OutputMode outputMode = OutputMode.print, @@ -182,7 +192,21 @@ Future runCommand(String executable, List arguments, { final CommandResult result = await command.onExit; - if ((result.exitCode == 0) == expectNonZeroExit || (expectedExitCode != null && result.exitCode != expectedExitCode)) { + // Currently, the test infrastructure fails if it doesn't find any tests to + // run, but in the case of tests tagged as "no-shuffle", there might either be + // none that can be shuffled, or none that shouldn't be shuffled, and since + // we're running it twice to get all the tests in either category, it + // shouldn't fail if no tests are run. + // + // TODO(gspencergoog): This is a workaround until + // https://github.com/dart-lang/test/issues/1546 is addressed. Remove the + // workaround (parsing the test output) when/if that issue is fixed. + final bool skipErrorExit = expectNoTests && + result != null && + result.flattenedStdout != null && + result.flattenedStdout.trimRight().endsWith('No tests ran.'); + + if (!skipErrorExit && ((result.exitCode == 0) == expectNonZeroExit || (expectedExitCode != null && result.exitCode != expectedExitCode))) { // Print the output when we get unexpected results (unless output was // printed already). switch (outputMode) { diff --git a/dev/bots/test.dart b/dev/bots/test.dart index ee098f80893..d2f69b332b2 100644 --- a/dev/bots/test.dart +++ b/dev/bots/test.dart @@ -620,6 +620,7 @@ Future _runFrameworkTests() async { path.join(flutterRoot, 'packages', 'flutter'), options: [trackWidgetCreationOption, ...soundNullSafetyOptions], tests: [ path.join('test', 'widgets') + path.separator ], + shuffleOrder: true, ); } // Try compiling code outside of the packages/flutter directory with and without --track-widget-creation @@ -627,6 +628,7 @@ Future _runFrameworkTests() async { await _runFlutterTest( path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery'), options: [trackWidgetCreationOption], + shuffleOrder: true, ); } // Run release mode tests (see packages/flutter/test_release/README.md) @@ -634,6 +636,7 @@ Future _runFrameworkTests() async { path.join(flutterRoot, 'packages', 'flutter'), options: ['--dart-define=dart.vm.product=true', ...soundNullSafetyOptions], tests: ['test_release${path.separator}'], + shuffleOrder: true, ); } @@ -650,6 +653,7 @@ Future _runFrameworkTests() async { path.join(flutterRoot, 'packages', 'flutter'), options: [trackWidgetCreationOption, ...soundNullSafetyOptions], tests: tests, + shuffleOrder: true, ); } } @@ -1376,6 +1380,18 @@ Future _pubRunTest(String workingDirectory, { } } +// Used as an argument to the runTests internal function in the _runFlutterTest +// function. +enum _ShuffleMode { + // Runs only the tests not tagged with "no-shuffle" in a shuffled order. + shuffled, + // Runs only the tests that would fail if shuffled (those tagged with + // "no-shuffle"). + unshuffled, + // Runs all tests without shuffling. + sequential, +} + Future _runFlutterTest(String workingDirectory, { String script, bool expectFailure = false, @@ -1383,20 +1399,20 @@ Future _runFlutterTest(String workingDirectory, { OutputChecker outputChecker, List options = const [], bool skip = false, + bool shuffleOrder = false, Map environment, List tests = const [], }) async { assert(!printOutput || outputChecker == null, 'Output either can be printed or checked but not both'); - final List args = [ - 'test', + final List testArgs = [ ...options, ...?flutterTestArgs, ]; final bool shouldProcessOutput = useFlutterTestFormatter && !expectFailure && !options.contains('--coverage'); if (shouldProcessOutput) - args.add('--machine'); + testArgs.add('--machine'); if (script != null) { final String fullScriptPath = path.join(workingDirectory, script); @@ -1410,56 +1426,98 @@ Future _runFlutterTest(String workingDirectory, { print('This is one of the tests that is normally skipped in this configuration.'); exit(1); } - args.add(script); + testArgs.add(script); } - args.addAll(tests); + testArgs.addAll(tests); - if (!shouldProcessOutput) { - final OutputMode outputMode = outputChecker == null && printOutput - ? OutputMode.print - : OutputMode.capture; - - final CommandResult result = await runCommand( - flutter, - args, - workingDirectory: workingDirectory, - expectNonZeroExit: expectFailure, - outputMode: outputMode, - skip: skip, - environment: environment, - ); - - if (outputChecker != null) { - final String message = outputChecker(result); - if (message != null) - exitWithError([message]); + Future runTests(_ShuffleMode shuffleMode) async { + int seed = 0; + final List shuffleArgs = []; + switch (shuffleMode) { + case _ShuffleMode.shuffled: + final DateTime now = DateTime.now(); + // Generates YYYYMMDD as the seed, so that testing continues to fail on the + // day it was originally run, and on other days the seed can be used to + // replicate failures. + seed = now.year * 10000 + now.month * 100 + now.day; + print('Shuffling test order using arbitrary daily seed $seed.'); + print('To run locally, run "flutter test -x no-shuffle --test-randomize-ordering-seed=$seed ' + '&& flutter test -t no-shuffle" to run all tests (the latter part runs all the tests that ' + "currently don't work when shuffled, the former skips those tests and shuffles the order " + 'of the remaining tests).'); + shuffleArgs.addAll(['-x', 'no-shuffle', '--test-randomize-ordering-seed=$seed']); + break; + case _ShuffleMode.unshuffled: + shuffleArgs.addAll(['-t', 'no-shuffle']); + break; + case _ShuffleMode.sequential: + break; } - return; - } + final List args = [ + 'test', + ...shuffleArgs, + ...testArgs, + ]; + if (!shouldProcessOutput) { + final OutputMode outputMode = outputChecker == null && printOutput + ? OutputMode.print + : OutputMode.capture; - if (useFlutterTestFormatter) { - final FlutterCompactFormatter formatter = FlutterCompactFormatter(); - Stream testOutput; - try { - testOutput = runAndGetStdout( + final CommandResult result = await runCommand( flutter, args, workingDirectory: workingDirectory, expectNonZeroExit: expectFailure, + expectNoTests: shuffleMode == _ShuffleMode.unshuffled, + outputMode: outputMode, + skip: skip, environment: environment, ); - } finally { - formatter.finish(); + if (outputChecker != null) { + final String message = outputChecker(result); + if (message != null) + exitWithError([message]); + } + return; } - await _processTestOutput(formatter, testOutput); + + if (useFlutterTestFormatter) { + final FlutterCompactFormatter formatter = FlutterCompactFormatter(); + Stream testOutput; + try { + testOutput = runAndGetStdout( + flutter, + args, + workingDirectory: workingDirectory, + expectNonZeroExit: expectFailure, + expectNoTests: shuffleMode == _ShuffleMode.unshuffled, + environment: environment, + ); + } finally { + formatter.finish(); + } + await _processTestOutput(formatter, testOutput); + } else { + await runCommand( + flutter, + args, + workingDirectory: workingDirectory, + expectNonZeroExit: expectFailure, + expectNoTests: shuffleMode == _ShuffleMode.unshuffled, + ); + } + } + + if (shuffleOrder) { + // Runs only the tests not tagged with "no-shuffle" in a shuffled order. + await runTests(_ShuffleMode.shuffled); + // Runs only the tests that would fail if shuffled (those tagged with + // "no-shuffle"). + await runTests(_ShuffleMode.unshuffled); } else { - await runCommand( - flutter, - args, - workingDirectory: workingDirectory, - expectNonZeroExit: expectFailure, - ); + // Runs all tests without shuffling. + await runTests(_ShuffleMode.sequential); } } diff --git a/packages/flutter/dart_test.yaml b/packages/flutter/dart_test.yaml new file mode 100644 index 00000000000..fd9ab11bcd9 --- /dev/null +++ b/packages/flutter/dart_test.yaml @@ -0,0 +1,4 @@ +tags: + # This tag tells the test framework to not shuffle the test order according to + # the --test-randomize-ordering-seed for the suites that have this tag. + no-shuffle: diff --git a/packages/flutter/test/animation/animation_controller_test.dart b/packages/flutter/test/animation/animation_controller_test.dart index 2aff62d3a83..526202f7984 100644 --- a/packages/flutter/test/animation/animation_controller_test.dart +++ b/packages/flutter/test/animation/animation_controller_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'dart:ui' as ui; import 'package:flutter/foundation.dart'; diff --git a/packages/flutter/test/cupertino/action_sheet_test.dart b/packages/flutter/test/cupertino/action_sheet_test.dart index 1f46897bc6e..a664705aa73 100644 --- a/packages/flutter/test/cupertino/action_sheet_test.dart +++ b/packages/flutter/test/cupertino/action_sheet_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/cupertino.dart'; import 'package:flutter/material.dart'; diff --git a/packages/flutter/test/cupertino/date_picker_test.dart b/packages/flutter/test/cupertino/date_picker_test.dart index 6180e15ff26..74020f97be3 100644 --- a/packages/flutter/test/cupertino/date_picker_test.dart +++ b/packages/flutter/test/cupertino/date_picker_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'dart:ui'; import 'package:flutter/cupertino.dart'; diff --git a/packages/flutter/test/cupertino/nav_bar_transition_test.dart b/packages/flutter/test/cupertino/nav_bar_transition_test.dart index d9a4ba8d963..9a4914118ef 100644 --- a/packages/flutter/test/cupertino/nav_bar_transition_test.dart +++ b/packages/flutter/test/cupertino/nav_bar_transition_test.dart @@ -3,6 +3,12 @@ // found in the LICENSE file. @TestOn('!chrome') +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=456" +@Tags(['no-shuffle']) + import 'package:flutter/cupertino.dart'; import 'package:flutter/rendering.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/packages/flutter/test/cupertino/text_field_test.dart b/packages/flutter/test/cupertino/text_field_test.dart index feccf73253b..f451e64d56a 100644 --- a/packages/flutter/test/cupertino/text_field_test.dart +++ b/packages/flutter/test/cupertino/text_field_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'dart:ui' as ui show BoxHeightStyle, BoxWidthStyle, Color; import 'package:flutter/cupertino.dart'; diff --git a/packages/flutter/test/foundation/service_extensions_test.dart b/packages/flutter/test/foundation/service_extensions_test.dart index e9f5e6c35fa..c9d62767d6a 100644 --- a/packages/flutter/test/foundation/service_extensions_test.dart +++ b/packages/flutter/test/foundation/service_extensions_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'dart:async'; import 'dart:convert'; import 'dart:ui' as ui; diff --git a/packages/flutter/test/gestures/debug_test.dart b/packages/flutter/test/gestures/debug_test.dart index b2b24bc04d4..c15c43398e7 100644 --- a/packages/flutter/test/gestures/debug_test.dart +++ b/packages/flutter/test/gestures/debug_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/foundation.dart'; import 'package:flutter/gestures.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/packages/flutter/test/gestures/long_press_test.dart b/packages/flutter/test/gestures/long_press_test.dart index a07de9471f1..4f962d8afa8 100644 --- a/packages/flutter/test/gestures/long_press_test.dart +++ b/packages/flutter/test/gestures/long_press_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/gestures.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/packages/flutter/test/gestures/tap_test.dart b/packages/flutter/test/gestures/tap_test.dart index 2266b5478db..5ef55355def 100644 --- a/packages/flutter/test/gestures/tap_test.dart +++ b/packages/flutter/test/gestures/tap_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=1000" +@Tags(['no-shuffle']) + import 'package:flutter/foundation.dart'; import 'package:flutter/gestures.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/packages/flutter/test/material/app_test.dart b/packages/flutter/test/material/app_test.dart index 55bd0fce88a..fbcc02f10a1 100644 --- a/packages/flutter/test/material/app_test.dart +++ b/packages/flutter/test/material/app_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/cupertino.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; diff --git a/packages/flutter/test/material/dropdown_test.dart b/packages/flutter/test/material/dropdown_test.dart index 45a6254cae7..065c2c14fd6 100644 --- a/packages/flutter/test/material/dropdown_test.dart +++ b/packages/flutter/test/material/dropdown_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=1408669812" +@Tags(['no-shuffle']) + import 'dart:math' as math; import 'dart:ui' show window; diff --git a/packages/flutter/test/material/paginated_data_table_test.dart b/packages/flutter/test/material/paginated_data_table_test.dart index d1b00649dee..9dc12e3b135 100644 --- a/packages/flutter/test/material/paginated_data_table_test.dart +++ b/packages/flutter/test/material/paginated_data_table_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=1000" +@Tags(['no-shuffle']) + import 'package:flutter/gestures.dart' show DragStartBehavior; import 'package:flutter/material.dart'; import 'package:flutter/rendering.dart'; diff --git a/packages/flutter/test/material/progress_indicator_test.dart b/packages/flutter/test/material/progress_indicator_test.dart index 9f7a8ccea23..8934b16d421 100644 --- a/packages/flutter/test/material/progress_indicator_test.dart +++ b/packages/flutter/test/material/progress_indicator_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=456" +@Tags(['no-shuffle']) + import 'package:flutter/cupertino.dart'; import 'package:flutter/material.dart'; import 'package:flutter/rendering.dart'; diff --git a/packages/flutter/test/material/scaffold_test.dart b/packages/flutter/test/material/scaffold_test.dart index 8f4a55fcd5e..47464a09555 100644 --- a/packages/flutter/test/material/scaffold_test.dart +++ b/packages/flutter/test/material/scaffold_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/foundation.dart'; import 'package:flutter/gestures.dart' show DragStartBehavior; import 'package:flutter/material.dart'; diff --git a/packages/flutter/test/material/scrollbar_test.dart b/packages/flutter/test/material/scrollbar_test.dart index 97b7facd969..4ae04ec6a45 100644 --- a/packages/flutter/test/material/scrollbar_test.dart +++ b/packages/flutter/test/material/scrollbar_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=382757700" +@Tags(['no-shuffle']) + import 'dart:ui' as ui; import 'package:flutter/cupertino.dart'; diff --git a/packages/flutter/test/material/text_field_test.dart b/packages/flutter/test/material/text_field_test.dart index c5f86ec41f1..17c77ea5eab 100644 --- a/packages/flutter/test/material/text_field_test.dart +++ b/packages/flutter/test/material/text_field_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=3890307731" +@Tags(['no-shuffle']) + import 'dart:math' as math; import 'dart:ui' as ui show window, BoxHeightStyle, BoxWidthStyle; diff --git a/packages/flutter/test/painting/image_stream_test.dart b/packages/flutter/test/painting/image_stream_test.dart index 274fd32a001..7356ef66e90 100644 --- a/packages/flutter/test/painting/image_stream_test.dart +++ b/packages/flutter/test/painting/image_stream_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=456" +@Tags(['no-shuffle']) + import 'dart:async'; import 'dart:typed_data'; import 'dart:ui'; diff --git a/packages/flutter/test/physics/newton_test.dart b/packages/flutter/test/physics/newton_test.dart index 34556213e46..c738b3188f8 100644 --- a/packages/flutter/test/physics/newton_test.dart +++ b/packages/flutter/test/physics/newton_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/foundation.dart'; import 'package:flutter/physics.dart'; import 'package:flutter/widgets.dart'; diff --git a/packages/flutter/test/rendering/editable_test.dart b/packages/flutter/test/rendering/editable_test.dart index a5a4b426d7f..9ab218d6f18 100644 --- a/packages/flutter/test/rendering/editable_test.dart +++ b/packages/flutter/test/rendering/editable_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=20210704" +@Tags(['no-shuffle']) + import 'dart:io' show Platform; import 'package:flutter/foundation.dart'; diff --git a/packages/flutter/test/rendering/viewport_test.dart b/packages/flutter/test/rendering/viewport_test.dart index 14a1b2c5be6..9a19ae6c37e 100644 --- a/packages/flutter/test/rendering/viewport_test.dart +++ b/packages/flutter/test/rendering/viewport_test.dart @@ -7,6 +7,12 @@ // initialize a binding, which rendering_tester will attempt to re-initialize // (or vice versa). +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/foundation.dart'; import 'package:flutter/rendering.dart'; import 'package:flutter/widgets.dart'; diff --git a/packages/flutter/test/scheduler/scheduler_test.dart b/packages/flutter/test/scheduler/scheduler_test.dart index a5913cfa98c..9687eb590d3 100644 --- a/packages/flutter/test/scheduler/scheduler_test.dart +++ b/packages/flutter/test/scheduler/scheduler_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'dart:async'; import 'dart:ui' show window; diff --git a/packages/flutter/test/scheduler/ticker_test.dart b/packages/flutter/test/scheduler/ticker_test.dart index 6c8dedc5bca..9e2dd3b358b 100644 --- a/packages/flutter/test/scheduler/ticker_test.dart +++ b/packages/flutter/test/scheduler/ticker_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=4281596210" +@Tags(['no-shuffle']) + import 'package:flutter/foundation.dart'; import 'package:flutter/scheduler.dart'; import 'package:flutter/services.dart'; diff --git a/packages/flutter/test/services/platform_channel_test.dart b/packages/flutter/test/services/platform_channel_test.dart index a97fa9bb1ff..e0ddfc89d79 100644 --- a/packages/flutter/test/services/platform_channel_test.dart +++ b/packages/flutter/test/services/platform_channel_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=456" +@Tags(['no-shuffle']) + import 'package:flutter/services.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/packages/flutter/test/widgets/app_overrides_test.dart b/packages/flutter/test/widgets/app_overrides_test.dart index 9599ca25165..f8c2675660f 100644 --- a/packages/flutter/test/widgets/app_overrides_test.dart +++ b/packages/flutter/test/widgets/app_overrides_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/widgets.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/packages/flutter/test/widgets/binding_test.dart b/packages/flutter/test/widgets/binding_test.dart index db64af8249b..705768f533f 100644 --- a/packages/flutter/test/widgets/binding_test.dart +++ b/packages/flutter/test/widgets/binding_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=382757700" +@Tags(['no-shuffle']) + import 'package:flutter/foundation.dart'; import 'package:flutter/services.dart'; import 'package:flutter/widgets.dart'; diff --git a/packages/flutter/test/widgets/clip_test.dart b/packages/flutter/test/widgets/clip_test.dart index 0d1f306d151..79e5fab035d 100644 --- a/packages/flutter/test/widgets/clip_test.dart +++ b/packages/flutter/test/widgets/clip_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=456" +@Tags(['no-shuffle']) + import 'package:flutter/material.dart'; import 'package:flutter/rendering.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/packages/flutter/test/widgets/debug_test.dart b/packages/flutter/test/widgets/debug_test.dart index 888b0799f0c..bf1a129741b 100644 --- a/packages/flutter/test/widgets/debug_test.dart +++ b/packages/flutter/test/widgets/debug_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/foundation.dart'; import 'package:flutter/widgets.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/packages/flutter/test/widgets/dismissible_test.dart b/packages/flutter/test/widgets/dismissible_test.dart index e8b55824491..1cc7165b09d 100644 --- a/packages/flutter/test/widgets/dismissible_test.dart +++ b/packages/flutter/test/widgets/dismissible_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=456" +@Tags(['no-shuffle']) + import 'package:flutter/gestures.dart' show DragStartBehavior; import 'package:flutter/widgets.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/packages/flutter/test/widgets/draggable_test.dart b/packages/flutter/test/widgets/draggable_test.dart index ef50d705c23..b52a5797af0 100644 --- a/packages/flutter/test/widgets/draggable_test.dart +++ b/packages/flutter/test/widgets/draggable_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/gestures.dart'; import 'package:flutter/material.dart'; import 'package:flutter/semantics.dart'; diff --git a/packages/flutter/test/widgets/platform_view_test.dart b/packages/flutter/test/widgets/platform_view_test.dart index d3cfbce40df..c645602571b 100644 --- a/packages/flutter/test/widgets/platform_view_test.dart +++ b/packages/flutter/test/widgets/platform_view_test.dart @@ -3,6 +3,12 @@ // found in the LICENSE file. @TestOn('!chrome') +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=4281596210" +@Tags(['no-shuffle']) + import 'dart:async'; import 'dart:typed_data'; import 'dart:ui'; diff --git a/packages/flutter/test/widgets/routes_test.dart b/packages/flutter/test/widgets/routes_test.dart index 6397fcf84bb..c07dad3227a 100644 --- a/packages/flutter/test/widgets/routes_test.dart +++ b/packages/flutter/test/widgets/routes_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'dart:collection'; import 'package:flutter/material.dart'; diff --git a/packages/flutter/test/widgets/scroll_aware_image_provider_test.dart b/packages/flutter/test/widgets/scroll_aware_image_provider_test.dart index 954d1e84b41..a5d66d2a383 100644 --- a/packages/flutter/test/widgets/scroll_aware_image_provider_test.dart +++ b/packages/flutter/test/widgets/scroll_aware_image_provider_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'dart:ui' as ui show Image; import 'package:flutter/widgets.dart'; diff --git a/packages/flutter/test/widgets/widget_inspector_test.dart b/packages/flutter/test/widgets/widget_inspector_test.dart index bebbc5749c6..229aa7cf612 100644 --- a/packages/flutter/test/widgets/widget_inspector_test.dart +++ b/packages/flutter/test/widgets/widget_inspector_test.dart @@ -3,6 +3,13 @@ // found in the LICENSE file. @TestOn('!chrome') + +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=456" +@Tags(['no-shuffle']) + import 'dart:async'; import 'dart:convert'; import 'dart:math'; @@ -1983,7 +1990,7 @@ class _TestWidgetInspectorService extends TestWidgetInspectorService { _CreationLocation location = knownLocations[id]!; expect(location.file, equals(file)); // ClockText widget. - expect(location.line, equals(53)); + expect(location.line, equals(60)); expect(location.column, equals(9)); expect(location.name, equals('ClockText')); expect(count, equals(1)); @@ -1993,7 +2000,7 @@ class _TestWidgetInspectorService extends TestWidgetInspectorService { location = knownLocations[id]!; expect(location.file, equals(file)); // Text widget in _ClockTextState build method. - expect(location.line, equals(91)); + expect(location.line, equals(98)); expect(location.column, equals(12)); expect(location.name, equals('Text')); expect(count, equals(1)); @@ -2020,7 +2027,7 @@ class _TestWidgetInspectorService extends TestWidgetInspectorService { location = knownLocations[id]!; expect(location.file, equals(file)); // ClockText widget. - expect(location.line, equals(53)); + expect(location.line, equals(60)); expect(location.column, equals(9)); expect(location.name, equals('ClockText')); expect(count, equals(3)); // 3 clock widget instances rebuilt. @@ -2030,7 +2037,7 @@ class _TestWidgetInspectorService extends TestWidgetInspectorService { location = knownLocations[id]!; expect(location.file, equals(file)); // Text widget in _ClockTextState build method. - expect(location.line, equals(91)); + expect(location.line, equals(98)); expect(location.column, equals(12)); expect(location.name, equals('Text')); expect(count, equals(3)); // 3 clock widget instances rebuilt.