diff --git a/dev/bots/run_command.dart b/dev/bots/run_command.dart index 5a5aaeb35e3..8e05a0d1405 100644 --- a/dev/bots/run_command.dart +++ b/dev/bots/run_command.dart @@ -22,6 +22,7 @@ Stream runAndGetStdout(String executable, List arguments, { String workingDirectory, Map environment, bool expectNonZeroExit = false, + bool expectNoTests = false, }) async* { final StreamController output = StreamController(); final Future command = runCommand( @@ -30,6 +31,7 @@ Stream runAndGetStdout(String executable, List arguments, { workingDirectory: workingDirectory, environment: environment, expectNonZeroExit: expectNonZeroExit, + expectNoTests: expectNoTests, // Capture the output so it's not printed to the console by default. outputMode: OutputMode.capture, outputListener: (String line, io.Process process) { @@ -113,7 +115,8 @@ Future startCommand(String executable, List arguments, { environment: environment, ); - Future>> savedStdout, savedStderr; + Future>> savedStdout = Future>>.value(>[]); + Future>> savedStderr = Future>>.value(>[]); final Stream> stdoutSource = process.stdout .transform(const Utf8Decoder()) .transform(const LineSplitter()) @@ -128,8 +131,14 @@ Future startCommand(String executable, List arguments, { .transform(const Utf8Encoder()); switch (outputMode) { case OutputMode.print: - stdoutSource.listen(io.stdout.add); - process.stderr.listen(io.stderr.add); + stdoutSource.listen((List output) { + io.stdout.add(output); + savedStdout.then((List> list) => list.add(output)); + }); + process.stderr.listen((List output) { + io.stderr.add(output); + savedStdout.then((List> list) => list.add(output)); + }); break; case OutputMode.capture: savedStdout = stdoutSource.toList(); @@ -158,6 +167,7 @@ Future runCommand(String executable, List arguments, { String workingDirectory, Map environment, bool expectNonZeroExit = false, + bool expectNoTests = false, int expectedExitCode, String failureMessage, OutputMode outputMode = OutputMode.print, @@ -182,7 +192,21 @@ Future runCommand(String executable, List arguments, { final CommandResult result = await command.onExit; - if ((result.exitCode == 0) == expectNonZeroExit || (expectedExitCode != null && result.exitCode != expectedExitCode)) { + // Currently, the test infrastructure fails if it doesn't find any tests to + // run, but in the case of tests tagged as "no-shuffle", there might either be + // none that can be shuffled, or none that shouldn't be shuffled, and since + // we're running it twice to get all the tests in either category, it + // shouldn't fail if no tests are run. + // + // TODO(gspencergoog): This is a workaround until + // https://github.com/dart-lang/test/issues/1546 is addressed. Remove the + // workaround (parsing the test output) when/if that issue is fixed. + final bool skipErrorExit = expectNoTests && + result != null && + result.flattenedStdout != null && + result.flattenedStdout.trimRight().endsWith('No tests ran.'); + + if (!skipErrorExit && ((result.exitCode == 0) == expectNonZeroExit || (expectedExitCode != null && result.exitCode != expectedExitCode))) { // Print the output when we get unexpected results (unless output was // printed already). switch (outputMode) { diff --git a/dev/bots/test.dart b/dev/bots/test.dart index ee098f80893..d2f69b332b2 100644 --- a/dev/bots/test.dart +++ b/dev/bots/test.dart @@ -620,6 +620,7 @@ Future _runFrameworkTests() async { path.join(flutterRoot, 'packages', 'flutter'), options: [trackWidgetCreationOption, ...soundNullSafetyOptions], tests: [ path.join('test', 'widgets') + path.separator ], + shuffleOrder: true, ); } // Try compiling code outside of the packages/flutter directory with and without --track-widget-creation @@ -627,6 +628,7 @@ Future _runFrameworkTests() async { await _runFlutterTest( path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery'), options: [trackWidgetCreationOption], + shuffleOrder: true, ); } // Run release mode tests (see packages/flutter/test_release/README.md) @@ -634,6 +636,7 @@ Future _runFrameworkTests() async { path.join(flutterRoot, 'packages', 'flutter'), options: ['--dart-define=dart.vm.product=true', ...soundNullSafetyOptions], tests: ['test_release${path.separator}'], + shuffleOrder: true, ); } @@ -650,6 +653,7 @@ Future _runFrameworkTests() async { path.join(flutterRoot, 'packages', 'flutter'), options: [trackWidgetCreationOption, ...soundNullSafetyOptions], tests: tests, + shuffleOrder: true, ); } } @@ -1376,6 +1380,18 @@ Future _pubRunTest(String workingDirectory, { } } +// Used as an argument to the runTests internal function in the _runFlutterTest +// function. +enum _ShuffleMode { + // Runs only the tests not tagged with "no-shuffle" in a shuffled order. + shuffled, + // Runs only the tests that would fail if shuffled (those tagged with + // "no-shuffle"). + unshuffled, + // Runs all tests without shuffling. + sequential, +} + Future _runFlutterTest(String workingDirectory, { String script, bool expectFailure = false, @@ -1383,20 +1399,20 @@ Future _runFlutterTest(String workingDirectory, { OutputChecker outputChecker, List options = const [], bool skip = false, + bool shuffleOrder = false, Map environment, List tests = const [], }) async { assert(!printOutput || outputChecker == null, 'Output either can be printed or checked but not both'); - final List args = [ - 'test', + final List testArgs = [ ...options, ...?flutterTestArgs, ]; final bool shouldProcessOutput = useFlutterTestFormatter && !expectFailure && !options.contains('--coverage'); if (shouldProcessOutput) - args.add('--machine'); + testArgs.add('--machine'); if (script != null) { final String fullScriptPath = path.join(workingDirectory, script); @@ -1410,56 +1426,98 @@ Future _runFlutterTest(String workingDirectory, { print('This is one of the tests that is normally skipped in this configuration.'); exit(1); } - args.add(script); + testArgs.add(script); } - args.addAll(tests); + testArgs.addAll(tests); - if (!shouldProcessOutput) { - final OutputMode outputMode = outputChecker == null && printOutput - ? OutputMode.print - : OutputMode.capture; - - final CommandResult result = await runCommand( - flutter, - args, - workingDirectory: workingDirectory, - expectNonZeroExit: expectFailure, - outputMode: outputMode, - skip: skip, - environment: environment, - ); - - if (outputChecker != null) { - final String message = outputChecker(result); - if (message != null) - exitWithError([message]); + Future runTests(_ShuffleMode shuffleMode) async { + int seed = 0; + final List shuffleArgs = []; + switch (shuffleMode) { + case _ShuffleMode.shuffled: + final DateTime now = DateTime.now(); + // Generates YYYYMMDD as the seed, so that testing continues to fail on the + // day it was originally run, and on other days the seed can be used to + // replicate failures. + seed = now.year * 10000 + now.month * 100 + now.day; + print('Shuffling test order using arbitrary daily seed $seed.'); + print('To run locally, run "flutter test -x no-shuffle --test-randomize-ordering-seed=$seed ' + '&& flutter test -t no-shuffle" to run all tests (the latter part runs all the tests that ' + "currently don't work when shuffled, the former skips those tests and shuffles the order " + 'of the remaining tests).'); + shuffleArgs.addAll(['-x', 'no-shuffle', '--test-randomize-ordering-seed=$seed']); + break; + case _ShuffleMode.unshuffled: + shuffleArgs.addAll(['-t', 'no-shuffle']); + break; + case _ShuffleMode.sequential: + break; } - return; - } + final List args = [ + 'test', + ...shuffleArgs, + ...testArgs, + ]; + if (!shouldProcessOutput) { + final OutputMode outputMode = outputChecker == null && printOutput + ? OutputMode.print + : OutputMode.capture; - if (useFlutterTestFormatter) { - final FlutterCompactFormatter formatter = FlutterCompactFormatter(); - Stream testOutput; - try { - testOutput = runAndGetStdout( + final CommandResult result = await runCommand( flutter, args, workingDirectory: workingDirectory, expectNonZeroExit: expectFailure, + expectNoTests: shuffleMode == _ShuffleMode.unshuffled, + outputMode: outputMode, + skip: skip, environment: environment, ); - } finally { - formatter.finish(); + if (outputChecker != null) { + final String message = outputChecker(result); + if (message != null) + exitWithError([message]); + } + return; } - await _processTestOutput(formatter, testOutput); + + if (useFlutterTestFormatter) { + final FlutterCompactFormatter formatter = FlutterCompactFormatter(); + Stream testOutput; + try { + testOutput = runAndGetStdout( + flutter, + args, + workingDirectory: workingDirectory, + expectNonZeroExit: expectFailure, + expectNoTests: shuffleMode == _ShuffleMode.unshuffled, + environment: environment, + ); + } finally { + formatter.finish(); + } + await _processTestOutput(formatter, testOutput); + } else { + await runCommand( + flutter, + args, + workingDirectory: workingDirectory, + expectNonZeroExit: expectFailure, + expectNoTests: shuffleMode == _ShuffleMode.unshuffled, + ); + } + } + + if (shuffleOrder) { + // Runs only the tests not tagged with "no-shuffle" in a shuffled order. + await runTests(_ShuffleMode.shuffled); + // Runs only the tests that would fail if shuffled (those tagged with + // "no-shuffle"). + await runTests(_ShuffleMode.unshuffled); } else { - await runCommand( - flutter, - args, - workingDirectory: workingDirectory, - expectNonZeroExit: expectFailure, - ); + // Runs all tests without shuffling. + await runTests(_ShuffleMode.sequential); } } diff --git a/packages/flutter/dart_test.yaml b/packages/flutter/dart_test.yaml new file mode 100644 index 00000000000..fd9ab11bcd9 --- /dev/null +++ b/packages/flutter/dart_test.yaml @@ -0,0 +1,4 @@ +tags: + # This tag tells the test framework to not shuffle the test order according to + # the --test-randomize-ordering-seed for the suites that have this tag. + no-shuffle: diff --git a/packages/flutter/test/animation/animation_controller_test.dart b/packages/flutter/test/animation/animation_controller_test.dart index 2aff62d3a83..526202f7984 100644 --- a/packages/flutter/test/animation/animation_controller_test.dart +++ b/packages/flutter/test/animation/animation_controller_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'dart:ui' as ui; import 'package:flutter/foundation.dart'; diff --git a/packages/flutter/test/cupertino/action_sheet_test.dart b/packages/flutter/test/cupertino/action_sheet_test.dart index 1f46897bc6e..a664705aa73 100644 --- a/packages/flutter/test/cupertino/action_sheet_test.dart +++ b/packages/flutter/test/cupertino/action_sheet_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/cupertino.dart'; import 'package:flutter/material.dart'; diff --git a/packages/flutter/test/cupertino/date_picker_test.dart b/packages/flutter/test/cupertino/date_picker_test.dart index 6180e15ff26..74020f97be3 100644 --- a/packages/flutter/test/cupertino/date_picker_test.dart +++ b/packages/flutter/test/cupertino/date_picker_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'dart:ui'; import 'package:flutter/cupertino.dart'; diff --git a/packages/flutter/test/cupertino/nav_bar_transition_test.dart b/packages/flutter/test/cupertino/nav_bar_transition_test.dart index d9a4ba8d963..9a4914118ef 100644 --- a/packages/flutter/test/cupertino/nav_bar_transition_test.dart +++ b/packages/flutter/test/cupertino/nav_bar_transition_test.dart @@ -3,6 +3,12 @@ // found in the LICENSE file. @TestOn('!chrome') +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=456" +@Tags(['no-shuffle']) + import 'package:flutter/cupertino.dart'; import 'package:flutter/rendering.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/packages/flutter/test/cupertino/text_field_test.dart b/packages/flutter/test/cupertino/text_field_test.dart index feccf73253b..f451e64d56a 100644 --- a/packages/flutter/test/cupertino/text_field_test.dart +++ b/packages/flutter/test/cupertino/text_field_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'dart:ui' as ui show BoxHeightStyle, BoxWidthStyle, Color; import 'package:flutter/cupertino.dart'; diff --git a/packages/flutter/test/foundation/service_extensions_test.dart b/packages/flutter/test/foundation/service_extensions_test.dart index e9f5e6c35fa..c9d62767d6a 100644 --- a/packages/flutter/test/foundation/service_extensions_test.dart +++ b/packages/flutter/test/foundation/service_extensions_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'dart:async'; import 'dart:convert'; import 'dart:ui' as ui; diff --git a/packages/flutter/test/gestures/debug_test.dart b/packages/flutter/test/gestures/debug_test.dart index b2b24bc04d4..c15c43398e7 100644 --- a/packages/flutter/test/gestures/debug_test.dart +++ b/packages/flutter/test/gestures/debug_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/foundation.dart'; import 'package:flutter/gestures.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/packages/flutter/test/gestures/long_press_test.dart b/packages/flutter/test/gestures/long_press_test.dart index a07de9471f1..4f962d8afa8 100644 --- a/packages/flutter/test/gestures/long_press_test.dart +++ b/packages/flutter/test/gestures/long_press_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/gestures.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/packages/flutter/test/gestures/tap_test.dart b/packages/flutter/test/gestures/tap_test.dart index 2266b5478db..5ef55355def 100644 --- a/packages/flutter/test/gestures/tap_test.dart +++ b/packages/flutter/test/gestures/tap_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=1000" +@Tags(['no-shuffle']) + import 'package:flutter/foundation.dart'; import 'package:flutter/gestures.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/packages/flutter/test/material/app_test.dart b/packages/flutter/test/material/app_test.dart index 55bd0fce88a..fbcc02f10a1 100644 --- a/packages/flutter/test/material/app_test.dart +++ b/packages/flutter/test/material/app_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/cupertino.dart'; import 'package:flutter/foundation.dart'; import 'package:flutter/material.dart'; diff --git a/packages/flutter/test/material/dropdown_test.dart b/packages/flutter/test/material/dropdown_test.dart index 45a6254cae7..065c2c14fd6 100644 --- a/packages/flutter/test/material/dropdown_test.dart +++ b/packages/flutter/test/material/dropdown_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=1408669812" +@Tags(['no-shuffle']) + import 'dart:math' as math; import 'dart:ui' show window; diff --git a/packages/flutter/test/material/paginated_data_table_test.dart b/packages/flutter/test/material/paginated_data_table_test.dart index d1b00649dee..9dc12e3b135 100644 --- a/packages/flutter/test/material/paginated_data_table_test.dart +++ b/packages/flutter/test/material/paginated_data_table_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=1000" +@Tags(['no-shuffle']) + import 'package:flutter/gestures.dart' show DragStartBehavior; import 'package:flutter/material.dart'; import 'package:flutter/rendering.dart'; diff --git a/packages/flutter/test/material/progress_indicator_test.dart b/packages/flutter/test/material/progress_indicator_test.dart index 9f7a8ccea23..8934b16d421 100644 --- a/packages/flutter/test/material/progress_indicator_test.dart +++ b/packages/flutter/test/material/progress_indicator_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=456" +@Tags(['no-shuffle']) + import 'package:flutter/cupertino.dart'; import 'package:flutter/material.dart'; import 'package:flutter/rendering.dart'; diff --git a/packages/flutter/test/material/scaffold_test.dart b/packages/flutter/test/material/scaffold_test.dart index 8f4a55fcd5e..47464a09555 100644 --- a/packages/flutter/test/material/scaffold_test.dart +++ b/packages/flutter/test/material/scaffold_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/foundation.dart'; import 'package:flutter/gestures.dart' show DragStartBehavior; import 'package:flutter/material.dart'; diff --git a/packages/flutter/test/material/scrollbar_test.dart b/packages/flutter/test/material/scrollbar_test.dart index 97b7facd969..4ae04ec6a45 100644 --- a/packages/flutter/test/material/scrollbar_test.dart +++ b/packages/flutter/test/material/scrollbar_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=382757700" +@Tags(['no-shuffle']) + import 'dart:ui' as ui; import 'package:flutter/cupertino.dart'; diff --git a/packages/flutter/test/material/text_field_test.dart b/packages/flutter/test/material/text_field_test.dart index c5f86ec41f1..17c77ea5eab 100644 --- a/packages/flutter/test/material/text_field_test.dart +++ b/packages/flutter/test/material/text_field_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=3890307731" +@Tags(['no-shuffle']) + import 'dart:math' as math; import 'dart:ui' as ui show window, BoxHeightStyle, BoxWidthStyle; diff --git a/packages/flutter/test/painting/image_stream_test.dart b/packages/flutter/test/painting/image_stream_test.dart index 274fd32a001..7356ef66e90 100644 --- a/packages/flutter/test/painting/image_stream_test.dart +++ b/packages/flutter/test/painting/image_stream_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=456" +@Tags(['no-shuffle']) + import 'dart:async'; import 'dart:typed_data'; import 'dart:ui'; diff --git a/packages/flutter/test/physics/newton_test.dart b/packages/flutter/test/physics/newton_test.dart index 34556213e46..c738b3188f8 100644 --- a/packages/flutter/test/physics/newton_test.dart +++ b/packages/flutter/test/physics/newton_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/foundation.dart'; import 'package:flutter/physics.dart'; import 'package:flutter/widgets.dart'; diff --git a/packages/flutter/test/rendering/editable_test.dart b/packages/flutter/test/rendering/editable_test.dart index a5a4b426d7f..9ab218d6f18 100644 --- a/packages/flutter/test/rendering/editable_test.dart +++ b/packages/flutter/test/rendering/editable_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=20210704" +@Tags(['no-shuffle']) + import 'dart:io' show Platform; import 'package:flutter/foundation.dart'; diff --git a/packages/flutter/test/rendering/viewport_test.dart b/packages/flutter/test/rendering/viewport_test.dart index 14a1b2c5be6..9a19ae6c37e 100644 --- a/packages/flutter/test/rendering/viewport_test.dart +++ b/packages/flutter/test/rendering/viewport_test.dart @@ -7,6 +7,12 @@ // initialize a binding, which rendering_tester will attempt to re-initialize // (or vice versa). +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/foundation.dart'; import 'package:flutter/rendering.dart'; import 'package:flutter/widgets.dart'; diff --git a/packages/flutter/test/scheduler/scheduler_test.dart b/packages/flutter/test/scheduler/scheduler_test.dart index a5913cfa98c..9687eb590d3 100644 --- a/packages/flutter/test/scheduler/scheduler_test.dart +++ b/packages/flutter/test/scheduler/scheduler_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'dart:async'; import 'dart:ui' show window; diff --git a/packages/flutter/test/scheduler/ticker_test.dart b/packages/flutter/test/scheduler/ticker_test.dart index 6c8dedc5bca..9e2dd3b358b 100644 --- a/packages/flutter/test/scheduler/ticker_test.dart +++ b/packages/flutter/test/scheduler/ticker_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=4281596210" +@Tags(['no-shuffle']) + import 'package:flutter/foundation.dart'; import 'package:flutter/scheduler.dart'; import 'package:flutter/services.dart'; diff --git a/packages/flutter/test/services/platform_channel_test.dart b/packages/flutter/test/services/platform_channel_test.dart index a97fa9bb1ff..e0ddfc89d79 100644 --- a/packages/flutter/test/services/platform_channel_test.dart +++ b/packages/flutter/test/services/platform_channel_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=456" +@Tags(['no-shuffle']) + import 'package:flutter/services.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/packages/flutter/test/widgets/app_overrides_test.dart b/packages/flutter/test/widgets/app_overrides_test.dart index 9599ca25165..f8c2675660f 100644 --- a/packages/flutter/test/widgets/app_overrides_test.dart +++ b/packages/flutter/test/widgets/app_overrides_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/widgets.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/packages/flutter/test/widgets/binding_test.dart b/packages/flutter/test/widgets/binding_test.dart index db64af8249b..705768f533f 100644 --- a/packages/flutter/test/widgets/binding_test.dart +++ b/packages/flutter/test/widgets/binding_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=382757700" +@Tags(['no-shuffle']) + import 'package:flutter/foundation.dart'; import 'package:flutter/services.dart'; import 'package:flutter/widgets.dart'; diff --git a/packages/flutter/test/widgets/clip_test.dart b/packages/flutter/test/widgets/clip_test.dart index 0d1f306d151..79e5fab035d 100644 --- a/packages/flutter/test/widgets/clip_test.dart +++ b/packages/flutter/test/widgets/clip_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=456" +@Tags(['no-shuffle']) + import 'package:flutter/material.dart'; import 'package:flutter/rendering.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/packages/flutter/test/widgets/debug_test.dart b/packages/flutter/test/widgets/debug_test.dart index 888b0799f0c..bf1a129741b 100644 --- a/packages/flutter/test/widgets/debug_test.dart +++ b/packages/flutter/test/widgets/debug_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/foundation.dart'; import 'package:flutter/widgets.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/packages/flutter/test/widgets/dismissible_test.dart b/packages/flutter/test/widgets/dismissible_test.dart index e8b55824491..1cc7165b09d 100644 --- a/packages/flutter/test/widgets/dismissible_test.dart +++ b/packages/flutter/test/widgets/dismissible_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=456" +@Tags(['no-shuffle']) + import 'package:flutter/gestures.dart' show DragStartBehavior; import 'package:flutter/widgets.dart'; import 'package:flutter_test/flutter_test.dart'; diff --git a/packages/flutter/test/widgets/draggable_test.dart b/packages/flutter/test/widgets/draggable_test.dart index ef50d705c23..b52a5797af0 100644 --- a/packages/flutter/test/widgets/draggable_test.dart +++ b/packages/flutter/test/widgets/draggable_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'package:flutter/gestures.dart'; import 'package:flutter/material.dart'; import 'package:flutter/semantics.dart'; diff --git a/packages/flutter/test/widgets/platform_view_test.dart b/packages/flutter/test/widgets/platform_view_test.dart index d3cfbce40df..c645602571b 100644 --- a/packages/flutter/test/widgets/platform_view_test.dart +++ b/packages/flutter/test/widgets/platform_view_test.dart @@ -3,6 +3,12 @@ // found in the LICENSE file. @TestOn('!chrome') +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=4281596210" +@Tags(['no-shuffle']) + import 'dart:async'; import 'dart:typed_data'; import 'dart:ui'; diff --git a/packages/flutter/test/widgets/routes_test.dart b/packages/flutter/test/widgets/routes_test.dart index 6397fcf84bb..c07dad3227a 100644 --- a/packages/flutter/test/widgets/routes_test.dart +++ b/packages/flutter/test/widgets/routes_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'dart:collection'; import 'package:flutter/material.dart'; diff --git a/packages/flutter/test/widgets/scroll_aware_image_provider_test.dart b/packages/flutter/test/widgets/scroll_aware_image_provider_test.dart index 954d1e84b41..a5d66d2a383 100644 --- a/packages/flutter/test/widgets/scroll_aware_image_provider_test.dart +++ b/packages/flutter/test/widgets/scroll_aware_image_provider_test.dart @@ -2,6 +2,12 @@ // Use of this source code is governed by a BSD-style license that can be // found in the LICENSE file. +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=123" +@Tags(['no-shuffle']) + import 'dart:ui' as ui show Image; import 'package:flutter/widgets.dart'; diff --git a/packages/flutter/test/widgets/widget_inspector_test.dart b/packages/flutter/test/widgets/widget_inspector_test.dart index bebbc5749c6..229aa7cf612 100644 --- a/packages/flutter/test/widgets/widget_inspector_test.dart +++ b/packages/flutter/test/widgets/widget_inspector_test.dart @@ -3,6 +3,13 @@ // found in the LICENSE file. @TestOn('!chrome') + +// TODO(gspencergoog): Remove this tag once this test's state leaks/test +// dependency have been fixed. +// https://github.com/flutter/flutter/issues/85160 +// Fails with "flutter test --test-randomize-ordering-seed=456" +@Tags(['no-shuffle']) + import 'dart:async'; import 'dart:convert'; import 'dart:math'; @@ -1983,7 +1990,7 @@ class _TestWidgetInspectorService extends TestWidgetInspectorService { _CreationLocation location = knownLocations[id]!; expect(location.file, equals(file)); // ClockText widget. - expect(location.line, equals(53)); + expect(location.line, equals(60)); expect(location.column, equals(9)); expect(location.name, equals('ClockText')); expect(count, equals(1)); @@ -1993,7 +2000,7 @@ class _TestWidgetInspectorService extends TestWidgetInspectorService { location = knownLocations[id]!; expect(location.file, equals(file)); // Text widget in _ClockTextState build method. - expect(location.line, equals(91)); + expect(location.line, equals(98)); expect(location.column, equals(12)); expect(location.name, equals('Text')); expect(count, equals(1)); @@ -2020,7 +2027,7 @@ class _TestWidgetInspectorService extends TestWidgetInspectorService { location = knownLocations[id]!; expect(location.file, equals(file)); // ClockText widget. - expect(location.line, equals(53)); + expect(location.line, equals(60)); expect(location.column, equals(9)); expect(location.name, equals('ClockText')); expect(count, equals(3)); // 3 clock widget instances rebuilt. @@ -2030,7 +2037,7 @@ class _TestWidgetInspectorService extends TestWidgetInspectorService { location = knownLocations[id]!; expect(location.file, equals(file)); // Text widget in _ClockTextState build method. - expect(location.line, equals(91)); + expect(location.line, equals(98)); expect(location.column, equals(12)); expect(location.name, equals('Text')); expect(count, equals(3)); // 3 clock widget instances rebuilt.