From a0fc3f3150b9bd011d4c59eb664ef7968b718482 Mon Sep 17 00:00:00 2001 From: Dan Field Date: Thu, 20 Jun 2019 14:35:33 -0700 Subject: [PATCH] Shard framework tests (#34812) --- .cirrus.yml | 53 ++++++++++++++++++-- dev/bots/test.dart | 122 ++++++++++++++++++++++++++++++++++----------- 2 files changed, 143 insertions(+), 32 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 9d9805a8e56..405a09dd574 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -58,10 +58,31 @@ task: - name: analyze test_script: - dart --enable-asserts ./dev/bots/analyze.dart - - name: tests-linux + - name: tests_widgets-linux env: GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] SHARD: tests + SUBSHARD: widgets + test_script: + - dart --enable-asserts ./dev/bots/test.dart + container: + cpu: 4 + memory: 12G + - name: tests_framework_other-linux + env: + GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] + SHARD: tests + SUBSHARD: framework_other + test_script: + - dart --enable-asserts ./dev/bots/test.dart + container: + cpu: 4 + memory: 12G + - name: tests_extras-linux + env: + GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] + SHARD: tests + SUBSHARD: extras test_script: - dart --enable-asserts ./dev/bots/test.dart container: @@ -152,10 +173,21 @@ task: test_all_script: - bin\cache\dart-sdk\bin\dart.exe --enable-asserts dev\bots\test.dart matrix: - - name: tests-windows + - name: tests_widgets-windows env: GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] SHARD: tests + SUBSHARD: widgets + - name: tests_framework_other-windows + env: + GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] + SHARD: tests + SUBSHARD: framework_other + - name: tests_extras-windows + env: + GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] + SHARD: tests + SUBSHARD: extras - name: tool_tests-windows env: GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] @@ -221,10 +253,21 @@ task: CIRRUS_WORKING_DIR: "/tmp/flutter sdk" COCOAPODS_DISABLE_STATS: true matrix: - - name: tests-macos + - name: tests_widgets-macos env: GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] SHARD: tests + SUBSHARD: widgets + - name: tests_framework_other-macos + env: + GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] + SHARD: tests + SUBSHARD: framework_other + - name: tests_extras-macos + env: + GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] + SHARD: tests + SUBSHARD: extras - name: tool_tests-macos env: GCLOUD_SERVICE_ACCOUNT_KEY: ENCRYPTED[f12abe60f5045d619ef4c79b83dd1e0722a0b0b13dbea95fbe334e2db7fffbcd841a5a92da8824848b539a19afe0c9fb] @@ -279,7 +322,9 @@ docker_builder: depends_on: - docs - analyze - - tests-linux + - tests_widgets-linux + - tests_framework_other-linux + - tests_extras-linux - tool_tests-linux - build_tests-linux - integration_tests-linux diff --git a/dev/bots/test.dart b/dev/bots/test.dart index ff3dbb61a5b..9980f432ea5 100644 --- a/dev/bots/test.dart +++ b/dev/bots/test.dart @@ -307,35 +307,97 @@ Future _runAdd2AppTest() async { Future _runTests() async { final bq.BigqueryApi bigqueryApi = await _getBigqueryApi(); await _runSmokeTests(); + final String subShard = Platform.environment['SUBSHARD']; - await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter'), tableData: bigqueryApi?.tabledata); - // Only packages/flutter/test/widgets/widget_inspector_test.dart really - // needs to be run with --track-widget-creation but it is nice to run - // all of the tests in package:flutter with the flag to ensure that - // the Dart kernel transformer triggered by the flag does not break anything. - await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter'), options: ['--track-widget-creation'], tableData: bigqueryApi?.tabledata); - await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_localizations'), tableData: bigqueryApi?.tabledata); - await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'), tableData: bigqueryApi?.tabledata); - await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test'), tableData: bigqueryApi?.tabledata); - await _runFlutterTest(path.join(flutterRoot, 'packages', 'fuchsia_remote_debug_protocol'), tableData: bigqueryApi?.tabledata); - await _pubRunTest(path.join(flutterRoot, 'dev', 'bots'), tableData: bigqueryApi?.tabledata); - await _pubRunTest(path.join(flutterRoot, 'dev', 'devicelab'), tableData: bigqueryApi?.tabledata); - await _pubRunTest(path.join(flutterRoot, 'dev', 'snippets'), tableData: bigqueryApi?.tabledata); - await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing'), tableData: bigqueryApi?.tabledata); - await _runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests'), tableData: bigqueryApi?.tabledata); - await _runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'vitool'), tableData: bigqueryApi?.tabledata); - await _runFlutterTest(path.join(flutterRoot, 'examples', 'hello_world'), tableData: bigqueryApi?.tabledata); - await _runFlutterTest(path.join(flutterRoot, 'examples', 'layers'), tableData: bigqueryApi?.tabledata); - await _runFlutterTest(path.join(flutterRoot, 'examples', 'stocks'), tableData: bigqueryApi?.tabledata); - await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'), tableData: bigqueryApi?.tabledata); - // Regression test to ensure that code outside of package:flutter can run - // with --track-widget-creation. - await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'), options: ['--track-widget-creation'], tableData: bigqueryApi?.tabledata); - await _runFlutterTest(path.join(flutterRoot, 'examples', 'catalog'), tableData: bigqueryApi?.tabledata); - // Smoke test for code generation. - await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'codegen'), tableData: bigqueryApi?.tabledata, environment: { - 'FLUTTER_EXPERIMENTAL_BUILD': 'true', - }); + Future runWidgets() async { + await _runFlutterTest( + path.join(flutterRoot, 'packages', 'flutter'), + tableData: bigqueryApi?.tabledata, + tests: [ + 'test/widgets/', + ], + ); + // Only packages/flutter/test/widgets/widget_inspector_test.dart really + // needs to be run with --track-widget-creation but it is nice to run + // all of the tests in package:flutter with the flag to ensure that + // the Dart kernel transformer triggered by the flag does not break anything. + await _runFlutterTest( + path.join(flutterRoot, 'packages', 'flutter'), + options: ['--track-widget-creation'], + tableData: bigqueryApi?.tabledata, + tests: [ + 'test/widgets/', + ], + ); + } + + Future runFrameworkOthers() async { + final List tests = Directory(path.join(flutterRoot, 'packages', 'flutter', 'test')) + .listSync(followLinks: false, recursive: false) + .whereType() + .map((Directory dir) => 'test/${path.basename(dir.path)}/') + .toList(); + + print('Running tests for: ${tests.join(';')}'); + + await _runFlutterTest( + path.join(flutterRoot, 'packages', 'flutter'), + tableData: bigqueryApi?.tabledata, + tests: tests, + ); + // Only packages/flutter/test/widgets/widget_inspector_test.dart really + // needs to be run with --track-widget-creation but it is nice to run + // all of the tests in package:flutter with the flag to ensure that + // the Dart kernel transformer triggered by the flag does not break anything. + await _runFlutterTest( + path.join(flutterRoot, 'packages', 'flutter'), + options: ['--track-widget-creation'], + tableData: bigqueryApi?.tabledata, + tests: tests, + ); + } + + Future runExtras() async { + await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_localizations'), tableData: bigqueryApi?.tabledata); + await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_driver'), tableData: bigqueryApi?.tabledata); + await _runFlutterTest(path.join(flutterRoot, 'packages', 'flutter_test'), tableData: bigqueryApi?.tabledata); + await _runFlutterTest(path.join(flutterRoot, 'packages', 'fuchsia_remote_debug_protocol'), tableData: bigqueryApi?.tabledata); + await _pubRunTest(path.join(flutterRoot, 'dev', 'bots'), tableData: bigqueryApi?.tabledata); + await _pubRunTest(path.join(flutterRoot, 'dev', 'devicelab'), tableData: bigqueryApi?.tabledata); + await _pubRunTest(path.join(flutterRoot, 'dev', 'snippets'), tableData: bigqueryApi?.tabledata); + await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'android_semantics_testing'), tableData: bigqueryApi?.tabledata); + await _runFlutterTest(path.join(flutterRoot, 'dev', 'manual_tests'), tableData: bigqueryApi?.tabledata); + await _runFlutterTest(path.join(flutterRoot, 'dev', 'tools', 'vitool'), tableData: bigqueryApi?.tabledata); + await _runFlutterTest(path.join(flutterRoot, 'examples', 'hello_world'), tableData: bigqueryApi?.tabledata); + await _runFlutterTest(path.join(flutterRoot, 'examples', 'layers'), tableData: bigqueryApi?.tabledata); + await _runFlutterTest(path.join(flutterRoot, 'examples', 'stocks'), tableData: bigqueryApi?.tabledata); + await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'), tableData: bigqueryApi?.tabledata); + // Regression test to ensure that code outside of package:flutter can run + // with --track-widget-creation. + await _runFlutterTest(path.join(flutterRoot, 'examples', 'flutter_gallery'), options: ['--track-widget-creation'], tableData: bigqueryApi?.tabledata); + await _runFlutterTest(path.join(flutterRoot, 'examples', 'catalog'), tableData: bigqueryApi?.tabledata); + // Smoke test for code generation. + await _runFlutterTest(path.join(flutterRoot, 'dev', 'integration_tests', 'codegen'), tableData: bigqueryApi?.tabledata, environment: { + 'FLUTTER_EXPERIMENTAL_BUILD': 'true', + }); + } + switch (subShard) { + case 'widgets': + await runWidgets(); + break; + case 'framework_other': + await runFrameworkOthers(); + break; + case 'extras': + runExtras(); + break; + default: + print('Unknown sub-shard $subShard, running all tests!'); + await runWidgets(); + await runFrameworkOthers(); + await runExtras(); + + } print('${bold}DONE: All tests successful.$reset'); } @@ -649,6 +711,7 @@ Future _runFlutterTest(String workingDirectory, { Duration timeout = _kLongTimeout, bq.TabledataResourceApi tableData, Map environment, + List tests = const [], }) async { final List args = ['test']..addAll(options); if (flutterTestArgs != null && flutterTestArgs.isNotEmpty) @@ -673,6 +736,9 @@ Future _runFlutterTest(String workingDirectory, { } args.add(script); } + + args.addAll(tests); + if (!shouldProcessOutput) { return runCommand(flutter, args, workingDirectory: workingDirectory,