From fd0e223b74f983ef04a681e99df3f8fd95de3252 Mon Sep 17 00:00:00 2001 From: godofredoc <54371434+godofredoc@users.noreply.github.com> Date: Fri, 25 Sep 2020 14:26:39 -0700 Subject: [PATCH] Add Firebase tests to flutter dashboard. (#66677) This PR also enables firebase tests on presubmit and removes code that won't be used anymore from test runner. Bug: https://github.com/flutter/infra/pull/225 --- .cirrus.yml | 21 ------------ dev/bots/test.dart | 74 ------------------------------------------ dev/prod_builders.json | 18 ++++++++++ dev/try_builders.json | 18 ++++++++++ 4 files changed, 36 insertions(+), 95 deletions(-) diff --git a/.cirrus.yml b/.cirrus.yml index 3287af81518..e16f40ebd8e 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -4,17 +4,6 @@ # YAML anchors used to share fields between tasks. # See https://confluence.atlassian.com/bitbucket/yaml-anchors-960154027.html -firebase_shard_template: &FIREBASE_SHARD_TEMPLATE - environment: - # Empirically, this shard runs in 20-25 minutes with just one CPU and 4G of RAM, as of - # October 2019. It does not seem to be sensitive to the number of CPUs or amount of RAM; - # doubling CPUs had no effect (mere seconds under 20 minutes), increasing RAM to 24G left it - # on the high end of the 20-25 minute range. (This makes sense, as it's just driving the - # Firebase test lab remotely.) Less than 4G of RAM made it go OOM. - CLOUDSDK_CORE_DISABLE_PROMPTS: 1 - GCLOUD_FIREBASE_TESTLAB_KEY: ENCRYPTED[1c140257edc48f5578fa5a0e5038b84c8e53270c405efa5a8e35ea303a4e0d135853989f448f72136206de854d17fbec] - script: - - dart --enable-asserts ./dev/bots/test.dart use_compute_credits: $CIRRUS_USER_COLLABORATOR == 'true' @@ -189,16 +178,6 @@ task: - git clone https://github.com/flutter/tests.git bin/cache/pkg/tests - dart --enable-asserts dev/customer_testing/run_tests.dart --skip-on-fetch-failure --skip-template bin/cache/pkg/tests/registry/*.test - # firebase_test_lab_tests are linux-only - - name: firebase_test_lab_tests-0-linux - <<: *FIREBASE_SHARD_TEMPLATE - - - name: firebase_test_lab_tests-1-linux - <<: *FIREBASE_SHARD_TEMPLATE - - - name: firebase_test_lab_tests-2_last-linux - <<: *FIREBASE_SHARD_TEMPLATE - - name: web_smoke_test only_if: "changesInclude('.cirrus.yml', 'examples/hello_world/**' ,'dev/**', 'packages/flutter/**', 'packages/flutter_test/**', 'packages/flutter_tools/lib/src/test/**', 'packages/flutter_web_plugins/**', 'bin/**') && $CIRRUS_PR != ''" environment: diff --git a/dev/bots/test.dart b/dev/bots/test.dart index 873217cd742..d534edfec79 100644 --- a/dev/bots/test.dart +++ b/dev/bots/test.dart @@ -55,11 +55,6 @@ final List flutterTestArgs = []; final bool useFlutterTestFormatter = Platform.environment['FLUTTER_TEST_FORMATTER'] == 'true'; -/// The number of Cirrus jobs that run host-only devicelab tests in parallel. -/// -/// WARNING: if you change this number, also change .cirrus.yml -/// and make sure it runs _all_ shards. -const int kDeviceLabShardCount = 2; /// The number of Cirrus jobs that run build tests in parallel. /// @@ -121,10 +116,8 @@ Future main(List args) async { await selectShard(const { 'add_to_app_life_cycle_tests': _runAddToAppLifeCycleTests, 'build_tests': _runBuildTests, - 'firebase_test_lab_tests': _runFirebaseTestLabTests, 'framework_coverage': _runFrameworkCoverage, 'framework_tests': _runFrameworkTests, - 'hostonly_devicelab_tests': _runHostOnlyDeviceLabTests, 'tool_coverage': _runToolCoverage, 'tool_tests': _runToolTests, 'web_tests': _runWebUnitTests, @@ -669,28 +662,6 @@ Future _runFrameworkTests() async { }); } -Future _runFirebaseTestLabTests() async { - // Firebase Lab tests take ~20 minutes per integration test, - // so only one test is run per shard. Therefore, there are as - // many shards available as there are integration tests in this list. - // If you add a new test, add a corresponding firebase_test_lab-#-linux - // to .cirrus.yml - final List integrationTests = [ - 'release_smoke_test', - 'abstract_method_smoke_test', - 'android_embedding_v2_smoke_test', - ]; - - final String firebaseScript = path.join(flutterRoot, 'dev', 'bots', 'firebase_testlab.sh'); - final String integrationTestDirectory = path.join(flutterRoot, 'dev', 'integration_tests'); - - final List tests = integrationTests.map((String integrationTest) => - () => runCommand(firebaseScript, [ path.join(integrationTestDirectory, integrationTest) ]) - ).toList(); - - await _selectIndexedSubshard(tests, integrationTests.length); -} - Future _runFrameworkCoverage() async { final File coverageFile = File(path.join(flutterRoot, 'packages', 'flutter', 'coverage', 'lcov.info')); if (!coverageFile.existsSync()) { @@ -1215,51 +1186,6 @@ Map _initGradleEnvironment() { final Map gradleEnvironment = _initGradleEnvironment(); -Future _runHostOnlyDeviceLabTests() async { - // Please don't add more tests here. We should not be using the devicelab - // logic to run tests outside devicelab, that's just confusing. - // Instead, create tests that are not devicelab tests, and run those. - - // TODO(ianh): Move the tests that are not running on devicelab any more out - // of the device lab directory. - - // List the tests to run. - // We split these into subshards. The tests are randomly distributed into - // those subshards so as to get a uniform distribution of costs, but the - // seed is fixed so that issues are reproducible. - final List tests = [ - // Keep this in alphabetical order. - () => _runDevicelabTest('build_aar_module_test', environment: gradleEnvironment), - () => _runDevicelabTest('gradle_jetifier_test', environment: gradleEnvironment), - () => _runDevicelabTest('gradle_plugin_light_apk_test', environment: gradleEnvironment), - () => _runDevicelabTest('module_custom_host_app_name_test', environment: gradleEnvironment), - () => _runDevicelabTest('module_test', environment: gradleEnvironment, testEmbeddingV2: true), - () => _runDevicelabTest('plugin_dependencies_test', environment: gradleEnvironment), - ]..shuffle(math.Random(0)); - - await _selectIndexedSubshard(tests, kDeviceLabShardCount); -} - -Future _runDevicelabTest(String testName, { - Map environment, - // testEmbeddingV2 is only supported by certain specific devicelab tests. - // Don't use it unless you're sure the test actually supports it. - // You can check by looking to see if the test examines the environment - // for the ENABLE_ANDROID_EMBEDDING_V2 variable. - bool testEmbeddingV2 = false, -}) async { - await runCommand( - dart, - ['bin/run.dart', '-t', testName], - workingDirectory: path.join(flutterRoot, 'dev', 'devicelab'), - environment: { - ...?environment, - if (testEmbeddingV2) - 'ENABLE_ANDROID_EMBEDDING_V2': 'true', - }, - ); -} - void deleteFile(String path) { // This is technically a race condition but nobody else should be running // while this script runs, so we should be ok. (Sadly recursive:true does not diff --git a/dev/prod_builders.json b/dev/prod_builders.json index a500146b710..90e14c3b8fb 100644 --- a/dev/prod_builders.json +++ b/dev/prod_builders.json @@ -30,6 +30,24 @@ "task_name": "linux_build_tests", "flaky": false }, + { + "name": "Linux firebase_abstract_method_smoke_test", + "repo": "flutter", + "task_name": "linux_firebase_abstract_method_smoke_test", + "flaky": false + }, + { + "name": "Linux firebase_android_embedding_v2_smoke_test", + "repo": "flutter", + "task_name": "linux_firebase_android_embedding_v2_smoke_test", + "flaky": false + }, + { + "name": "Linux firebase_release_smoke_test", + "repo": "flutter", + "task_name": "linux_firebase_release_smoke_test", + "flaky": false + }, { "name": "Linux framework_tests", "repo": "flutter", diff --git a/dev/try_builders.json b/dev/try_builders.json index 3f47ae9153d..a6d0ffe7f61 100644 --- a/dev/try_builders.json +++ b/dev/try_builders.json @@ -39,6 +39,24 @@ "enabled":true, "run_if":["dev/", "packages/flutter/", "packages/flutter_test/", "packages/flutter_goldens/", "packages/flutter_tools/lib/src/test/", "bin/"] }, + { + "name":"Linux firebase_abstract_method_smoke_test", + "repo":"flutter", + "task_name":"linux_firebase_abstract_method_smoke_test", + "enabled":true + }, + { + "name":"Linux firebase_android_embedding_v2_smoke_test", + "repo":"flutter", + "task_name":"linux_firebase_android_embedding_v2_smoke_test", + "enabled":true + }, + { + "name":"Linux firebase_release_smoke_test", + "repo":"flutter", + "task_name":"linux_firebase_release_smoke_test", + "enabled":true + }, { "name":"Linux fuchsia_precache", "repo":"flutter",