diff --git a/.cirrus.yml b/.cirrus.yml index 85c1c5d0942..16e84e60f2d 100644 --- a/.cirrus.yml +++ b/.cirrus.yml @@ -236,7 +236,17 @@ task: - name: web_tests-7_last-linux # last Web shard must end with _last << : *WEB_SHARD_TEMPLATE - - name: build_tests-linux + - name: build_tests-0-linux + environment: + # With 1 CPU and 4G of RAM, as of October 2019, build_tests-linux would get OOM-killed. + # Increasing the RAM to 12G allowed it to finish in about 30 minutes, any extra CPU (tried 2 + # and 4) reduced that to just over 20 minutes. 6G was enough not to get OOM-killed. + CPU: 2 + MEMORY: 6G + script: + - dart --enable-asserts ./dev/bots/test.dart + + - name: build_tests-1_last-linux environment: # With 1 CPU and 4G of RAM, as of October 2019, build_tests-linux would get OOM-killed. # Increasing the RAM to 12G allowed it to finish in about 30 minutes, any extra CPU (tried 2 @@ -289,7 +299,7 @@ task: - name: firebase_test_lab_tests-1-linux <<: *FIREBASE_SHARD_TEMPLATE - - name: firebase_test_lab_tests-2-linux + - name: firebase_test_lab_tests-2_last-linux <<: *FIREBASE_SHARD_TEMPLATE - name: web_smoke_test @@ -436,7 +446,18 @@ task: # TODO(ianh): Enable Web tests on Windows - - name: build_tests-windows + - name: build_tests-0-windows + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41941 + environment: + # As of December 2019, the build_tests-windows shard requires 6 GB RAM to pass. + # Emperically, using 6 CPUs and 10 GB RAM yielded optimal results (~33 minutes); + # bumping beyond these limits yielded no extra gain. + CPU: 6 + MEMORY: 10G + script: + - dart --enable-asserts dev\bots\test.dart + + - name: build_tests-1_last-windows only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41941 environment: # As of December 2019, the build_tests-windows shard requires 6 GB RAM to pass. @@ -577,7 +598,12 @@ task: # TODO(ianh): Enable Web tests on macOS. - - name: build_tests-macos + - name: build_tests-0-macos + only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41940 + script: + - dart --enable-asserts ./dev/bots/test.dart + + - name: build_tests-1_last-macos only_if: "changesInclude('.cirrus.yml', 'dev/**', 'bin/internal/**') || $CIRRUS_PR == ''" # https://github.com/flutter/flutter/issues/41940 script: - dart --enable-asserts ./dev/bots/test.dart diff --git a/dev/bots/test.dart b/dev/bots/test.dart index 5fc9d17f19d..7ec56902225 100644 --- a/dev/bots/test.dart +++ b/dev/bots/test.dart @@ -61,6 +61,12 @@ final bool useFlutterTestFormatter = Platform.environment['FLUTTER_TEST_FORMATTE /// and make sure it runs _all_ shards. const int kDeviceLabShardCount = 4; +/// The number of Cirrus jobs that run build tests in parallel. +/// +/// WARNING: if you change this number, also change .cirrus.yml +/// and make sure it runs _all_ shards. +const int kBuildTestShardCount = 2; + /// The number of Cirrus jobs that run Web tests in parallel. /// /// The default is 8 shards. Typically .cirrus.yml would define the @@ -336,45 +342,55 @@ Future _runBuildTests() async { final List exampleDirectories = Directory(path.join(flutterRoot, 'examples')).listSync() ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'non_nullable'))) ..add(Directory(path.join(flutterRoot, 'dev', 'integration_tests', 'flutter_gallery'))); - for (final FileSystemEntity fileEntity in exampleDirectories) { - // Only verify caching with flutter gallery. - final bool verifyCaching = fileEntity.path.contains('flutter_gallery'); - if (fileEntity is! Directory) { - continue; - } - final String examplePath = fileEntity.path; - final bool hasNullSafety = File(path.join(examplePath, 'null_safety')).existsSync(); - final List additionalArgs = hasNullSafety - ? ['--enable-experiment', 'non-nullable'] - : []; - if (Directory(path.join(examplePath, 'android')).existsSync()) { - await _flutterBuildApk(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching); - await _flutterBuildApk(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching); - } else { - print('Example project ${path.basename(examplePath)} has no android directory, skipping apk'); - } - if (Platform.isMacOS) { - if (Directory(path.join(examplePath, 'ios')).existsSync()) { - await _flutterBuildIpa(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching); - await _flutterBuildIpa(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching); - } else { - print('Example project ${path.basename(examplePath)} has no ios directory, skipping ipa'); - } - } - } final String branch = Platform.environment['CIRRUS_BRANCH']; - if (branch != 'beta' && branch != 'stable') { - // Web compilation tests. - await _flutterBuildDart2js( - path.join('dev', 'integration_tests', 'web'), - path.join('lib', 'main.dart'), - ); - // Should not fail to compile with dart:io. - await _flutterBuildDart2js( - path.join('dev', 'integration_tests', 'web_compile_tests'), - path.join('lib', 'dart_io_import.dart'), - ); + // The tests are randomly distributed into subshards so as to get a uniform + // distribution of costs, but the seed is fixed so that issues are reproducible. + final List tests = [ + for (final FileSystemEntity exampleDirectory in exampleDirectories) + () => _runExampleProjectBuildTests(exampleDirectory), + if (branch != 'beta' && branch != 'stable') + ...[ + // Web compilation tests. + () => _flutterBuildDart2js( + path.join('dev', 'integration_tests', 'web'), + path.join('lib', 'main.dart'), + ), + // Should not fail to compile with dart:io. + () => _flutterBuildDart2js( + path.join('dev', 'integration_tests', 'web_compile_tests'), + path.join('lib', 'dart_io_import.dart'), + ), + ], + ]..shuffle(math.Random(0)); + + await _selectIndexedSubshard(tests, kBuildTestShardCount); +} + +Future _runExampleProjectBuildTests(FileSystemEntity exampleDirectory) async { + // Only verify caching with flutter gallery. + final bool verifyCaching = exampleDirectory.path.contains('flutter_gallery'); + if (exampleDirectory is! Directory) { + return; + } + final String examplePath = exampleDirectory.path; + final bool hasNullSafety = File(path.join(examplePath, 'null_safety')).existsSync(); + final List additionalArgs = hasNullSafety + ? ['--enable-experiment', 'non-nullable'] + : []; + if (Directory(path.join(examplePath, 'android')).existsSync()) { + await _flutterBuildApk(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching); + await _flutterBuildApk(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching); + } else { + print('Example project ${path.basename(examplePath)} has no android directory, skipping apk'); + } + if (Platform.isMacOS) { + if (Directory(path.join(examplePath, 'ios')).existsSync()) { + await _flutterBuildIpa(examplePath, release: false, additionalArgs: additionalArgs, verifyCaching: verifyCaching); + await _flutterBuildIpa(examplePath, release: true, additionalArgs: additionalArgs, verifyCaching: verifyCaching); + } else { + print('Example project ${path.basename(examplePath)} has no ios directory, skipping ipa'); + } } } @@ -646,21 +662,15 @@ Future _runFirebaseTestLabTests() async { 'abstract_method_smoke_test', 'android_embedding_v2_smoke_test', ]; - final Map subshards = {}; final String firebaseScript = path.join(flutterRoot, 'dev', 'bots', 'firebase_testlab.sh'); final String integrationTestDirectory = path.join(flutterRoot, 'dev', 'integration_tests'); - for (int index = 0; index < integrationTests.length; index += 1) { - final String integrationTestPath = path.join(integrationTestDirectory, integrationTests[index]); - subshards['$index'] = () => runCommand( - firebaseScript, - [ integrationTestPath ], - workingDirectory: flutterRoot, - ); - } + final List tests = integrationTests.map((String integrationTest) => + () => runCommand(firebaseScript, [ path.join(integrationTestDirectory, integrationTest) ]) + ).toList(); - await selectSubshard(subshards); + await _selectIndexedSubshard(tests, integrationTests.length); } Future _runFrameworkCoverage() async { @@ -1169,27 +1179,7 @@ Future _runHostOnlyDeviceLabTests() async { if (Platform.isLinux) () => _runDevicelabTest('web_benchmarks_canvaskit', environment: kChromeVariables), ]..shuffle(math.Random(0)); - final int testsPerShard = tests.length ~/ kDeviceLabShardCount; - final Map subshards = {}; - - for (int subshard = 0; subshard < kDeviceLabShardCount; subshard += 1) { - String last = ''; - List sublist; - if (subshard < kDeviceLabShardCount - 1) { - sublist = tests.sublist(subshard * testsPerShard, (subshard + 1) * testsPerShard); - } else { - sublist = tests.sublist(subshard * testsPerShard, tests.length); - // We make sure the last shard ends in _last so it's easier to catch mismatches - // between `.cirrus.yml` and `test.dart`. - last = '_last'; - } - subshards['$subshard$last'] = () async { - for (final ShardRunner test in sublist) - await test(); - }; - } - - await selectSubshard(subshards); + await _selectIndexedSubshard(tests, kDeviceLabShardCount); } Future _runDevicelabTest(String testName, { @@ -1366,6 +1356,37 @@ Future verifyVersion(File file) async { return null; } +/// Parse (zero-)index-named subshards and equally distribute [tests] +/// between them. Last shard should end in "_last" to catch mismatches +/// between `.cirrus.yml` and `test.dart`. See [selectShard] for naming details. +/// +/// Examples: +/// build_tests-0-linux +/// build_tests-1-linux +/// build_tests-2_last-linux +Future _selectIndexedSubshard(List tests, int numberOfShards) async { + final int testsPerShard = tests.length ~/ numberOfShards; + final Map subshards = {}; + + for (int subshard = 0; subshard < numberOfShards; subshard += 1) { + String last = ''; + List sublist; + if (subshard < numberOfShards - 1) { + sublist = tests.sublist(subshard * testsPerShard, (subshard + 1) * testsPerShard); + } else { + sublist = tests.sublist(subshard * testsPerShard, tests.length); + // We make sure the last shard ends in _last. + last = '_last'; + } + subshards['$subshard$last'] = () async { + for (final ShardRunner test in sublist) + await test(); + }; + } + + await selectSubshard(subshards); +} + /// If the CIRRUS_TASK_NAME environment variable exists, we use that to determine /// the shard and sub-shard (parsing it in the form shard-subshard-platform, ignoring /// the platform).