From d104106e785e591f73db24bf8defdb02ef20af1f Mon Sep 17 00:00:00 2001 From: Ian Hickson Date: Tue, 28 Nov 2017 17:21:01 -0800 Subject: [PATCH] Refactor compilation devicelab tests (#13218) This fills out the following matrix: App: `flutter create` default app, Complex Layout, and Gallery. Platforms: Linux/Android, Mac/iOS, Windows/Android Build modes: full release, aot blobs, debug Dart: normal, 2.0 preview It also renames all the tests that used "build" to mean compilation build to use "compile" instead, so that it's much clearer on the dashboard. Also, mark some flaky tests that are no longer flaky as not flaky. Also, remove uses of call() since that's probably going to go away at some point. --- ... basic_material_app_android__compile.dart} | 4 +- ...t => basic_material_app_ios__compile.dart} | 4 +- .../basic_material_app_win__compile.dart | 14 +++ .../complex_layout_android__compile.dart | 14 +++ ....dart => complex_layout_ios__compile.dart} | 4 +- .../tasks/complex_layout_win__compile.dart | 14 +++ ...flutter_gallery__preview_dart_2_build.dart | 12 -- .../flutter_gallery_android__compile.dart | 14 +++ ...dart => flutter_gallery_ios__compile.dart} | 6 +- .../bin/tasks/flutter_gallery_win__build.dart | 11 -- .../tasks/flutter_gallery_win__compile.dart | 14 +++ dev/devicelab/lib/tasks/perf_tests.dart | 119 ++++++++++++------ dev/devicelab/lib/tasks/size_tests.dart | 58 --------- dev/devicelab/manifest.yaml | 115 ++++++++++------- 14 files changed, 231 insertions(+), 172 deletions(-) rename dev/devicelab/bin/tasks/{basic_material_app__size.dart => basic_material_app_android__compile.dart} (78%) rename dev/devicelab/bin/tasks/{basic_material_app_ios__size.dart => basic_material_app_ios__compile.dart} (78%) create mode 100644 dev/devicelab/bin/tasks/basic_material_app_win__compile.dart create mode 100644 dev/devicelab/bin/tasks/complex_layout_android__compile.dart rename dev/devicelab/bin/tasks/{complex_layout__build.dart => complex_layout_ios__compile.dart} (68%) create mode 100644 dev/devicelab/bin/tasks/complex_layout_win__compile.dart delete mode 100644 dev/devicelab/bin/tasks/flutter_gallery__preview_dart_2_build.dart create mode 100644 dev/devicelab/bin/tasks/flutter_gallery_android__compile.dart rename dev/devicelab/bin/tasks/{flutter_gallery__build.dart => flutter_gallery_ios__compile.dart} (68%) delete mode 100644 dev/devicelab/bin/tasks/flutter_gallery_win__build.dart create mode 100644 dev/devicelab/bin/tasks/flutter_gallery_win__compile.dart delete mode 100644 dev/devicelab/lib/tasks/size_tests.dart diff --git a/dev/devicelab/bin/tasks/basic_material_app__size.dart b/dev/devicelab/bin/tasks/basic_material_app_android__compile.dart similarity index 78% rename from dev/devicelab/bin/tasks/basic_material_app__size.dart rename to dev/devicelab/bin/tasks/basic_material_app_android__compile.dart index c3aa6acae9e..3fe93103a0a 100644 --- a/dev/devicelab/bin/tasks/basic_material_app__size.dart +++ b/dev/devicelab/bin/tasks/basic_material_app_android__compile.dart @@ -6,9 +6,9 @@ import 'dart:async'; import 'package:flutter_devicelab/framework/adb.dart'; import 'package:flutter_devicelab/framework/framework.dart'; -import 'package:flutter_devicelab/tasks/size_tests.dart'; +import 'package:flutter_devicelab/tasks/perf_tests.dart'; Future main() async { deviceOperatingSystem = DeviceOperatingSystem.android; - await task(createBasicMaterialAppSizeTest()); + await task(createBasicMaterialCompileTest()); } diff --git a/dev/devicelab/bin/tasks/basic_material_app_ios__size.dart b/dev/devicelab/bin/tasks/basic_material_app_ios__compile.dart similarity index 78% rename from dev/devicelab/bin/tasks/basic_material_app_ios__size.dart rename to dev/devicelab/bin/tasks/basic_material_app_ios__compile.dart index a84e5baa93e..f56c53dbd77 100644 --- a/dev/devicelab/bin/tasks/basic_material_app_ios__size.dart +++ b/dev/devicelab/bin/tasks/basic_material_app_ios__compile.dart @@ -6,9 +6,9 @@ import 'dart:async'; import 'package:flutter_devicelab/framework/adb.dart'; import 'package:flutter_devicelab/framework/framework.dart'; -import 'package:flutter_devicelab/tasks/size_tests.dart'; +import 'package:flutter_devicelab/tasks/perf_tests.dart'; Future main() async { deviceOperatingSystem = DeviceOperatingSystem.ios; - await task(createBasicMaterialAppSizeTest()); + await task(createBasicMaterialCompileTest()); } diff --git a/dev/devicelab/bin/tasks/basic_material_app_win__compile.dart b/dev/devicelab/bin/tasks/basic_material_app_win__compile.dart new file mode 100644 index 00000000000..3fe93103a0a --- /dev/null +++ b/dev/devicelab/bin/tasks/basic_material_app_win__compile.dart @@ -0,0 +1,14 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/framework/adb.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/tasks/perf_tests.dart'; + +Future main() async { + deviceOperatingSystem = DeviceOperatingSystem.android; + await task(createBasicMaterialCompileTest()); +} diff --git a/dev/devicelab/bin/tasks/complex_layout_android__compile.dart b/dev/devicelab/bin/tasks/complex_layout_android__compile.dart new file mode 100644 index 00000000000..9af9087ce69 --- /dev/null +++ b/dev/devicelab/bin/tasks/complex_layout_android__compile.dart @@ -0,0 +1,14 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/framework/adb.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/tasks/perf_tests.dart'; + +Future main() async { + deviceOperatingSystem = DeviceOperatingSystem.android; + await task(createComplexLayoutCompileTest()); +} diff --git a/dev/devicelab/bin/tasks/complex_layout__build.dart b/dev/devicelab/bin/tasks/complex_layout_ios__compile.dart similarity index 68% rename from dev/devicelab/bin/tasks/complex_layout__build.dart rename to dev/devicelab/bin/tasks/complex_layout_ios__compile.dart index 7fd0f9eb238..de02cd6c5b0 100644 --- a/dev/devicelab/bin/tasks/complex_layout__build.dart +++ b/dev/devicelab/bin/tasks/complex_layout_ios__compile.dart @@ -5,8 +5,10 @@ import 'dart:async'; import 'package:flutter_devicelab/tasks/perf_tests.dart'; +import 'package:flutter_devicelab/framework/adb.dart'; import 'package:flutter_devicelab/framework/framework.dart'; Future main() async { - await task(createComplexLayoutBuildTest()); + deviceOperatingSystem = DeviceOperatingSystem.ios; + await task(createComplexLayoutCompileTest()); } diff --git a/dev/devicelab/bin/tasks/complex_layout_win__compile.dart b/dev/devicelab/bin/tasks/complex_layout_win__compile.dart new file mode 100644 index 00000000000..9af9087ce69 --- /dev/null +++ b/dev/devicelab/bin/tasks/complex_layout_win__compile.dart @@ -0,0 +1,14 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/framework/adb.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/tasks/perf_tests.dart'; + +Future main() async { + deviceOperatingSystem = DeviceOperatingSystem.android; + await task(createComplexLayoutCompileTest()); +} diff --git a/dev/devicelab/bin/tasks/flutter_gallery__preview_dart_2_build.dart b/dev/devicelab/bin/tasks/flutter_gallery__preview_dart_2_build.dart deleted file mode 100644 index fe67faab6bc..00000000000 --- a/dev/devicelab/bin/tasks/flutter_gallery__preview_dart_2_build.dart +++ /dev/null @@ -1,12 +0,0 @@ -// Copyright 2017 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'dart:async'; - -import 'package:flutter_devicelab/tasks/perf_tests.dart'; -import 'package:flutter_devicelab/framework/framework.dart'; - -Future main() async { - await task(createFlutterGalleryPreviewDart2BuildTest()); -} diff --git a/dev/devicelab/bin/tasks/flutter_gallery_android__compile.dart b/dev/devicelab/bin/tasks/flutter_gallery_android__compile.dart new file mode 100644 index 00000000000..01afc148102 --- /dev/null +++ b/dev/devicelab/bin/tasks/flutter_gallery_android__compile.dart @@ -0,0 +1,14 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/framework/adb.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/tasks/perf_tests.dart'; + +Future main() async { + deviceOperatingSystem = DeviceOperatingSystem.android; + await task(createFlutterGalleryCompileTest()); +} diff --git a/dev/devicelab/bin/tasks/flutter_gallery__build.dart b/dev/devicelab/bin/tasks/flutter_gallery_ios__compile.dart similarity index 68% rename from dev/devicelab/bin/tasks/flutter_gallery__build.dart rename to dev/devicelab/bin/tasks/flutter_gallery_ios__compile.dart index 0d743db4ace..19807a6d23c 100644 --- a/dev/devicelab/bin/tasks/flutter_gallery__build.dart +++ b/dev/devicelab/bin/tasks/flutter_gallery_ios__compile.dart @@ -4,9 +4,11 @@ import 'dart:async'; -import 'package:flutter_devicelab/tasks/perf_tests.dart'; +import 'package:flutter_devicelab/framework/adb.dart'; import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/tasks/perf_tests.dart'; Future main() async { - await task(createFlutterGalleryBuildTest()); + deviceOperatingSystem = DeviceOperatingSystem.ios; + await task(createFlutterGalleryCompileTest()); } diff --git a/dev/devicelab/bin/tasks/flutter_gallery_win__build.dart b/dev/devicelab/bin/tasks/flutter_gallery_win__build.dart deleted file mode 100644 index 931475ee537..00000000000 --- a/dev/devicelab/bin/tasks/flutter_gallery_win__build.dart +++ /dev/null @@ -1,11 +0,0 @@ -// Copyright (c) 2017 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'dart:async'; - -import 'flutter_gallery__build.dart' as flutter_gallery_build; - -Future main() async { - await flutter_gallery_build.main(); -} diff --git a/dev/devicelab/bin/tasks/flutter_gallery_win__compile.dart b/dev/devicelab/bin/tasks/flutter_gallery_win__compile.dart new file mode 100644 index 00000000000..01afc148102 --- /dev/null +++ b/dev/devicelab/bin/tasks/flutter_gallery_win__compile.dart @@ -0,0 +1,14 @@ +// Copyright 2016 The Chromium Authors. All rights reserved. +// Use of this source code is governed by a BSD-style license that can be +// found in the LICENSE file. + +import 'dart:async'; + +import 'package:flutter_devicelab/framework/adb.dart'; +import 'package:flutter_devicelab/framework/framework.dart'; +import 'package:flutter_devicelab/tasks/perf_tests.dart'; + +Future main() async { + deviceOperatingSystem = DeviceOperatingSystem.android; + await task(createFlutterGalleryCompileTest()); +} diff --git a/dev/devicelab/lib/tasks/perf_tests.dart b/dev/devicelab/lib/tasks/perf_tests.dart index dad5b7edcb8..8c582050164 100644 --- a/dev/devicelab/lib/tasks/perf_tests.dart +++ b/dev/devicelab/lib/tasks/perf_tests.dart @@ -4,6 +4,7 @@ import 'dart:async'; import 'dart:convert' show JSON; +import 'dart:io'; import '../framework/adb.dart'; import '../framework/framework.dart'; @@ -15,7 +16,7 @@ TaskFunction createComplexLayoutScrollPerfTest() { '${flutterDirectory.path}/dev/benchmarks/complex_layout', 'test_driver/scroll_perf.dart', 'complex_layout_scroll_perf', - ); + ).run; } TaskFunction createComplexLayoutScrollMemoryTest() { @@ -23,38 +24,34 @@ TaskFunction createComplexLayoutScrollMemoryTest() { '${flutterDirectory.path}/dev/benchmarks/complex_layout', 'com.yourcompany.complexLayout', testTarget: 'test_driver/scroll_perf.dart', - ); + ).run; } TaskFunction createFlutterGalleryStartupTest() { return new StartupTest( '${flutterDirectory.path}/examples/flutter_gallery', - ); + ).run; } TaskFunction createComplexLayoutStartupTest() { return new StartupTest( '${flutterDirectory.path}/dev/benchmarks/complex_layout', - ); + ).run; } -TaskFunction createFlutterGalleryBuildTest() { - return new BuildTest('${flutterDirectory.path}/examples/flutter_gallery'); +TaskFunction createFlutterGalleryCompileTest() { + return new CompileTest('${flutterDirectory.path}/examples/flutter_gallery').run; } -TaskFunction createFlutterGalleryPreviewDart2BuildTest() { - return new BuildTest('${flutterDirectory.path}/examples/flutter_gallery', previewDart2: true); -} - -TaskFunction createComplexLayoutBuildTest() { - return new BuildTest('${flutterDirectory.path}/dev/benchmarks/complex_layout'); +TaskFunction createComplexLayoutCompileTest() { + return new CompileTest('${flutterDirectory.path}/dev/benchmarks/complex_layout').run; } TaskFunction createHelloWorldMemoryTest() { return new MemoryTest( '${flutterDirectory.path}/examples/hello_world', 'io.flutter.examples.hello_world', - ); + ).run; } TaskFunction createGalleryNavigationMemoryTest() { @@ -62,7 +59,7 @@ TaskFunction createGalleryNavigationMemoryTest() { '${flutterDirectory.path}/examples/flutter_gallery', 'io.flutter.examples.gallery', testTarget: 'test_driver/memory_nav.dart', - ); + ).run; } TaskFunction createGalleryBackButtonMemoryTest() { @@ -70,16 +67,29 @@ TaskFunction createGalleryBackButtonMemoryTest() { '${flutterDirectory.path}/examples/flutter_gallery', 'io.flutter.examples.gallery', 'io.flutter.examples.gallery.MainActivity', - ); + ).run; } TaskFunction createFlutterViewStartupTest() { return new StartupTest( '${flutterDirectory.path}/examples/flutter_view', reportMetrics: false, - ); + ).run; } +TaskFunction createBasicMaterialCompileTest() { + return () async { + const String sampleAppName = 'sample_flutter_app'; + final Directory sampleDir = dir('${Directory.systemTemp.path}/$sampleAppName'); + + if (await sampleDir.exists()) + rmTree(sampleDir); + + return new CompileTest(sampleDir.path).run(); + }; +} + + /// Measure application startup performance. class StartupTest { static const Duration _startupTimeout = const Duration(minutes: 5); @@ -89,7 +99,7 @@ class StartupTest { final String testDirectory; final bool reportMetrics; - Future call() async { + Future run() async { return await inDirectory(testDirectory, () async { final String deviceId = (await devices.workingDevice).deviceId; await flutter('packages', options: ['get']); @@ -119,14 +129,13 @@ class StartupTest { /// Measures application runtime performance, specifically per-frame /// performance. class PerfTest { - const PerfTest(this.testDirectory, this.testTarget, this.timelineFileName); final String testDirectory; final String testTarget; final String timelineFileName; - Future call() { + Future run() { return inDirectory(testDirectory, () async { final Device device = await devices.workingDevice; await device.unlock(); @@ -166,32 +175,39 @@ class PerfTest { } } -/// Measures how long it takes to build a Flutter app and how big the compiled +/// Measures how long it takes to compile a Flutter app and how big the compiled /// code is. -class BuildTest { - const BuildTest(this.testDirectory, {this.previewDart2: false}); +class CompileTest { + const CompileTest(this.testDirectory); - final bool previewDart2; final String testDirectory; - Future call() async { + Future run() async { return await inDirectory(testDirectory, () async { final Device device = await devices.workingDevice; await device.unlock(); await flutter('packages', options: ['get']); - final Map aotResults = await _buildAot(previewDart2); - final Map debugResults = await _buildDebug(previewDart2); - final Map metrics = {} - ..addAll(aotResults) - ..addAll(debugResults); + ..addAll(await _compileAot()) + ..addAll(await _compileApp()) + ..addAll(await _compileDebug()) + ..addAll(_suffix(await _compileAot(previewDart2: true), '__preview_dart_2')) + ..addAll(_suffix(await _compileApp(previewDart2: true), '__preview_dart_2')) + ..addAll(_suffix(await _compileDebug(previewDart2: true), '__preview_dart_2')); return new TaskResult.success(metrics, benchmarkScoreKeys: metrics.keys.toList()); }); } - static Future> _buildAot(bool previewDart2) async { + static Map _suffix(Map map, String suffix) { + return new Map.fromIterables( + map.keys.map((String key) => '$key$suffix'), + map.values, + ); + } + + static Future> _compileAot({ bool previewDart2: false }) async { await flutter('clean'); final Stopwatch watch = new Stopwatch()..start(); final List options = [ @@ -203,23 +219,52 @@ class BuildTest { ]; if (previewDart2) options.add('--preview-dart-2'); - final String buildLog = await evalFlutter('build', options: options); + final String compileLog = await evalFlutter('build', options: options); watch.stop(); final RegExp metricExpression = new RegExp(r'([a-zA-Z]+)\(CodeSize\)\: (\d+)'); final Map metrics = new Map.fromIterable( - metricExpression.allMatches(buildLog), + metricExpression.allMatches(compileLog), key: (Match m) => _sdkNameToMetricName(m.group(1)), value: (Match m) => int.parse(m.group(2)), ); - metrics['aot_snapshot_build_millis'] = watch.elapsedMilliseconds; + metrics['aot_snapshot_compile_millis'] = watch.elapsedMilliseconds; return metrics; } - static Future> _buildDebug(bool previewDart2) async { + static Future> _compileApp({ bool previewDart2: false }) async { await flutter('clean'); + final Stopwatch watch = new Stopwatch(); + int releaseSizeInBytes; + final List options = ['--release']; + if (previewDart2) + options.add('--preview-dart-2'); + if (deviceOperatingSystem == DeviceOperatingSystem.ios) { + options.add('ios'); + await prepareProvisioningCertificates(cwd); + watch.start(); + await flutter('build', options: options); + watch.stop(); + // IPAs are created manually AFAICT + await exec('tar', ['-zcf', 'build/app.ipa', 'build/ios/Release-iphoneos/Runner.app/']); + releaseSizeInBytes = await file('$cwd/build/app.ipa').length(); + } else { + options.add('apk'); + watch.start(); + await flutter('build', options: options); + watch.stop(); + releaseSizeInBytes = await file('$cwd/build/app/outputs/apk/app-release.apk').length(); + } + return { + 'release_full_compile_millis': watch.elapsedMilliseconds, + 'release_size_bytes': releaseSizeInBytes, + }; + } + + static Future> _compileDebug({ bool previewDart2: false }) async { + await flutter('clean'); final Stopwatch watch = new Stopwatch(); if (deviceOperatingSystem == DeviceOperatingSystem.ios) { await prepareProvisioningCertificates(cwd); @@ -236,7 +281,7 @@ class BuildTest { } return { - 'debug_full_build_millis': watch.elapsedMilliseconds, + 'debug_full_compile_millis': watch.elapsedMilliseconds, }; } @@ -268,7 +313,7 @@ class MemoryTest { /// If not specified, then the test will start the app, gather statistics, and then exit. final String testTarget; - Future call() { + Future run() { return inDirectory(testDirectory, () async { final Device device = await devices.workingDevice; await device.unlock(); @@ -330,7 +375,7 @@ class AndroidBackButtonMemoryTest { final String packageName; final String activityName; - Future call() { + Future run() { return inDirectory(testDirectory, () async { if (deviceOperatingSystem != DeviceOperatingSystem.android) { throw 'This test is only supported on Android'; diff --git a/dev/devicelab/lib/tasks/size_tests.dart b/dev/devicelab/lib/tasks/size_tests.dart deleted file mode 100644 index 8dbe75df8f4..00000000000 --- a/dev/devicelab/lib/tasks/size_tests.dart +++ /dev/null @@ -1,58 +0,0 @@ -// Copyright 2016 The Chromium Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'dart:io'; - -import 'package:flutter_devicelab/framework/adb.dart'; - -import '../framework/framework.dart'; -import '../framework/ios.dart'; -import '../framework/utils.dart'; - -TaskFunction createBasicMaterialAppSizeTest() { - return () async { - const String sampleAppName = 'sample_flutter_app'; - final Directory sampleDir = dir('${Directory.systemTemp.path}/$sampleAppName'); - - if (await sampleDir.exists()) - rmTree(sampleDir); - - final Stopwatch watch = new Stopwatch(); - int releaseSizeInBytes; - - await inDirectory(Directory.systemTemp, () async { - await flutter('create', options: [sampleAppName]); - - if (!(await sampleDir.exists())) - throw 'Failed to create sample Flutter app in ${sampleDir.path}'; - - await inDirectory(sampleDir, () async { - await flutter('packages', options: ['get']); - await flutter('clean'); - - if (deviceOperatingSystem == DeviceOperatingSystem.ios) { - await prepareProvisioningCertificates(sampleDir.path); - watch.start(); - await flutter('build', options: ['ios', '--release']); - watch.stop(); - // IPAs are created manually AFAICT - await exec('tar', ['-zcf', 'build/app.ipa', 'build/ios/Release-iphoneos/Runner.app/']); - releaseSizeInBytes = await file('${sampleDir.path}/build/app.ipa').length(); - } else { - watch.start(); - await flutter('build', options: ['apk', '--release']); - watch.stop(); - releaseSizeInBytes = await file('${sampleDir.path}/build/app/outputs/apk/app-release.apk').length(); - } - }); - }); - - return new TaskResult.success( - { - 'release_size_in_bytes': releaseSizeInBytes, - 'build_time_millis': watch.elapsedMilliseconds, - }, - benchmarkScoreKeys: ['release_size_in_bytes', 'build_time_millis']); - }; -} diff --git a/dev/devicelab/manifest.yaml b/dev/devicelab/manifest.yaml index 43542fbd554..c4badf5db8d 100644 --- a/dev/devicelab/manifest.yaml +++ b/dev/devicelab/manifest.yaml @@ -32,23 +32,74 @@ # a custom task timeout, specified in minutes. tasks: - # Deviceless tests + # Tests of compiling in a variety of modes - # TODO: make these not require "has-android-device"; it is only there to - # ensure we have the Android SDK. - - complex_layout__build: + complex_layout_android__compile: description: > - Collects various performance metrics from AOT builds of the Complex - Layout sample app. + Collects various performance metrics of compiling the Complex + Layout sample app for Android from Linux. stage: devicelab - required_agent_capabilities: ["has-android-device"] + required_agent_capabilities: ["linux/android"] + flaky: true - basic_material_app__size: + complex_layout_ios__compile: description: > - Measures the APK size of a basic material app. + Collects various performance metrics of compiling the Complex + Layout sample app for iOS from Mac. + stage: devicelab_ios + required_agent_capabilities: ["has-ios-device"] + flaky: true + + complex_layout_win__compile: + description: > + Collects various performance metrics of compiling the Complex + Layout for Android from Windows. + stage: devicelab_win + required_agent_capabilities: ["windows"] + + basic_material_app_android__compile: + description: > + Collects various performance metrics of compiling the default + app for Android from Linux. stage: devicelab - required_agent_capabilities: ["has-android-device"] + required_agent_capabilities: ["linux/android"] + flaky: true + + basic_material_app_ios__compile: + description: > + Collects various performance metrics of compiling the default + app for iOS from Mac. + stage: devicelab_ios + required_agent_capabilities: ["has-ios-device"] + flaky: true + + basic_material_app_win__compile: + description: > + Collects various performance metrics of compiling the default + app for Android from Windows. + stage: devicelab_win + required_agent_capabilities: ["windows"] + + flutter_gallery_android__compile: + description: > + Collects various performance metrics of compiling the Flutter + Gallery for Android from Linux. + stage: devicelab + required_agent_capabilities: ["linux/android"] + + flutter_gallery_ios__compile: + description: > + Collects various performance metrics of compiling the Flutter + Gallery for iOS from Mac. + stage: devicelab_ios + required_agent_capabilities: ["has-ios-device"] + + flutter_gallery_win__compile: + description: > + Collects various performance metrics of compiling the Flutter + Gallery for Android from Windows. + stage: devicelab_win + required_agent_capabilities: ["windows"] # Android on-device tests @@ -197,7 +248,6 @@ tasks: Checks that external UIs work on iOS. stage: devicelab_ios required_agent_capabilities: ["has-ios-device"] - flaky: true channels_integration_test_ios: description: > @@ -229,14 +279,12 @@ tasks: Measures the startup time of the Flutter Gallery app on iOS. stage: devicelab_ios required_agent_capabilities: ["has-ios-device"] - flaky: true complex_layout_ios__start_up: description: > Measures the startup time of the Complex Layout sample app on iOS. stage: devicelab_ios required_agent_capabilities: ["has-ios-device"] - flaky: true flutter_gallery_ios__transition_perf: description: > @@ -245,12 +293,6 @@ tasks: stage: devicelab_ios required_agent_capabilities: ["has-ios-device"] - basic_material_app_ios__size: - description: > - Measures the IPA size of a basic material app. - stage: devicelab_ios - required_agent_capabilities: ["has-ios-device"] - microbenchmarks_ios: description: > Runs benchmarks from dev/benchmarks/microbenchmarks on iOS. @@ -259,12 +301,12 @@ tasks: timeout_in_minutes: 30 flaky: true -# flutter_view_ios__start_up: -# description: > -# Verifies that Flutter View can be used from an iOS project. -# stage: devicelab_ios -# required_agent_capabilities: ["has-ios-device"] -# flaky: true + flutter_view_ios__start_up: + description: > + Verifies that Flutter View can be used from an iOS project. + stage: devicelab_ios + required_agent_capabilities: ["has-ios-device"] + flaky: true integration_ui_ios: description: > @@ -292,13 +334,6 @@ tasks: stage: devicelab_win required_agent_capabilities: ["windows"] - flutter_gallery_win__build: - description: > - Collects various performance metrics from AOT builds of the Flutter - Gallery on Windows. - stage: devicelab_win - required_agent_capabilities: ["windows"] - hot_mode_dev_cycle_win__benchmark: description: > Measures the performance of Dart VM hot patching feature on Windows. @@ -325,20 +360,6 @@ tasks: stage: devicelab required_agent_capabilities: ["linux/android"] - flutter_gallery__build: - description: > - Collects various performance metrics from AOT builds of the Flutter - Gallery. - stage: devicelab - required_agent_capabilities: ["linux/android"] - - flutter_gallery__preview_dart_2_build: - description: > - Collects various performance metrics from AOT builds of the Flutter - Gallery under --preview-dart-2 that enables new Dart 2.0 frontend. - stage: devicelab - required_agent_capabilities: ["linux/android"] - flutter_gallery__start_up: description: > Measures the startup time of the Flutter Gallery app on Android.