From 46db10137aec025247b215f6ddf2bbff498a075f Mon Sep 17 00:00:00 2001 From: yusufm <463376+yusufm@users.noreply.github.com> Date: Thu, 29 Apr 2021 12:28:15 -0700 Subject: [PATCH] Adding win devicelab bot results to dashboard. (#80974) (#81077) * Adding win devicelab bot results to dashboard. (#80974) * Update prod_builders.json * Update prod_builders.json and remove agent manifest.yaml Changes to switch off agent results, which are no longer being used. Co-authored-by: yusufm --- dev/devicelab/manifest.yaml | 349 -------------------------- dev/devicelab/test/manifest_test.dart | 207 --------------- dev/prod_builders.json | 42 ++++ 3 files changed, 42 insertions(+), 556 deletions(-) delete mode 100644 dev/devicelab/manifest.yaml delete mode 100644 dev/devicelab/test/manifest_test.dart diff --git a/dev/devicelab/manifest.yaml b/dev/devicelab/manifest.yaml deleted file mode 100644 index 86806f7bf39..00000000000 --- a/dev/devicelab/manifest.yaml +++ /dev/null @@ -1,349 +0,0 @@ -# Describes the tasks we run in the continuous integration (CI) environment. -# -# Cocoon[1] uses this file to generate a checklist of tasks to be performed for -# every master commit. -# -# [1] github.com/flutter/cocoon - -# CI tasks. -# -# Each key in this dictionary is the unique name of a task, which also -# corresponds to a file in the "bin/" directory that the task runner will run. -# -# Required properties: -# -# * description: free form string -# describes what the task is about -# * stage: one of "devicelab", "devicelab_ios", "devicelab_win" -# tasks are grouped by stage so they appear next to each on the dashboard and have a distinct -# icon attached to them. Stages can be used to create dependencies, e.g. we may decide to not -# run device tests on red Cirrus builds. -# * required_agent_capabilities: a list of strings -# list of capabilities a devicelab agent must have to be able to run this task. -# -# Optional properties: -# -# * flaky: boolean true or false -# whether the task is considered flaky; the result of running a flaky task does not affect -# the overall build status. Tests should be marked flaky when newly added, until they have -# been proved for a few cycles. Tests that are actually flaky but not being actively worked -# on should be hidden because they just cause confusion otherwise. -# * on_luci: boolean true or false -# whether the task runs on LUCI builders. Tests will be gradually marked on_luci during the -# devicelab to LUCI migration. And when the migration is complete, on_luci flag will be -# removed. - -tasks: - # Tests of compiling in a variety of modes - - complex_layout_ios__compile: - description: > - Collects various performance metrics of compiling the Complex - Layout sample app for iOS from Mac. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - complex_layout_win__compile: - description: > - Collects various performance metrics of compiling the Complex - Layout for Android from Windows. - stage: devicelab_win - required_agent_capabilities: ["windows/android"] - - basic_material_app_ios__compile: - description: > - Collects various performance metrics of compiling the default - app for iOS from Mac. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - basic_material_app_win__compile: - description: > - Collects various performance metrics of compiling the default - app for Android from Windows. - stage: devicelab_win - required_agent_capabilities: ["windows/android"] - - flutter_gallery_ios__compile: - description: > - Collects various performance metrics of compiling the Flutter - Gallery for iOS from Mac. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - flutter_gallery_win__compile: - description: > - Collects various performance metrics of compiling the Flutter - Gallery for Android from Windows. - stage: devicelab_win - required_agent_capabilities: ["windows/android"] - - windows_chrome_dev_mode: - description: > - Run flutter web on the devicelab and hot restart. - stage: devicelab_win - required_agent_capabilities: ["windows/android"] - - # iOS on-device tests - - flutter_gallery_sksl_warmup__transition_perf_e2e_ios32: - description: > - Measures the runtime performance of Flutter gallery transitions on iPhone4s - with SkSL shader warm-up with e2e. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios32"] - flaky: true - - ios_defines_test: - description: > - Builds a Framework with a --dart-define and verifies it can be used as a constant - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - ios_content_validation_test: - description: > - Builds an obfuscated app and verifies contents and structure - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - on_luci: true - - tiles_scroll_perf_ios__timeline_summary: - description: > - Measures the runtime performance of the tiles tab in the Complex Layout sample app on iPhone 6. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - platform_views_scroll_perf_ios__timeline_summary: - description: > - Measures the runtime performance of platform views in the Complex Layout sample app on iPhone 6. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - flutter_gallery_ios32__start_up: - description: > - Measures the startup time of the Flutter Gallery app on 32-bit iOS (iPhone 4S). - stage: devicelab_ios - required_agent_capabilities: ["mac/ios32"] - - flavors_test_ios: - description: > - Checks that flavored builds work on iPhone 6. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - external_ui_integration_test_ios: - description: > - Checks that external UIs work on iPhone 6. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - channels_integration_test_ios: - description: > - Checks that platform channels work on iPhone 6. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - platform_interaction_test_ios: - description: > - Checks platform interaction on iPhone 6. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - platform_channel_sample_test_ios: - description: > - Runs a driver test on the Platform Channel sample app on iPhone 6 Objective-C project. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - platform_channel_sample_test_swift: - description: > - Runs a driver test on the Platform Channel sample app on iPhone 6 Swift project. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - platform_view_ios__start_up: - description: > - Verifies that Platform View can be used from an iOS project. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - backdrop_filter_perf_ios__timeline_summary: - description: > - Measures the runtime performance of backdrop filter blurs on iOS. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - post_backdrop_filter_perf_ios__timeline_summary: - description: > - Measures the runtime performance of animations after a backdrop filter is removed on iOS. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - complex_layout_scroll_perf_ios__timeline_summary: - description: > - Measures the runtime performance of the Complex Layout sample app on - iOS. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - flutter_gallery_ios__start_up: - description: > - Measures the startup time of the Flutter Gallery app on iPhone 6. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - complex_layout_ios__start_up: - description: > - Measures the startup time of the Complex Layout sample app on iPhone 6. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - flutter_gallery_ios__transition_perf: - description: > - Measures the performance of screen transitions in Flutter Gallery on - iOS. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - hello_world_ios__compile: - description: > - Measures the IPA size of Hello World. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - microbenchmarks_ios: - description: > - Runs benchmarks from dev/benchmarks/microbenchmarks on iPhone 6. - stage: devicelab_ios - #TODO(godofredoc): Remove ios/12 capability once - # https://github.com/flutter/flutter/issues/49635 is fixed. - required_agent_capabilities: ["mac/ios", "ios/12"] - - flutter_view_ios__start_up: - description: > - Verifies that Flutter View can be used from an iOS project. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - integration_ui_ios_driver: - description: > - Runs end-to-end Flutter tests on iPhone 6. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - integration_ui_ios_keyboard_resize: - description: > - Runs end-to-end Flutter tests on iPhone 6. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - integration_ui_ios_screenshot: - description: > - Runs end-to-end Flutter tests on iPhone 6. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - integration_ui_ios_textfield: - description: > - Runs end-to-end Flutter tests on iPhone 6. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - ios_platform_view_tests: - description: > - Runs end-to-end tests with platform views in the scene. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - macos_chrome_dev_mode: - description: > - Run flutter web on the devicelab and hot restart. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - simple_animation_perf_ios: - description: > - Measure CPU/GPU usage percentages of a simple animation. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - smoke_catalina_hot_mode_dev_cycle_ios__benchmark: - description: > - A some test that runs on macOS Catalina, which is a clone of the Dart VM hot patching performance benchmarking. - stage: devicelab_ios - required_agent_capabilities: ["mac-catalina/ios"] - - smoke_catalina_start_up: - description: > - A smoke test that runs on macOS Catalina, which is a clone of the Gallery startup latency test. - stage: devicelab - required_agent_capabilities: ["mac-catalina/android"] - - # macOS target platform tests - hot_mode_dev_cycle_macos_target__benchmark: - description: > - Checks the functionality and performance of hot reload on a macOS target platform - stage: devicelab - required_agent_capabilities: ["mac/ios"] - - ios_app_with_extensions_test: - description: > - Checks that an iOS app with extensions can be built for physical and simulated devices. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - # Tests running on Windows host - - flavors_test_win: - description: > - Checks that flavored builds work on Windows. - stage: devicelab_win - required_agent_capabilities: ["windows/android"] - - channels_integration_test_win: - description: > - Checks that platform channels work when app is launched from Windows. - stage: devicelab_win - required_agent_capabilities: ["windows/android"] - - hot_mode_dev_cycle_win__benchmark: - description: > - Measures the performance of Dart VM hot patching feature on Windows. - stage: devicelab_win - required_agent_capabilities: ["windows/android"] - - # Tests running on Linux hosts - - flutter_gallery__transition_perf_e2e_ios32: - description: > - Measures the performance of screen transitions in Flutter Gallery on - Android with e2e. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios32"] - flaky: true - - flutter_gallery__transition_perf_e2e_ios: - description: > - Measures the performance of screen transitions in Flutter Gallery on - Android with e2e. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - new_gallery_ios__transition_perf: - description: > - Measures the performance of screen transitions in the new Flutter Gallery on iOS. - stage: devicelab_ios - required_agent_capabilities: ["mac/ios"] - - large_image_changer_perf_ios: - description: > - Measures memory, cpu, and gpu usage when rotating through a series of large images. - stage: devicelab - required_agent_capabilities: ["mac/ios"] - - # android_splash_screen_integration_test: - # description: > - # Runs end-to-end test of Flutter's Android splash behavior. - # stage: devicelab - # required_agent_capabilities: ["linux/android"] - # flaky: true diff --git a/dev/devicelab/test/manifest_test.dart b/dev/devicelab/test/manifest_test.dart deleted file mode 100644 index c0cc1724ed8..00000000000 --- a/dev/devicelab/test/manifest_test.dart +++ /dev/null @@ -1,207 +0,0 @@ -// Copyright 2014 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'dart:io'; - -import 'package:flutter_devicelab/framework/manifest.dart'; - -import 'common.dart'; - -void main() { - group('production manifest', () { - test('must be valid', () { - final Manifest manifest = loadTaskManifest(); - expect(manifest.tasks, isNotEmpty); - - final ManifestTask task = manifest.tasks.firstWhere((ManifestTask task) => task.name == 'simple_animation_perf_ios'); - expect(task.description, 'Measure CPU/GPU usage percentages of a simple animation.\n'); - expect(task.stage, 'devicelab_ios'); - expect(task.requiredAgentCapabilities, ['mac/ios']); - - for (final ManifestTask task in manifest.tasks) { - final File taskFile = File('bin/tasks/${task.name}.dart'); - expect(taskFile.existsSync(), true, - reason: 'File ${taskFile.path} corresponding to manifest task "${task.name}" not found'); - } - }); - }); - - group('manifest parser', () { - void testManifestError( - String testDescription, - String errorMessage, - String yaml, - ) { - test(testDescription, () { - try { - loadTaskManifest(yaml); - } on ManifestError catch (error) { - expect(error.message, errorMessage); - } - }); - } - - test('accepts task with minimum amount of configuration', () { - final Manifest manifest = loadTaskManifest(''' -tasks: - minimum_configuration_task: - description: Description is mandatory. - stage: stage_is_mandatory_too - required_agent_capabilities: ["so-is-capability"] -'''); - - expect(manifest.tasks.single.description, 'Description is mandatory.'); - expect(manifest.tasks.single.stage, 'stage_is_mandatory_too'); - expect(manifest.tasks.single.requiredAgentCapabilities, ['so-is-capability']); - expect(manifest.tasks.single.isFlaky, false); - expect(manifest.tasks.single.timeoutInMinutes, null); - }); - - testManifestError( - 'invalid top-level type', - 'Manifest must be a dictionary but was YamlScalar: null', - '', - ); - - testManifestError( - 'invalid top-level key', - 'Unrecognized property "bad" in manifest. Allowed properties: tasks', - ''' - bad: - key: yes - ''', - ); - - testManifestError( - 'invalid tasks list type', - 'Value of "tasks" must be a dictionary but was YamlList: [a, b]', - ''' - tasks: - - a - - b - ''', - ); - - testManifestError( - 'invalid task name type', - 'Task name must be a string but was int: 1', - ''' - tasks: - 1: 2 - ''', - ); - - testManifestError( - 'invalid task type', - 'Value of task "foo" must be a dictionary but was int: 2', - ''' - tasks: - foo: 2 - ''', - ); - - testManifestError( - 'invalid task property', - 'Unrecognized property "bar" in Value of task "foo". Allowed properties: description, stage, required_agent_capabilities, flaky, timeout_in_minutes, on_luci', - ''' - tasks: - foo: - bar: 2 - ''', - ); - - testManifestError( - 'invalid required_agent_capabilities type', - 'required_agent_capabilities must be a list but was int: 1', - ''' - tasks: - foo: - required_agent_capabilities: 1 - ''', - ); - - testManifestError( - 'invalid required_agent_capabilities element type', - 'required_agent_capabilities[0] must be a string but was int: 1', - ''' - tasks: - foo: - required_agent_capabilities: [1] - ''', - ); - - testManifestError( - 'missing description', - 'Task description must not be empty in task "foo".', - ''' - tasks: - foo: - required_agent_capabilities: ["a"] - ''', - ); - - testManifestError( - 'missing stage', - 'Task stage must not be empty in task "foo".', - ''' - tasks: - foo: - description: b - required_agent_capabilities: ["a"] - ''', - ); - - testManifestError( - 'missing required_agent_capabilities', - 'requiredAgentCapabilities must not be empty in task "foo".', - ''' - tasks: - foo: - description: b - stage: c - required_agent_capabilities: [] - ''', - ); - - testManifestError( - 'bad flaky type', - 'flaky must be a boolean but was String: not-a-boolean', - ''' - tasks: - foo: - description: b - stage: c - required_agent_capabilities: ["a"] - flaky: not-a-boolean - ''', - ); - - test('accepts boolean flaky option', () { - final Manifest manifest = loadTaskManifest(''' -tasks: - flaky_task: - description: d - stage: s - required_agent_capabilities: ["c"] - flaky: true -'''); - - expect(manifest.tasks.single.name, 'flaky_task'); - expect(manifest.tasks.single.isFlaky, isTrue); - }); - - test('accepts custom timeout_in_minutes option', () { - final Manifest manifest = loadTaskManifest(''' -tasks: - task_with_custom_timeout: - description: d - stage: s - required_agent_capabilities: ["c"] - timeout_in_minutes: 120 -'''); - - expect(manifest.tasks.single.timeoutInMinutes, 120); - }); - }); -} diff --git a/dev/prod_builders.json b/dev/prod_builders.json index 1f005eee2da..e4609cee36e 100644 --- a/dev/prod_builders.json +++ b/dev/prod_builders.json @@ -863,6 +863,48 @@ "repo": "flutter", "task_name": "win_web_tool_tests", "flaky": true + }, + { + "name": "Windows_android complex_layout_win__compile", + "repo": "flutter", + "task_name": "complex_layout_win__compile", + "flaky": false + }, + { + "name": "Windows_android basic_material_app_win__compile", + "repo": "flutter", + "task_name": "basic_material_app_win__compile", + "flaky": false + }, + { + "name": "Windows_android flutter_gallery_win__compile", + "repo": "flutter", + "task_name": "flutter_gallery_win__compile", + "flaky": false + }, + { + "name": "Windows_android windows_chrome_dev_mode", + "repo": "flutter", + "task_name": "windows_chrome_dev_mode", + "flaky": false + }, + { + "name": "Windows_android flavors_test_win", + "repo": "flutter", + "task_name": "flavors_test_win", + "flaky": false + }, + { + "name": "Windows_android channels_integration_test_win", + "repo": "flutter", + "task_name": "channels_integration_test_win", + "flaky": false + }, + { + "name": "Windows_android hot_mode_dev_cycle_win__benchmark", + "repo": "flutter", + "task_name": "hot_mode_dev_cycle_win__benchmark", + "flaky": false } ] }