mirror of
https://github.com/flutter/flutter.git
synced 2026-02-20 02:29:02 +08:00
* Adding win devicelab bot results to dashboard. (#80974) * Update prod_builders.json * Update prod_builders.json and remove agent manifest.yaml Changes to switch off agent results, which are no longer being used. Co-authored-by: yusufm <mohsinally@google.com>
This commit is contained in:
yusufm
GitHub
parent
adc687823a
commit
46db10137a
@ -1,349 +0,0 @@
|
||||
# Describes the tasks we run in the continuous integration (CI) environment.
|
||||
#
|
||||
# Cocoon[1] uses this file to generate a checklist of tasks to be performed for
|
||||
# every master commit.
|
||||
#
|
||||
# [1] github.com/flutter/cocoon
|
||||
|
||||
# CI tasks.
|
||||
#
|
||||
# Each key in this dictionary is the unique name of a task, which also
|
||||
# corresponds to a file in the "bin/" directory that the task runner will run.
|
||||
#
|
||||
# Required properties:
|
||||
#
|
||||
# * description: free form string
|
||||
# describes what the task is about
|
||||
# * stage: one of "devicelab", "devicelab_ios", "devicelab_win"
|
||||
# tasks are grouped by stage so they appear next to each on the dashboard and have a distinct
|
||||
# icon attached to them. Stages can be used to create dependencies, e.g. we may decide to not
|
||||
# run device tests on red Cirrus builds.
|
||||
# * required_agent_capabilities: a list of strings
|
||||
# list of capabilities a devicelab agent must have to be able to run this task.
|
||||
#
|
||||
# Optional properties:
|
||||
#
|
||||
# * flaky: boolean true or false
|
||||
# whether the task is considered flaky; the result of running a flaky task does not affect
|
||||
# the overall build status. Tests should be marked flaky when newly added, until they have
|
||||
# been proved for a few cycles. Tests that are actually flaky but not being actively worked
|
||||
# on should be hidden because they just cause confusion otherwise.
|
||||
# * on_luci: boolean true or false
|
||||
# whether the task runs on LUCI builders. Tests will be gradually marked on_luci during the
|
||||
# devicelab to LUCI migration. And when the migration is complete, on_luci flag will be
|
||||
# removed.
|
||||
|
||||
tasks:
|
||||
# Tests of compiling in a variety of modes
|
||||
|
||||
complex_layout_ios__compile:
|
||||
description: >
|
||||
Collects various performance metrics of compiling the Complex
|
||||
Layout sample app for iOS from Mac.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
complex_layout_win__compile:
|
||||
description: >
|
||||
Collects various performance metrics of compiling the Complex
|
||||
Layout for Android from Windows.
|
||||
stage: devicelab_win
|
||||
required_agent_capabilities: ["windows/android"]
|
||||
|
||||
basic_material_app_ios__compile:
|
||||
description: >
|
||||
Collects various performance metrics of compiling the default
|
||||
app for iOS from Mac.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
basic_material_app_win__compile:
|
||||
description: >
|
||||
Collects various performance metrics of compiling the default
|
||||
app for Android from Windows.
|
||||
stage: devicelab_win
|
||||
required_agent_capabilities: ["windows/android"]
|
||||
|
||||
flutter_gallery_ios__compile:
|
||||
description: >
|
||||
Collects various performance metrics of compiling the Flutter
|
||||
Gallery for iOS from Mac.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
flutter_gallery_win__compile:
|
||||
description: >
|
||||
Collects various performance metrics of compiling the Flutter
|
||||
Gallery for Android from Windows.
|
||||
stage: devicelab_win
|
||||
required_agent_capabilities: ["windows/android"]
|
||||
|
||||
windows_chrome_dev_mode:
|
||||
description: >
|
||||
Run flutter web on the devicelab and hot restart.
|
||||
stage: devicelab_win
|
||||
required_agent_capabilities: ["windows/android"]
|
||||
|
||||
# iOS on-device tests
|
||||
|
||||
flutter_gallery_sksl_warmup__transition_perf_e2e_ios32:
|
||||
description: >
|
||||
Measures the runtime performance of Flutter gallery transitions on iPhone4s
|
||||
with SkSL shader warm-up with e2e.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios32"]
|
||||
flaky: true
|
||||
|
||||
ios_defines_test:
|
||||
description: >
|
||||
Builds a Framework with a --dart-define and verifies it can be used as a constant
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
ios_content_validation_test:
|
||||
description: >
|
||||
Builds an obfuscated app and verifies contents and structure
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
on_luci: true
|
||||
|
||||
tiles_scroll_perf_ios__timeline_summary:
|
||||
description: >
|
||||
Measures the runtime performance of the tiles tab in the Complex Layout sample app on iPhone 6.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
platform_views_scroll_perf_ios__timeline_summary:
|
||||
description: >
|
||||
Measures the runtime performance of platform views in the Complex Layout sample app on iPhone 6.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
flutter_gallery_ios32__start_up:
|
||||
description: >
|
||||
Measures the startup time of the Flutter Gallery app on 32-bit iOS (iPhone 4S).
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios32"]
|
||||
|
||||
flavors_test_ios:
|
||||
description: >
|
||||
Checks that flavored builds work on iPhone 6.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
external_ui_integration_test_ios:
|
||||
description: >
|
||||
Checks that external UIs work on iPhone 6.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
channels_integration_test_ios:
|
||||
description: >
|
||||
Checks that platform channels work on iPhone 6.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
platform_interaction_test_ios:
|
||||
description: >
|
||||
Checks platform interaction on iPhone 6.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
platform_channel_sample_test_ios:
|
||||
description: >
|
||||
Runs a driver test on the Platform Channel sample app on iPhone 6 Objective-C project.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
platform_channel_sample_test_swift:
|
||||
description: >
|
||||
Runs a driver test on the Platform Channel sample app on iPhone 6 Swift project.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
platform_view_ios__start_up:
|
||||
description: >
|
||||
Verifies that Platform View can be used from an iOS project.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
backdrop_filter_perf_ios__timeline_summary:
|
||||
description: >
|
||||
Measures the runtime performance of backdrop filter blurs on iOS.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
post_backdrop_filter_perf_ios__timeline_summary:
|
||||
description: >
|
||||
Measures the runtime performance of animations after a backdrop filter is removed on iOS.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
complex_layout_scroll_perf_ios__timeline_summary:
|
||||
description: >
|
||||
Measures the runtime performance of the Complex Layout sample app on
|
||||
iOS.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
flutter_gallery_ios__start_up:
|
||||
description: >
|
||||
Measures the startup time of the Flutter Gallery app on iPhone 6.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
complex_layout_ios__start_up:
|
||||
description: >
|
||||
Measures the startup time of the Complex Layout sample app on iPhone 6.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
flutter_gallery_ios__transition_perf:
|
||||
description: >
|
||||
Measures the performance of screen transitions in Flutter Gallery on
|
||||
iOS.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
hello_world_ios__compile:
|
||||
description: >
|
||||
Measures the IPA size of Hello World.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
microbenchmarks_ios:
|
||||
description: >
|
||||
Runs benchmarks from dev/benchmarks/microbenchmarks on iPhone 6.
|
||||
stage: devicelab_ios
|
||||
#TODO(godofredoc): Remove ios/12 capability once
|
||||
# https://github.com/flutter/flutter/issues/49635 is fixed.
|
||||
required_agent_capabilities: ["mac/ios", "ios/12"]
|
||||
|
||||
flutter_view_ios__start_up:
|
||||
description: >
|
||||
Verifies that Flutter View can be used from an iOS project.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
integration_ui_ios_driver:
|
||||
description: >
|
||||
Runs end-to-end Flutter tests on iPhone 6.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
integration_ui_ios_keyboard_resize:
|
||||
description: >
|
||||
Runs end-to-end Flutter tests on iPhone 6.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
integration_ui_ios_screenshot:
|
||||
description: >
|
||||
Runs end-to-end Flutter tests on iPhone 6.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
integration_ui_ios_textfield:
|
||||
description: >
|
||||
Runs end-to-end Flutter tests on iPhone 6.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
ios_platform_view_tests:
|
||||
description: >
|
||||
Runs end-to-end tests with platform views in the scene.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
macos_chrome_dev_mode:
|
||||
description: >
|
||||
Run flutter web on the devicelab and hot restart.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
simple_animation_perf_ios:
|
||||
description: >
|
||||
Measure CPU/GPU usage percentages of a simple animation.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
smoke_catalina_hot_mode_dev_cycle_ios__benchmark:
|
||||
description: >
|
||||
A some test that runs on macOS Catalina, which is a clone of the Dart VM hot patching performance benchmarking.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac-catalina/ios"]
|
||||
|
||||
smoke_catalina_start_up:
|
||||
description: >
|
||||
A smoke test that runs on macOS Catalina, which is a clone of the Gallery startup latency test.
|
||||
stage: devicelab
|
||||
required_agent_capabilities: ["mac-catalina/android"]
|
||||
|
||||
# macOS target platform tests
|
||||
hot_mode_dev_cycle_macos_target__benchmark:
|
||||
description: >
|
||||
Checks the functionality and performance of hot reload on a macOS target platform
|
||||
stage: devicelab
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
ios_app_with_extensions_test:
|
||||
description: >
|
||||
Checks that an iOS app with extensions can be built for physical and simulated devices.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
# Tests running on Windows host
|
||||
|
||||
flavors_test_win:
|
||||
description: >
|
||||
Checks that flavored builds work on Windows.
|
||||
stage: devicelab_win
|
||||
required_agent_capabilities: ["windows/android"]
|
||||
|
||||
channels_integration_test_win:
|
||||
description: >
|
||||
Checks that platform channels work when app is launched from Windows.
|
||||
stage: devicelab_win
|
||||
required_agent_capabilities: ["windows/android"]
|
||||
|
||||
hot_mode_dev_cycle_win__benchmark:
|
||||
description: >
|
||||
Measures the performance of Dart VM hot patching feature on Windows.
|
||||
stage: devicelab_win
|
||||
required_agent_capabilities: ["windows/android"]
|
||||
|
||||
# Tests running on Linux hosts
|
||||
|
||||
flutter_gallery__transition_perf_e2e_ios32:
|
||||
description: >
|
||||
Measures the performance of screen transitions in Flutter Gallery on
|
||||
Android with e2e.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios32"]
|
||||
flaky: true
|
||||
|
||||
flutter_gallery__transition_perf_e2e_ios:
|
||||
description: >
|
||||
Measures the performance of screen transitions in Flutter Gallery on
|
||||
Android with e2e.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
new_gallery_ios__transition_perf:
|
||||
description: >
|
||||
Measures the performance of screen transitions in the new Flutter Gallery on iOS.
|
||||
stage: devicelab_ios
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
large_image_changer_perf_ios:
|
||||
description: >
|
||||
Measures memory, cpu, and gpu usage when rotating through a series of large images.
|
||||
stage: devicelab
|
||||
required_agent_capabilities: ["mac/ios"]
|
||||
|
||||
# android_splash_screen_integration_test:
|
||||
# description: >
|
||||
# Runs end-to-end test of Flutter's Android splash behavior.
|
||||
# stage: devicelab
|
||||
# required_agent_capabilities: ["linux/android"]
|
||||
# flaky: true
|
||||
@ -1,207 +0,0 @@
|
||||
// Copyright 2014 The Flutter Authors. All rights reserved.
|
||||
// Use of this source code is governed by a BSD-style license that can be
|
||||
// found in the LICENSE file.
|
||||
|
||||
import 'dart:io';
|
||||
|
||||
import 'package:flutter_devicelab/framework/manifest.dart';
|
||||
|
||||
import 'common.dart';
|
||||
|
||||
void main() {
|
||||
group('production manifest', () {
|
||||
test('must be valid', () {
|
||||
final Manifest manifest = loadTaskManifest();
|
||||
expect(manifest.tasks, isNotEmpty);
|
||||
|
||||
final ManifestTask task = manifest.tasks.firstWhere((ManifestTask task) => task.name == 'simple_animation_perf_ios');
|
||||
expect(task.description, 'Measure CPU/GPU usage percentages of a simple animation.\n');
|
||||
expect(task.stage, 'devicelab_ios');
|
||||
expect(task.requiredAgentCapabilities, <String>['mac/ios']);
|
||||
|
||||
for (final ManifestTask task in manifest.tasks) {
|
||||
final File taskFile = File('bin/tasks/${task.name}.dart');
|
||||
expect(taskFile.existsSync(), true,
|
||||
reason: 'File ${taskFile.path} corresponding to manifest task "${task.name}" not found');
|
||||
}
|
||||
});
|
||||
});
|
||||
|
||||
group('manifest parser', () {
|
||||
void testManifestError(
|
||||
String testDescription,
|
||||
String errorMessage,
|
||||
String yaml,
|
||||
) {
|
||||
test(testDescription, () {
|
||||
try {
|
||||
loadTaskManifest(yaml);
|
||||
} on ManifestError catch (error) {
|
||||
expect(error.message, errorMessage);
|
||||
}
|
||||
});
|
||||
}
|
||||
|
||||
test('accepts task with minimum amount of configuration', () {
|
||||
final Manifest manifest = loadTaskManifest('''
|
||||
tasks:
|
||||
minimum_configuration_task:
|
||||
description: Description is mandatory.
|
||||
stage: stage_is_mandatory_too
|
||||
required_agent_capabilities: ["so-is-capability"]
|
||||
''');
|
||||
|
||||
expect(manifest.tasks.single.description, 'Description is mandatory.');
|
||||
expect(manifest.tasks.single.stage, 'stage_is_mandatory_too');
|
||||
expect(manifest.tasks.single.requiredAgentCapabilities, <String>['so-is-capability']);
|
||||
expect(manifest.tasks.single.isFlaky, false);
|
||||
expect(manifest.tasks.single.timeoutInMinutes, null);
|
||||
});
|
||||
|
||||
testManifestError(
|
||||
'invalid top-level type',
|
||||
'Manifest must be a dictionary but was YamlScalar: null',
|
||||
'',
|
||||
);
|
||||
|
||||
testManifestError(
|
||||
'invalid top-level key',
|
||||
'Unrecognized property "bad" in manifest. Allowed properties: tasks',
|
||||
'''
|
||||
bad:
|
||||
key: yes
|
||||
''',
|
||||
);
|
||||
|
||||
testManifestError(
|
||||
'invalid tasks list type',
|
||||
'Value of "tasks" must be a dictionary but was YamlList: [a, b]',
|
||||
'''
|
||||
tasks:
|
||||
- a
|
||||
- b
|
||||
''',
|
||||
);
|
||||
|
||||
testManifestError(
|
||||
'invalid task name type',
|
||||
'Task name must be a string but was int: 1',
|
||||
'''
|
||||
tasks:
|
||||
1: 2
|
||||
''',
|
||||
);
|
||||
|
||||
testManifestError(
|
||||
'invalid task type',
|
||||
'Value of task "foo" must be a dictionary but was int: 2',
|
||||
'''
|
||||
tasks:
|
||||
foo: 2
|
||||
''',
|
||||
);
|
||||
|
||||
testManifestError(
|
||||
'invalid task property',
|
||||
'Unrecognized property "bar" in Value of task "foo". Allowed properties: description, stage, required_agent_capabilities, flaky, timeout_in_minutes, on_luci',
|
||||
'''
|
||||
tasks:
|
||||
foo:
|
||||
bar: 2
|
||||
''',
|
||||
);
|
||||
|
||||
testManifestError(
|
||||
'invalid required_agent_capabilities type',
|
||||
'required_agent_capabilities must be a list but was int: 1',
|
||||
'''
|
||||
tasks:
|
||||
foo:
|
||||
required_agent_capabilities: 1
|
||||
''',
|
||||
);
|
||||
|
||||
testManifestError(
|
||||
'invalid required_agent_capabilities element type',
|
||||
'required_agent_capabilities[0] must be a string but was int: 1',
|
||||
'''
|
||||
tasks:
|
||||
foo:
|
||||
required_agent_capabilities: [1]
|
||||
''',
|
||||
);
|
||||
|
||||
testManifestError(
|
||||
'missing description',
|
||||
'Task description must not be empty in task "foo".',
|
||||
'''
|
||||
tasks:
|
||||
foo:
|
||||
required_agent_capabilities: ["a"]
|
||||
''',
|
||||
);
|
||||
|
||||
testManifestError(
|
||||
'missing stage',
|
||||
'Task stage must not be empty in task "foo".',
|
||||
'''
|
||||
tasks:
|
||||
foo:
|
||||
description: b
|
||||
required_agent_capabilities: ["a"]
|
||||
''',
|
||||
);
|
||||
|
||||
testManifestError(
|
||||
'missing required_agent_capabilities',
|
||||
'requiredAgentCapabilities must not be empty in task "foo".',
|
||||
'''
|
||||
tasks:
|
||||
foo:
|
||||
description: b
|
||||
stage: c
|
||||
required_agent_capabilities: []
|
||||
''',
|
||||
);
|
||||
|
||||
testManifestError(
|
||||
'bad flaky type',
|
||||
'flaky must be a boolean but was String: not-a-boolean',
|
||||
'''
|
||||
tasks:
|
||||
foo:
|
||||
description: b
|
||||
stage: c
|
||||
required_agent_capabilities: ["a"]
|
||||
flaky: not-a-boolean
|
||||
''',
|
||||
);
|
||||
|
||||
test('accepts boolean flaky option', () {
|
||||
final Manifest manifest = loadTaskManifest('''
|
||||
tasks:
|
||||
flaky_task:
|
||||
description: d
|
||||
stage: s
|
||||
required_agent_capabilities: ["c"]
|
||||
flaky: true
|
||||
''');
|
||||
|
||||
expect(manifest.tasks.single.name, 'flaky_task');
|
||||
expect(manifest.tasks.single.isFlaky, isTrue);
|
||||
});
|
||||
|
||||
test('accepts custom timeout_in_minutes option', () {
|
||||
final Manifest manifest = loadTaskManifest('''
|
||||
tasks:
|
||||
task_with_custom_timeout:
|
||||
description: d
|
||||
stage: s
|
||||
required_agent_capabilities: ["c"]
|
||||
timeout_in_minutes: 120
|
||||
''');
|
||||
|
||||
expect(manifest.tasks.single.timeoutInMinutes, 120);
|
||||
});
|
||||
});
|
||||
}
|
||||
@ -863,6 +863,48 @@
|
||||
"repo": "flutter",
|
||||
"task_name": "win_web_tool_tests",
|
||||
"flaky": true
|
||||
},
|
||||
{
|
||||
"name": "Windows_android complex_layout_win__compile",
|
||||
"repo": "flutter",
|
||||
"task_name": "complex_layout_win__compile",
|
||||
"flaky": false
|
||||
},
|
||||
{
|
||||
"name": "Windows_android basic_material_app_win__compile",
|
||||
"repo": "flutter",
|
||||
"task_name": "basic_material_app_win__compile",
|
||||
"flaky": false
|
||||
},
|
||||
{
|
||||
"name": "Windows_android flutter_gallery_win__compile",
|
||||
"repo": "flutter",
|
||||
"task_name": "flutter_gallery_win__compile",
|
||||
"flaky": false
|
||||
},
|
||||
{
|
||||
"name": "Windows_android windows_chrome_dev_mode",
|
||||
"repo": "flutter",
|
||||
"task_name": "windows_chrome_dev_mode",
|
||||
"flaky": false
|
||||
},
|
||||
{
|
||||
"name": "Windows_android flavors_test_win",
|
||||
"repo": "flutter",
|
||||
"task_name": "flavors_test_win",
|
||||
"flaky": false
|
||||
},
|
||||
{
|
||||
"name": "Windows_android channels_integration_test_win",
|
||||
"repo": "flutter",
|
||||
"task_name": "channels_integration_test_win",
|
||||
"flaky": false
|
||||
},
|
||||
{
|
||||
"name": "Windows_android hot_mode_dev_cycle_win__benchmark",
|
||||
"repo": "flutter",
|
||||
"task_name": "hot_mode_dev_cycle_win__benchmark",
|
||||
"flaky": false
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user