DEV/flutter_flutter
1
0
Fork 0
mirror of https://github.com/flutter/flutter.git synced 2026-02-20 02:29:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

[web] delete e2etests and all related tooling (flutter/engine#27299)

Browse Source 
* [web] delete e2etests and all related tooling
This commit is contained in:
Yegor 2021-07-12 12:16:31 -07:00 committed by GitHub
parent 9602957de7
commit 6dfbdc3f05
14 changed files with 10 additions and 1111 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
engine/src/flutter/.ci.yaml
View File

@ -53,7 +53,6 @@ targets:
- DEPS
- lib/web_ui/**
- web_sdk/**
- e2etests/web/**
- tools/**
- ci/**
- flutter_frontend_server/**
@ -65,7 +64,6 @@ targets:
- DEPS
- lib/web_ui/**
- web_sdk/**
- e2etests/web/**
- tools/**
- ci/**
- flutter_frontend_server/**
@ -97,7 +95,6 @@ targets:
- DEPS
- lib/web_ui/**
- web_sdk/**
- e2etests/web/**
- tools/**
- ci/**
- flutter_frontend_server/**

2
engine/src/flutter/ci/licenses_golden/tool_signature
View File

@ -1,2 +1,2 @@
Signature: 6f6132591430df787d08771477f3374d
Signature: affc80a35fe2dc9647bf293dd8afcaa7

6
engine/src/flutter/lib/web_ui/dev/README.md
View File

@ -163,7 +163,9 @@ _**Note**: if `FELT_USE_SNAPSHOT` is omitted or has any value other than "false"
## Upgrade Browser Version
Since the engine code and infra recipes do not live in the same repository there are few steps to follow in order to upgrade a browser's version. For now these instructins are most relevant to Chrome.
Since the engine code and infra recipes do not live in the same repository
there are few steps to follow in order to upgrade a browser's version. For
now these instructins are most relevant to Chrome.
1. Dowload the binaries for the new browser/driver for each operaing system (macOS, linux, windows).
2. Create CIPD packages for these packages. (More documentation is available for Googlers. go/cipd-flutter-web)
@ -174,7 +176,7 @@ Note that for LUCI builders, for Chrome both unit and integration tests are usin
Some useful links:
1. For Chrome downloads [link](https://commondatastorage.googleapis.com/chromium-browser-snapshots/index.html)
2. Browser and driver CIPD [packages](https://chrome-infra-packages.appspot.com/p/flutter_internal) (Note: Access rights are restricted for these packages.)
2. Browser and driver CIPD [packages](https://chrome-infra-packages.appspot.com/p/flutter_internal) (note: access is restricted for these packages)
3. LUCI web [recipe](https://flutter.googlesource.com/recipes/+/refs/heads/master/recipes/web_engine.py)
4. More general reading on CIPD packages [link](https://chromium.googlesource.com/chromium/src.git/+/master/docs/cipd.md#What-is-CIPD)

22
engine/src/flutter/lib/web_ui/dev/browser_lock.yaml
View File

@ -1,32 +1,16 @@
## Driver version in use.
## For an integration test to run, the browser's major version and the driver's
## major version should be equal. Please make sure the major version of
## the binary for `chrome` is the same with `required_driver_version`.
## (Major version meaning: For a browser that has version 13.0.5, the major
## version is 13.)
## Please refer to README's `Upgrade Browser Version` section for more details
## on how to update the version number.
# Please refer to the "Upgrade Browser Version" section in the README.md for
# more details on how to update browser version numbers.
chrome:
# It seems Chrome can't always release from the same build for all operating
# systems, so we specify per-OS build number.
Linux: 768968 # Major version 84
Mac: 768985 # Major version 84
Win: 768975 # Major version 84
required_driver_version:
## Make sure the major version of the binary in `browser_lock.yaml` is the
## same for Chrome.
chrome: '84'
## Firefox does not use CIPD. To update the version, simply update it in this
## file. However geckodriver does use CIPD. When upgrading geckodriver, update
## it here, then follow the process in go/cipd-flutter-web (googlers only) to
## create a new CIPD package.
## file.
firefox:
version: '83.0'
## geckodriver is used for testing Firefox Browser. It works with multiple
## Firefox Browser versions.
## See: https://github.com/mozilla/geckodriver/releases
geckodriver: 'v0.28.0'
edge:
launcher_version: '1.2.0.0'

64
engine/src/flutter/lib/web_ui/dev/chrome_installer.dart
View File

@ -279,18 +279,6 @@ Future<String> fetchLatestChromeVersion() async {
}
}
/// Get the Chrome Driver version for the system Chrome.
// TODO(nurhan): https://github.com/flutter/flutter/issues/53179
Future<String> queryChromeDriverVersion() async {
final int chromeVersion = await _querySystemChromeMajorVersion();
final io.File lockFile = io.File(
path.join(environment.webUiRootDir.path, 'dev', 'driver_version.yaml'));
YamlMap _configuration = loadYaml(lockFile.readAsStringSync()) as YamlMap;
final String chromeDriverVersion =
_configuration['chrome'][chromeVersion] as String;
return chromeDriverVersion;
}
/// Make sure LUCI bot has the pinned Chrome version and return the executable.
///
/// We are using CIPD packages in LUCI. The pinned chrome version from the
@ -311,55 +299,3 @@ String preinstalledChromeExecutable() {
);
}
}
Future<int> _querySystemChromeMajorVersion() async {
String chromeExecutable = '';
// LUCI uses the Chrome from CIPD packages.
if (isLuci) {
chromeExecutable = preinstalledChromeExecutable();
} else if (io.Platform.isLinux) {
chromeExecutable = 'google-chrome';
} else if (io.Platform.isMacOS) {
chromeExecutable = await _findChromeExecutableOnMac();
} else {
throw UnimplementedError('Web installers only work on Linux and Mac.');
}
final io.ProcessResult versionResult =
await io.Process.run('$chromeExecutable', <String>['--version']);
if (versionResult.exitCode != 0) {
throw Exception('Failed to locate system Chrome.');
}
// The output looks like: Google Chrome 79.0.3945.36.
final String output = versionResult.stdout as String;
print('INFO: chrome version in use $output');
// Version number such as 79.0.3945.36.
try {
final String versionAsString = output.trim().split(' ').last;
final String majorVersion = versionAsString.split('.')[0];
return int.parse(majorVersion);
} catch (e) {
throw Exception(
'Was expecting a version of the form Google Chrome 79.0.3945.36., '
'received $output');
}
}
/// Find Google Chrome App on Mac.
Future<String> _findChromeExecutableOnMac() async {
io.Directory chromeDirectory = io.Directory('/Applications')
.listSync()
.whereType<io.Directory>()
.firstWhere(
(d) => path.basename(d.path).endsWith('Chrome.app'),
orElse: () => throw Exception('Failed to locate system Chrome'),
);
final io.File chromeExecutableDir = io.File(
path.join(chromeDirectory.path, 'Contents', 'MacOS', 'Google Chrome'));
return chromeExecutableDir.path;
}

236
engine/src/flutter/lib/web_ui/dev/driver_manager.dart
View File

@ -1,236 +0,0 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:io' as io;
import 'package:path/path.dart' as pathlib;
import 'package:web_driver_installer/chrome_driver_installer.dart';
import 'package:web_driver_installer/firefox_driver_installer.dart';
import 'package:web_driver_installer/safari_driver_runner.dart';
import 'package:yaml/yaml.dart';
import 'chrome_installer.dart';
import 'common.dart';
import 'environment.dart';
import 'utils.dart';
/// [DriverManager] implementation for Chrome.
///
/// This manager can be used for both macOS and Linux.
class ChromeDriverManager extends DriverManager {
/// Directory which contains the Chrome's major version.
///
/// On LUCI we are using the CIPD packages to control Chrome binaries we use.
/// There will be multiple CIPD packages loaded at the same time. Yaml file
/// `driver_version.yaml` contains the version number we want to use.
///
/// Initialized to the current first to avoid the `Non-nullable` error.
// TODO: https://github.com/flutter/flutter/issues/53179. Local integration
// tests are still using the system Chrome.
late final io.Directory _browserDriverDirWithVersion;
ChromeDriverManager(String browser) : super(browser) {
final io.File lockFile = io.File(pathlib.join(
environment.webUiRootDir.path, 'dev', 'browser_lock.yaml'));
final YamlMap _configuration =
loadYaml(lockFile.readAsStringSync()) as YamlMap;
final String requiredChromeDriverVersion =
_configuration['required_driver_version']['chrome'] as String;
print('INFO: Major version for Chrome Driver $requiredChromeDriverVersion');
_browserDriverDirWithVersion = io.Directory(pathlib.join(
environment.webUiDartToolDir.path,
'drivers',
browser,
requiredChromeDriverVersion,
'${browser}driver-${io.Platform.operatingSystem.toString()}'));
}
@override
Future<void> _installDriver() async {
if (_browserDriverDirWithVersion.existsSync()) {
_browserDriverDirWithVersion.deleteSync(recursive: true);
}
_browserDriverDirWithVersion.createSync(recursive: true);
temporaryDirectories.add(_drivers);
final io.Directory temp = io.Directory.current;
io.Directory.current = _browserDriverDirWithVersion;
try {
// TODO(nurhan): https://github.com/flutter/flutter/issues/53179
final String chromeDriverVersion = await queryChromeDriverVersion();
ChromeDriverInstaller chromeDriverInstaller =
ChromeDriverInstaller.withVersion(chromeDriverVersion);
await chromeDriverInstaller.install(alwaysInstall: true);
} finally {
io.Directory.current = temp;
}
}
/// Throw an error if driver directory does not exists.
///
/// Driver should already exist on LUCI as a CIPD package.
@override
Future<void> _verifyDriverForLUCI() {
if (!_browserDriverDirWithVersion.existsSync()) {
throw StateError('Failed to locate Chrome driver on LUCI on path:'
'${_browserDriverDirWithVersion.path}');
}
return Future<void>.value();
}
@override
Future<void> _startDriver() async {
await startProcess('./chromedriver/chromedriver', ['--port=4444'],
workingDirectory: _browserDriverDirWithVersion.path);
print('INFO: Driver started');
}
}
/// [DriverManager] implementation for Firefox.
///
/// This manager can be used for both macOS and Linux.
class FirefoxDriverManager extends DriverManager {
FirefoxDriverManager(String browser) : super(browser);
FirefoxDriverInstaller firefoxDriverInstaller =
FirefoxDriverInstaller(geckoDriverVersion: getLockedGeckoDriverVersion());
@override
Future<void> _installDriver() async {
if (_browserDriverDir.existsSync()) {
_browserDriverDir.deleteSync(recursive: true);
}
_browserDriverDir.createSync(recursive: true);
temporaryDirectories.add(_drivers);
final io.Directory temp = io.Directory.current;
io.Directory.current = _browserDriverDir;
try {
await firefoxDriverInstaller.install(alwaysInstall: false);
} finally {
io.Directory.current = temp;
}
}
/// Throw an error if driver directory does not exist.
///
/// Driver should already exist on LUCI as a CIPD package.
@override
Future<void> _verifyDriverForLUCI() {
if (!_browserDriverDir.existsSync()) {
throw StateError('Failed to locate Firefox driver on LUCI on path:'
'${_browserDriverDir.path}');
}
return Future<void>.value();
}
@override
Future<void> _startDriver() async {
await startProcess('./firefoxdriver/geckodriver', ['--port=4444'],
workingDirectory: _browserDriverDir.path);
print('INFO: Driver started');
}
/// Get the geckodriver version to be used with [FirefoxDriverInstaller].
///
/// For different versions of geckodriver. See:
/// https://github.com/mozilla/geckodriver/releases
static String getLockedGeckoDriverVersion() {
final YamlMap browserLock = BrowserLock.instance.configuration;
String geckoDriverReleaseVersion = browserLock['geckodriver'] as String;
return geckoDriverReleaseVersion;
}
}
/// [DriverManager] implementation for Safari.
///
/// This manager is will only be created/used for macOS.
class SafariDriverManager extends DriverManager {
SafariDriverManager(String browser) : super(browser);
@override
Future<void> _installDriver() {
// No-op.
// macOS comes with Safari Driver installed.
return new Future<void>.value();
}
@override
Future<void> _verifyDriverForLUCI() {
// No-op.
// macOS comes with Safari Driver installed.
return Future<void>.value();
}
@override
Future<void> _startDriver() async {
final SafariDriverRunner safariDriverRunner = SafariDriverRunner();
final io.Process process =
await safariDriverRunner.runDriver(version: 'system');
processesToCleanUp.add(process);
}
}
/// Abstract class for preparing the browser driver before running the integration
/// tests.
abstract class DriverManager {
/// Installation directory for browser's driver.
final io.Directory _browserDriverDir;
/// This is the parent directory for all drivers.
///
/// This directory is saved to [temporaryDirectories] and deleted before
/// tests shutdown.
final io.Directory _drivers;
DriverManager(String browser)
: this._browserDriverDir = io.Directory(pathlib.join(
environment.webUiDartToolDir.path,
'drivers',
browser,
'${browser}driver-${io.Platform.operatingSystem.toString()}')),
this._drivers = io.Directory(
pathlib.join(environment.webUiDartToolDir.path, 'drivers'));
Future<void> prepareDriver() async {
if (!isLuci) {
// LUCI installs driver from CIPD, so we skip installing it on LUCI.
await _installDriver();
} else {
await _verifyDriverForLUCI();
}
await _startDriver();
}
/// Always re-install since driver can change frequently.
/// It usually changes with each the browser version changes.
/// A better solution would be installing the browser and the driver at the
/// same time.
/// TODO(nurhan): https://github.com/flutter/flutter/issues/53179. Partly
// solved. Remaining local integration tests using the locked Chrome version.
Future<void> _installDriver();
Future<void> _verifyDriverForLUCI();
Future<void> _startDriver();
static DriverManager chooseDriver(String browser) {
if (browser == 'chrome') {
return ChromeDriverManager(browser);
} else if (browser == 'firefox') {
return FirefoxDriverManager(browser);
} else if (browser == 'safari' && io.Platform.isMacOS) {
return SafariDriverManager(browser);
} else {
throw StateError('Integration tests are only supported on Firefox, Chrome'
' and on Safari (running on macOS)');
}
}
}

15
engine/src/flutter/lib/web_ui/dev/driver_version.yaml
View File

@ -1,15 +0,0 @@
## Map for driver versions to use for each browser version.
## See: https://chromedriver.chromium.org/downloads
chrome:
86: '86.0.4240.22'
85: '85.0.4183.38'
84: '84.0.4147.30'
83: '83.0.4103.39'
81: '81.0.4044.69'
80: '80.0.3987.106'
79: '79.0.3945.36'
78: '78.0.3904.105'
77: '77.0.3865.40'
76: '76.0.3809.126'
75: '75.0.3770.140'
74: '74.0.3729.6'

7
engine/src/flutter/lib/web_ui/dev/environment.dart
View File

@ -29,8 +29,6 @@ class Environment {
io.Directory(pathlib.join(hostDebugUnoptDir.path, 'dart-sdk'));
final io.Directory webUiRootDir = io.Directory(
pathlib.join(engineSrcDir.path, 'flutter', 'lib', 'web_ui'));
final io.Directory integrationTestsDir = io.Directory(
pathlib.join(engineSrcDir.path, 'flutter', 'e2etests', 'web'));
for (io.Directory expectedDirectory in <io.Directory>[
engineSrcDir,
@ -49,7 +47,6 @@ class Environment {
webUiRootDir: webUiRootDir,
engineSrcDir: engineSrcDir,
engineToolsDir: engineToolsDir,
integrationTestsDir: integrationTestsDir,
outDir: outDir,
hostDebugUnoptDir: hostDebugUnoptDir,
dartSdkDir: dartSdkDir,
@ -61,7 +58,6 @@ class Environment {
required this.webUiRootDir,
required this.engineSrcDir,
required this.engineToolsDir,
required this.integrationTestsDir,
required this.outDir,
required this.hostDebugUnoptDir,
required this.dartSdkDir,
@ -79,9 +75,6 @@ class Environment {
/// Path to the engine's "tools" directory.
final io.Directory engineToolsDir;
/// Path to the web integration tests.
final io.Directory integrationTestsDir;
/// Path to the engine's "out" directory.
///
/// This is where you'll find the ninja output, such as the Dart SDK.

9
engine/src/flutter/lib/web_ui/dev/exceptions.dart
View File

@ -11,15 +11,6 @@ class BrowserInstallerException implements Exception {
String toString() => message;
}
class DriverException implements Exception {
DriverException(this.message);
final String message;
@override
String toString() => message;
}
class ToolException implements Exception {
ToolException(this.message);

658
engine/src/flutter/lib/web_ui/dev/integration_tests_manager.dart
View File

@ -1,658 +0,0 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'dart:io' as io;
import 'package:args/args.dart';
import 'package:path/path.dart' as pathlib;
import 'chrome_installer.dart';
import 'driver_manager.dart';
import 'environment.dart';
import 'exceptions.dart';
import 'common.dart';
import 'utils.dart';
const String _unsupportedConfigurationWarning = 'WARNING: integration tests '
'are only supported on Chrome, Firefox and on Safari (running on macOS)';
class IntegrationTestsManager {
final String _browser;
final bool _useSystemFlutter;
final DriverManager _driverManager;
final bool _doUpdateScreenshotGoldens;
IntegrationTestsManager(
this._browser, this._useSystemFlutter, this._doUpdateScreenshotGoldens)
: _driverManager = DriverManager.chooseDriver(_browser);
Future<bool> runTests() async {
if (validateIfTestsShouldRun()) {
await _driverManager.prepareDriver();
return await _runTests();
} else {
return false;
}
}
Future<void> _runPubGet(String workingDirectory) async {
if (!_useSystemFlutter) {
await _cloneFlutterRepo();
await _enableWeb(workingDirectory);
}
await runFlutter(workingDirectory, <String>['pub', 'get'],
useSystemFlutter: _useSystemFlutter);
}
/// Clone flutter repository, use the youngest commit older than the engine
/// commit.
///
/// Use engine/src/flutter/.dart_tools to clone the Flutter repo.
/// TODO(nurhan): Use git pull instead if repo exists.
Future<void> _cloneFlutterRepo() async {
// Delete directory if exists.
if (environment.engineDartToolDir.existsSync()) {
environment.engineDartToolDir.deleteSync(recursive: true);
}
environment.engineDartToolDir.createSync();
final int exitCode = await runProcess(
environment.cloneFlutterScript.path,
<String>[
environment.engineDartToolDir.path,
],
workingDirectory: environment.webUiRootDir.path,
);
if (exitCode != 0) {
throw ToolException('ERROR: Failed to clone flutter repo. Exited with '
'exit code $exitCode');
}
}
Future<void> _enableWeb(String workingDirectory) async {
await runFlutter(workingDirectory, <String>['config', '--enable-web'],
useSystemFlutter: _useSystemFlutter);
}
/// Runs all the web tests under e2e_tests/web.
Future<bool> _runTests() async {
// Only list the files under e2e_tests/web.
final List<io.FileSystemEntity> entities =
environment.integrationTestsDir.listSync(followLinks: false);
bool allTestsPassed = true;
for (io.FileSystemEntity e in entities) {
// The tests should be under this directories.
if (e is io.Directory) {
allTestsPassed = allTestsPassed && await _validateAndRunTests(e);
}
}
return allTestsPassed;
}
/// Run tests in a single directory under: e2e_tests/web.
///
/// Run `flutter pub get` as the first step.
///
/// Validate the directory before running the tests. Each directory is
/// expected to be a test project which includes a `pubspec.yaml` file
/// and a `test_driver` directory.
Future<bool> _validateAndRunTests(io.Directory directory) async {
_validateTestDirectory(directory);
await _runPubGet(directory.path);
final bool testResults = await _runTestsInDirectory(directory);
return testResults;
}
int _numberOfPassedTests = 0;
int _numberOfFailedTests = 0;
Future<bool> _runTestsInDirectory(io.Directory directory) async {
final io.Directory testDirectory =
io.Directory(pathlib.join(directory.path, 'test_driver'));
final List<io.File> entities = testDirectory
.listSync(followLinks: false)
.whereType<io.File>()
.toList();
final List<String> e2eTestsToRun = <String>[];
final List<String> blockedTests =
blockedTestsListsMap[getBlockedTestsListMapKey(_browser)] ?? <String>[];
// If no target is specified run all the tests.
if (_runAllTestTargets) {
// The following loops over the contents of the directory and saves an
// expected driver file name for each e2e test assuming any dart file
// not ending with `_test.dart` is an e2e test.
// Other files are not considered since developers can add files such as
// README.
for (io.File f in entities) {
final String basename = pathlib.basename(f.path);
if (!basename.contains('_test.dart') && basename.endsWith('.dart')) {
// Do not add the basename if it is in the `blockedTests`.
if (!blockedTests.contains(basename)) {
e2eTestsToRun.add(basename);
} else {
print('INFO: Test $basename is skipped since it is blocked for '
'${getBlockedTestsListMapKey(_browser)}');
}
}
}
if (isVerboseLoggingEnabled) {
print(
'INFO: In project ${directory} ${e2eTestsToRun.length} tests to run.');
}
} else {
// If a target is specified it will run regardless of if it's blocked or
// not. There will be an info note to warn the developer.
final String targetTest =
IntegrationTestsArgumentParser.instance.testTarget;
final io.File file =
entities.singleWhere((f) => pathlib.basename(f.path) == targetTest);
final String basename = pathlib.basename(file.path);
if (blockedTests.contains(basename) && isVerboseLoggingEnabled) {
print('INFO: Test $basename do not run on CI environments. Please '
'remove it from the blocked tests list if you want to enable this '
'test on CI.');
}
e2eTestsToRun.add(basename);
}
final Set<String> buildModes = _getBuildModes();
for (String fileName in e2eTestsToRun) {
await _runTestsTarget(directory, fileName, buildModes);
}
final int numberOfTestsRun = _numberOfPassedTests + _numberOfFailedTests;
print('INFO: ${numberOfTestsRun} tests run. ${_numberOfPassedTests} passed '
'and ${_numberOfFailedTests} failed.');
return _numberOfFailedTests == 0;
}
Future<void> _runTestsTarget(
io.Directory directory, String target, Set<String> buildModes) async {
final Set<String> renderingBackends = _getRenderingBackends();
for (String renderingBackend in renderingBackends) {
for (String mode in buildModes) {
if (!blockedTestsListsMapForModes[mode]!.contains(target) &&
!blockedTestsListsMapForRenderBackends[renderingBackend]!
.contains(target)) {
final bool result = await _runTestsInMode(directory, target,
mode: mode, webRenderer: renderingBackend);
if (result) {
_numberOfPassedTests++;
} else {
_numberOfFailedTests++;
}
}
}
}
}
Future<bool> _runTestsInMode(io.Directory directory, String testName,
{String mode = 'profile', String webRenderer = 'html'}) async {
String executable =
_useSystemFlutter ? 'flutter' : environment.flutterCommand.path;
Map<String, String> flutterEnvironment = Map<String, String>();
if (_doUpdateScreenshotGoldens) {
flutterEnvironment['UPDATE_GOLDENS'] = 'true';
}
final IntegrationArguments arguments =
IntegrationArguments.fromBrowser(_browser);
final int exitCode = await runProcess(
executable,
arguments.getTestArguments(testName, mode, webRenderer),
workingDirectory: directory.path,
environment: flutterEnvironment,
);
if (exitCode != 0) {
final String command =
arguments.getCommandToRun(testName, mode, webRenderer);
io.stderr
.writeln('ERROR: Failed to run test. Exited with exit code $exitCode'
'. To run $testName locally use the following command:'
'\n\n$command');
return false;
} else {
return true;
}
}
Set<String> _getRenderingBackends() {
Set<String> renderingBackends;
if (_renderingBackendSelected) {
final String mode = IntegrationTestsArgumentParser.instance.webRenderer;
renderingBackends = <String>{mode};
} else {
// TODO(nurhan): Enable `auto` when recipe is sharded.
renderingBackends = {'html', 'canvaskit'};
}
return renderingBackends;
}
Set<String> _getBuildModes() {
Set<String> buildModes;
if (_buildModeSelected) {
final String mode = IntegrationTestsArgumentParser.instance.buildMode;
if (mode == 'debug' && _browser != 'chrome') {
throw ToolException('Debug mode is only supported for Chrome.');
} else {
buildModes = <String>{mode};
}
} else {
buildModes = _browser == 'chrome'
? {'debug', 'profile', 'release'}
: {'profile', 'release'};
}
return buildModes;
}
/// Validate the directory has a `pubspec.yaml` file and a `test_driver`
/// directory.
///
/// Also check the validity of files under `test_driver` directory calling
/// [_checkE2ETestsValidity] method.
void _validateTestDirectory(io.Directory directory) {
final List<io.FileSystemEntity> entities =
directory.listSync(followLinks: false);
// Whether the project has the pubspec.yaml file.
bool pubSpecFound = false;
// The test directory 'test_driver'.
io.Directory? testDirectory;
for (io.FileSystemEntity e in entities) {
// The tests should be under this directories.
final String baseName = pathlib.basename(e.path);
if (e is io.Directory && baseName == 'test_driver') {
testDirectory = e;
}
if (e is io.File && baseName == 'pubspec.yaml') {
pubSpecFound = true;
}
}
if (!pubSpecFound) {
throw StateError('ERROR: pubspec.yaml file not found in the test project '
'in the directory ${directory.path}.');
}
if (testDirectory == null) {
throw StateError(
'ERROR: test_driver folder not found in the test project.'
'in the directory ${directory.path}.');
} else {
_checkE2ETestsValidity(testDirectory);
}
}
/// Checks if each e2e test file in the directory has a driver test
/// file to run it.
///
/// Prints informative message to the developer if an error has found.
/// For each e2e test which has name {name}.dart there will be a driver
/// file which drives it. The driver file should be named:
/// {name}_test.dart
void _checkE2ETestsValidity(io.Directory testDirectory) {
final Iterable<io.Directory> directories =
testDirectory.listSync(followLinks: false).whereType<io.Directory>();
if (directories.length > 0) {
throw StateError('${testDirectory.path} directory should not contain '
'any sub-directories');
}
final Iterable<io.File> entities =
testDirectory.listSync(followLinks: false).whereType<io.File>();
final Set<String> expectedDriverFileNames = Set<String>();
final Set<String> foundDriverFileNames = Set<String>();
int numberOfTests = 0;
// The following loops over the contents of the directory and saves an
// expected driver file name for each e2e test assuming any file
// not ending with `_test.dart` is an e2e test.
for (io.File f in entities) {
final String basename = pathlib.basename(f.path);
if (basename.contains('_test.dart')) {
// First remove this from expectedSet if not there add to the foundSet.
if (!expectedDriverFileNames.remove(basename)) {
foundDriverFileNames.add(basename);
}
} else if (basename.contains('.dart')) {
// Only run on dart files.
final String e2efileName = pathlib.basenameWithoutExtension(f.path);
final String expectedDriverName = '${e2efileName}_test.dart';
numberOfTests++;
// First remove this from foundSet if not there add to the expectedSet.
if (!foundDriverFileNames.remove(expectedDriverName)) {
expectedDriverFileNames.add(expectedDriverName);
}
}
}
if (numberOfTests == 0) {
throw StateError(
'WARNING: No tests to run in this directory ${testDirectory.path}');
}
// TODO(nurhan): In order to reduce the work required from team members,
// remove the need for driver file, by using the same template file.
// Some driver files are missing.
if (expectedDriverFileNames.length > 0) {
for (String expectedDriverName in expectedDriverFileNames) {
print('ERROR: Test driver file named has ${expectedDriverName} '
'not found under directory ${testDirectory.path}. Stopping the '
'integration tests. Please add ${expectedDriverName}. Check to '
'README file on more details on how to set up integration tests.');
}
throw StateError('Error in test files. Check the logs for '
'further instructions');
}
}
bool get _buildModeSelected =>
!IntegrationTestsArgumentParser.instance.buildMode.isEmpty;
bool get _renderingBackendSelected =>
!IntegrationTestsArgumentParser.instance.webRenderer.isEmpty;
bool get _runAllTestTargets =>
IntegrationTestsArgumentParser.instance.testTarget.isEmpty;
/// Validate the given `browser`, `platform` combination is suitable for
/// integration tests to run.
bool validateIfTestsShouldRun() {
if (_buildModeSelected) {
final String mode = IntegrationTestsArgumentParser.instance.buildMode;
if (mode == 'debug' && _browser != 'chrome') {
throw ToolException('Debug mode is only supported for Chrome.');
}
}
// Chrome tests should run at all Platforms (Linux, macOS, Windows).
// They can also run successfully on CI and local.
if (_browser == 'chrome') {
return true;
} else if (_browser == 'firefox' &&
(io.Platform.isLinux || io.Platform.isMacOS)) {
return true;
} else if (_browser == 'safari' && io.Platform.isMacOS && !isLuci) {
return true;
} else {
io.stderr.writeln(_unsupportedConfigurationWarning);
return false;
}
}
}
/// Interface for collecting arguments to give `flutter drive` to run the
/// integration tests.
abstract class IntegrationArguments {
IntegrationArguments();
factory IntegrationArguments.fromBrowser(String browser) {
if (browser == 'chrome') {
return ChromeIntegrationArguments();
} else if (browser == 'firefox') {
return FirefoxIntegrationArguments();
} else if (browser == 'safari' && io.Platform.isMacOS) {
return SafariIntegrationArguments();
} else {
throw StateError(_unsupportedConfigurationWarning);
}
}
List<String> getTestArguments(
String testName, String mode, String webRenderer);
String getCommandToRun(String testName, String mode, String webRenderer);
}
/// Arguments to give `flutter drive` to run the integration tests on Chrome.
class ChromeIntegrationArguments extends IntegrationArguments {
List<String> getTestArguments(
String testName, String mode, String webRenderer) {
return <String>[
'drive',
'--target=test_driver/${testName}',
'-d',
'web-server',
'--$mode',
'--browser-name=chrome',
if (isLuci) '--chrome-binary=${preinstalledChromeExecutable()}',
'--headless',
'--local-engine=host_debug_unopt',
'--web-renderer=$webRenderer',
];
}
String getCommandToRun(String testName, String mode, String webRenderer) {
String statementToRun = 'flutter drive '
'--target=test_driver/${testName} -d web-server --$mode '
'--browser-name=chrome --local-engine=host_debug_unopt '
'--web-renderer=$webRenderer';
if (isLuci) {
statementToRun = '$statementToRun --chrome-binary='
'${preinstalledChromeExecutable()}';
}
return statementToRun;
}
}
/// Arguments to give `flutter drive` to run the integration tests on Firefox.
class FirefoxIntegrationArguments extends IntegrationArguments {
List<String> getTestArguments(
String testName, String mode, String webRenderer) {
return <String>[
'drive',
'--target=test_driver/${testName}',
'-d',
'web-server',
'--$mode',
'--browser-name=firefox',
'--headless',
'--local-engine=host_debug_unopt',
'--web-renderer=$webRenderer',
];
}
String getCommandToRun(String testName, String mode, String webRenderer) {
final String arguments =
getTestArguments(testName, mode, webRenderer).join(' ');
return 'flutter $arguments';
}
}
/// Arguments to give `flutter drive` to run the integration tests on Safari.
class SafariIntegrationArguments extends IntegrationArguments {
SafariIntegrationArguments();
List<String> getTestArguments(
String testName, String mode, String webRenderer) {
return <String>[
'drive',
'--target=test_driver/${testName}',
'-d',
'web-server',
'--$mode',
'--browser-name=safari',
'--local-engine=host_debug_unopt',
'--web-renderer=$webRenderer',
];
}
String getCommandToRun(String testName, String mode, String webRenderer) {
final String arguments =
getTestArguments(testName, mode, webRenderer).join(' ');
return 'flutter $arguments';
}
}
/// Parses additional options that can be used when running integration tests.
class IntegrationTestsArgumentParser {
static final IntegrationTestsArgumentParser _singletonInstance =
IntegrationTestsArgumentParser._();
/// The [IntegrationTestsArgumentParser] singleton.
static IntegrationTestsArgumentParser get instance => _singletonInstance;
IntegrationTestsArgumentParser._();
/// If target name is provided integration tests can run that one test
/// instead of running all the tests.
late final String testTarget;
/// The build mode to run the integration tests.
///
/// If not specified, these tests will run using 'debug, profile, release'
/// modes on Chrome and will run using 'profile, release' on other browsers.
///
/// In order to skip a test for one of the modes, add the test to the
/// `blockedTestsListsMapForModes` list for the relevant compile mode.
late final String buildMode;
/// Whether to use html, canvaskit or auto for web renderer.
///
/// If not set all backends will be used one after another for integration
/// tests. If set only the provided option will be used.
late final String webRenderer;
void populateOptions(ArgParser argParser) {
argParser
..addOption(
'target',
defaultsTo: '',
help: 'By default integration tests are run for all the tests under'
'flutter/e2etests/web directory. If a test name is specified, that '
'only that test will run. The test name will be the name of the '
'integration test (e2e test) file. For example: '
'text_editing_integration.dart or '
'profile_diagnostics_integration.dart',
)
..addOption('build-mode',
defaultsTo: '',
help: 'Flutter supports three modes when building your app. This '
'option sets the build mode for the integration tests. '
'By default an integration test will sequentially run on '
'multiple modes. All three modes (debug, release, profile) are '
'used for Chrome. Only profile, release modes will be used for '
'other browsers. In other words, if a build mode is selected '
'tests will only be run using that mode. '
'See https://flutter.dev/docs/testing/build-modes for more '
'details on the build modes.')
..addOption('web-renderer',
defaultsTo: '',
help: 'By default all three options (`html`, `canvaskit`, `auto`) '
' for rendering backends are tested when running integration '
' tests. If this option is set only the backend provided by this '
' option will be used. `auto`, `canvaskit` and `html`'
' are the available options. ');
}
/// Populate results of the arguments passed.
void parseOptions(ArgResults argResults) {
testTarget = argResults['target'] as String;
buildMode = argResults['build-mode'] as String;
if (!buildMode.isEmpty &&
buildMode != 'debug' &&
buildMode != 'profile' &&
buildMode != 'release') {
throw ArgumentError('Unexpected build mode: $buildMode');
}
webRenderer = argResults['web-renderer'] as String;
if (!webRenderer.isEmpty &&
webRenderer != 'html' &&
webRenderer != 'canvaskit' &&
webRenderer != 'auto') {
throw ArgumentError('Unexpected rendering backend: $webRenderer');
}
}
}
/// Prepares a key for the [blackList] map.
///
/// Uses the browser name and the operating system name.
String getBlockedTestsListMapKey(String browser) =>
'${browser}-${io.Platform.operatingSystem}';
/// Tests that should be skipped run for a specific platform-browser
/// combination.
///
/// These tests might be failing or might have been implemented for a specific
/// configuration.
///
/// For example when adding a tests only intended for mobile browsers, it should
/// be added to [blockedTests] for `chrome-linux`, `safari-macos` and
/// `chrome-macos`. It will work on `chrome-android`, `safari-ios`.
///
/// Note that integration tests are only running on chrome for now.
const Map<String, List<String>> blockedTestsListsMap = <String, List<String>>{
'chrome-linux': [
'target_platform_android_integration.dart',
'target_platform_ios_integration.dart',
'target_platform_macos_integration.dart',
],
'chrome-macos': [
'target_platform_ios_integration.dart',
'target_platform_android_integration.dart',
],
'safari-macos': [
'target_platform_ios_integration.dart',
'target_platform_android_integration.dart',
'image_loading_integration.dart',
],
'firefox-linux': [
'target_platform_android_integration.dart',
'target_platform_ios_integration.dart',
'target_platform_macos_integration.dart',
],
'firefox-macos': [
'target_platform_android_integration.dart',
'target_platform_ios_integration.dart',
],
};
/// Tests blocked for one of the build modes.
///
/// If a test is not supposed to run for one of the modes also add that test
/// to the corresponding list.
// TODO(nurhan): Remove the failing test after fixing.
const Map<String, List<String>> blockedTestsListsMapForModes =
<String, List<String>>{
'debug': [
'treeshaking_integration.dart',
'text_editing_integration.dart',
'url_strategy_integration.dart',
],
'profile': [],
'release': [],
};
/// Tests blocked for one of the rendering backends.
///
/// If a test is not suppose to run for one of the backends also add that test
/// to the corresponding list.
// TODO(nurhan): Remove the failing test after fixing.
const Map<String, List<String>> blockedTestsListsMapForRenderBackends =
<String, List<String>>{
'auto': [
'image_loading_integration.dart',
'platform_messages_integration.dart',
'profile_diagnostics_integration.dart',
'scroll_wheel_integration.dart',
'text_editing_integration.dart',
'treeshaking_integration.dart',
'url_strategy_integration.dart',
],
'html': [],
// This test failed on canvaskit on all three build modes.
'canvaskit': [
'image_loading_integration.dart',
],
};

84
engine/src/flutter/lib/web_ui/dev/test_runner.dart
View File

@ -38,7 +38,6 @@ import 'environment.dart';
import 'exceptions.dart';
import 'firefox.dart';
import 'firefox_installer.dart';
import 'integration_tests_manager.dart';
import 'macos_info.dart';
import 'safari_installation.dart';
import 'safari_ios.dart';
@ -58,18 +57,6 @@ List<String> failedShards = <String>[];
/// Whether all test shards succeeded.
bool get allShardsPassed => failedShards.isEmpty;
/// The type of tests requested by the tool user.
enum TestTypesRequested {
/// For running the unit tests only.
unit,
/// For running the integration tests only.
integration,
/// For running both unit and integration tests.
all,
}
/// Command-line argument parsers that parse browser-specific options.
final List<BrowserArgParser> _browserArgParsers = <BrowserArgParser>[
ChromeArgParser.instance,
@ -116,13 +103,6 @@ class TestCommand extends Command<bool> with ArgUtils {
help: 'felt test command runs the unit tests and the integration tests '
'at the same time. If this flag is set, only run the unit tests.',
)
..addFlag(
'integration-tests-only',
defaultsTo: false,
help: 'felt test command runs the unit tests and the integration tests '
'at the same time. If this flag is set, only run the integration '
'tests.',
)
..addFlag('use-system-flutter',
defaultsTo: false,
help:
@ -170,7 +150,6 @@ class TestCommand extends Command<bool> with ArgUtils {
browserArgParser.populateOptions(argParser);
}
GeneralTestsArgumentParser.instance.populateOptions(argParser);
IntegrationTestsArgumentParser.instance.populateOptions(argParser);
}
@override
@ -193,25 +172,6 @@ class TestCommand extends Command<bool> with ArgUtils {
/// repeat them.
bool _testPreparationReady = false;
/// Check the flags to see what type of tests are requested.
TestTypesRequested get testType {
if (boolArg('unit-tests-only')! && boolArg('integration-tests-only')!) {
throw ArgumentError('Conflicting arguments: unit-tests-only and '
'integration-tests-only are both set');
} else if (boolArg('unit-tests-only')!) {
print('Running the unit tests only');
return TestTypesRequested.unit;
} else if (boolArg('integration-tests-only')!) {
if (!isChrome && !isSafariOnMacOS && !isFirefox) {
throw UnimplementedError(
'Integration tests are only available on Chrome Desktop for now');
}
return TestTypesRequested.integration;
} else {
return TestTypesRequested.all;
}
}
@override
Future<bool> run() async {
for (BrowserArgParser browserArgParser in _browserArgParsers) {
@ -277,46 +237,7 @@ class TestCommand extends Command<bool> with ArgUtils {
return message.toString();
}
Future<bool> runTests() async {
try {
switch (testType) {
case TestTypesRequested.unit:
return runUnitTests();
case TestTypesRequested.integration:
return runIntegrationTests();
case TestTypesRequested.all:
if (runAllTests && isIntegrationTestsAvailable) {
bool unitTestResult = await runUnitTests();
bool integrationTestResult = await runIntegrationTests();
if (integrationTestResult != unitTestResult) {
print(
'Tests run. Integration tests passed: $integrationTestResult '
'unit tests passed: $unitTestResult');
}
return integrationTestResult && unitTestResult;
} else {
return await runUnitTests();
}
}
} on TestFailureException {
return true;
}
}
Future<bool> runIntegrationTests() async {
// Parse additional arguments specific for integration testing.
IntegrationTestsArgumentParser.instance.parseOptions(argResults!);
await _prepare();
final bool result = await IntegrationTestsManager(
browser, useSystemFlutter, doUpdateScreenshotGoldens)
.runTests();
if (!result) {
failedShards.add('Integration tests');
}
return result;
}
Future<bool> runUnitTests() async {
Future<void> runUnitTests() async {
_copyTestFontsIntoWebUi();
await _buildHostPage();
await _prepare();
@ -327,7 +248,6 @@ class TestCommand extends Command<bool> with ArgUtils {
} else {
await _runSpecificTests(targetFiles);
}
return true;
}
/// Preparations before running the tests such as booting simulators or
@ -887,6 +807,6 @@ class TestRunnerStep implements PipelineStep {
@override
Future<void> run() async {
await testCommand.runTests();
await testCommand.runUnitTests();
}
}

5
engine/src/flutter/lib/web_ui/pubspec.yaml
View File

@ -28,8 +28,3 @@ dev_dependencies:
url: git://github.com/flutter/web_installers.git
path: packages/simulators/
ref: 4a7b0a2c84b8993bf4d19030218de38c18838c26
web_driver_installer:
git:
url: git://github.com/flutter/web_installers.git
path: packages/web_drivers/
ref: 4a7b0a2c84b8993bf4d19030218de38c18838c26

3
engine/src/flutter/tools/licenses/lib/main.dart
View File

@ -2122,9 +2122,6 @@ class _RepositoryFlutterDirectory extends _RepositoryDirectory {
&& entry.name != 'build'
&& entry.name != 'ci'
&& entry.name != 'flutter_frontend_server'
// The directory that contains end to end tests.
// Should be excluded from the licence checks.
&& entry.name != 'e2etests'
// None of the web_sdk code is linked into Flutter apps. It's only used
// by engine tests and tools.
&& entry.name != 'web_sdk'

7
engine/src/flutter/web_sdk/web_test_utils/lib/environment.dart
View File

@ -48,8 +48,6 @@ class Environment {
io.Directory(pathlib.join(hostDebugUnoptDir.path, 'dart-sdk'));
final io.Directory webUiRootDir = io.Directory(
pathlib.join(engineSrcDir.path, 'flutter', 'lib', 'web_ui'));
final io.Directory integrationTestsDir = io.Directory(
pathlib.join(engineSrcDir.path, 'flutter', 'e2etests', 'web'));
for (final io.Directory expectedDirectory in <io.Directory>[
engineSrcDir,
@ -68,7 +66,6 @@ class Environment {
webUiRootDir: webUiRootDir,
engineSrcDir: engineSrcDir,
engineToolsDir: engineToolsDir,
integrationTestsDir: integrationTestsDir,
outDir: outDir,
hostDebugUnoptDir: hostDebugUnoptDir,
dartSdkDir: dartSdkDir,
@ -80,7 +77,6 @@ class Environment {
required this.webUiRootDir,
required this.engineSrcDir,
required this.engineToolsDir,
required this.integrationTestsDir,
required this.outDir,
required this.hostDebugUnoptDir,
required this.dartSdkDir,
@ -98,9 +94,6 @@ class Environment {
/// Path to the engine's "tools" directory.
final io.Directory engineToolsDir;
/// Path to the web integration tests.
final io.Directory integrationTestsDir;
/// Path to the engine's "out" directory.
///
/// This is where you'll find the ninja output, such as the Dart SDK.