From 6dfbdc3f05912eb4759cbceec597fa21403ba475 Mon Sep 17 00:00:00 2001 From: Yegor Date: Mon, 12 Jul 2021 12:16:31 -0700 Subject: [PATCH] [web] delete e2etests and all related tooling (flutter/engine#27299) * [web] delete e2etests and all related tooling --- engine/src/flutter/.ci.yaml | 3 - .../flutter/ci/licenses_golden/tool_signature | 2 +- engine/src/flutter/lib/web_ui/dev/README.md | 6 +- .../flutter/lib/web_ui/dev/browser_lock.yaml | 22 +- .../lib/web_ui/dev/chrome_installer.dart | 64 -- .../lib/web_ui/dev/driver_manager.dart | 236 ------- .../lib/web_ui/dev/driver_version.yaml | 15 - .../flutter/lib/web_ui/dev/environment.dart | 7 - .../flutter/lib/web_ui/dev/exceptions.dart | 9 - .../web_ui/dev/integration_tests_manager.dart | 658 ------------------ .../flutter/lib/web_ui/dev/test_runner.dart | 84 +-- engine/src/flutter/lib/web_ui/pubspec.yaml | 5 - .../src/flutter/tools/licenses/lib/main.dart | 3 - .../web_test_utils/lib/environment.dart | 7 - 14 files changed, 10 insertions(+), 1111 deletions(-) delete mode 100644 engine/src/flutter/lib/web_ui/dev/driver_manager.dart delete mode 100644 engine/src/flutter/lib/web_ui/dev/driver_version.yaml delete mode 100644 engine/src/flutter/lib/web_ui/dev/integration_tests_manager.dart diff --git a/engine/src/flutter/.ci.yaml b/engine/src/flutter/.ci.yaml index 30df56c206f..5e7ccf172f8 100644 --- a/engine/src/flutter/.ci.yaml +++ b/engine/src/flutter/.ci.yaml @@ -53,7 +53,6 @@ targets: - DEPS - lib/web_ui/** - web_sdk/** - - e2etests/web/** - tools/** - ci/** - flutter_frontend_server/** @@ -65,7 +64,6 @@ targets: - DEPS - lib/web_ui/** - web_sdk/** - - e2etests/web/** - tools/** - ci/** - flutter_frontend_server/** @@ -97,7 +95,6 @@ targets: - DEPS - lib/web_ui/** - web_sdk/** - - e2etests/web/** - tools/** - ci/** - flutter_frontend_server/** diff --git a/engine/src/flutter/ci/licenses_golden/tool_signature b/engine/src/flutter/ci/licenses_golden/tool_signature index 2f87be53aa4..556caa7a653 100644 --- a/engine/src/flutter/ci/licenses_golden/tool_signature +++ b/engine/src/flutter/ci/licenses_golden/tool_signature @@ -1,2 +1,2 @@ -Signature: 6f6132591430df787d08771477f3374d +Signature: affc80a35fe2dc9647bf293dd8afcaa7 diff --git a/engine/src/flutter/lib/web_ui/dev/README.md b/engine/src/flutter/lib/web_ui/dev/README.md index 71fa5daaafc..edbb743df33 100644 --- a/engine/src/flutter/lib/web_ui/dev/README.md +++ b/engine/src/flutter/lib/web_ui/dev/README.md @@ -163,7 +163,9 @@ _**Note**: if `FELT_USE_SNAPSHOT` is omitted or has any value other than "false" ## Upgrade Browser Version -Since the engine code and infra recipes do not live in the same repository there are few steps to follow in order to upgrade a browser's version. For now these instructins are most relevant to Chrome. +Since the engine code and infra recipes do not live in the same repository +there are few steps to follow in order to upgrade a browser's version. For +now these instructins are most relevant to Chrome. 1. Dowload the binaries for the new browser/driver for each operaing system (macOS, linux, windows). 2. Create CIPD packages for these packages. (More documentation is available for Googlers. go/cipd-flutter-web) @@ -174,7 +176,7 @@ Note that for LUCI builders, for Chrome both unit and integration tests are usin Some useful links: 1. For Chrome downloads [link](https://commondatastorage.googleapis.com/chromium-browser-snapshots/index.html) -2. Browser and driver CIPD [packages](https://chrome-infra-packages.appspot.com/p/flutter_internal) (Note: Access rights are restricted for these packages.) +2. Browser and driver CIPD [packages](https://chrome-infra-packages.appspot.com/p/flutter_internal) (note: access is restricted for these packages) 3. LUCI web [recipe](https://flutter.googlesource.com/recipes/+/refs/heads/master/recipes/web_engine.py) 4. More general reading on CIPD packages [link](https://chromium.googlesource.com/chromium/src.git/+/master/docs/cipd.md#What-is-CIPD) diff --git a/engine/src/flutter/lib/web_ui/dev/browser_lock.yaml b/engine/src/flutter/lib/web_ui/dev/browser_lock.yaml index d5f9e5baeff..51baa3e5171 100644 --- a/engine/src/flutter/lib/web_ui/dev/browser_lock.yaml +++ b/engine/src/flutter/lib/web_ui/dev/browser_lock.yaml @@ -1,32 +1,16 @@ -## Driver version in use. -## For an integration test to run, the browser's major version and the driver's -## major version should be equal. Please make sure the major version of -## the binary for `chrome` is the same with `required_driver_version`. -## (Major version meaning: For a browser that has version 13.0.5, the major -## version is 13.) -## Please refer to README's `Upgrade Browser Version` section for more details -## on how to update the version number. +# Please refer to the "Upgrade Browser Version" section in the README.md for +# more details on how to update browser version numbers. chrome: # It seems Chrome can't always release from the same build for all operating # systems, so we specify per-OS build number. Linux: 768968 # Major version 84 Mac: 768985 # Major version 84 Win: 768975 # Major version 84 -required_driver_version: - ## Make sure the major version of the binary in `browser_lock.yaml` is the - ## same for Chrome. - chrome: '84' ## Firefox does not use CIPD. To update the version, simply update it in this -## file. However geckodriver does use CIPD. When upgrading geckodriver, update -## it here, then follow the process in go/cipd-flutter-web (googlers only) to -## create a new CIPD package. +## file. firefox: version: '83.0' -## geckodriver is used for testing Firefox Browser. It works with multiple -## Firefox Browser versions. -## See: https://github.com/mozilla/geckodriver/releases -geckodriver: 'v0.28.0' edge: launcher_version: '1.2.0.0' diff --git a/engine/src/flutter/lib/web_ui/dev/chrome_installer.dart b/engine/src/flutter/lib/web_ui/dev/chrome_installer.dart index 9a518d96ab6..d84e2c1c115 100644 --- a/engine/src/flutter/lib/web_ui/dev/chrome_installer.dart +++ b/engine/src/flutter/lib/web_ui/dev/chrome_installer.dart @@ -279,18 +279,6 @@ Future fetchLatestChromeVersion() async { } } -/// Get the Chrome Driver version for the system Chrome. -// TODO(nurhan): https://github.com/flutter/flutter/issues/53179 -Future queryChromeDriverVersion() async { - final int chromeVersion = await _querySystemChromeMajorVersion(); - final io.File lockFile = io.File( - path.join(environment.webUiRootDir.path, 'dev', 'driver_version.yaml')); - YamlMap _configuration = loadYaml(lockFile.readAsStringSync()) as YamlMap; - final String chromeDriverVersion = - _configuration['chrome'][chromeVersion] as String; - return chromeDriverVersion; -} - /// Make sure LUCI bot has the pinned Chrome version and return the executable. /// /// We are using CIPD packages in LUCI. The pinned chrome version from the @@ -311,55 +299,3 @@ String preinstalledChromeExecutable() { ); } } - -Future _querySystemChromeMajorVersion() async { - String chromeExecutable = ''; - // LUCI uses the Chrome from CIPD packages. - if (isLuci) { - chromeExecutable = preinstalledChromeExecutable(); - } else if (io.Platform.isLinux) { - chromeExecutable = 'google-chrome'; - } else if (io.Platform.isMacOS) { - chromeExecutable = await _findChromeExecutableOnMac(); - } else { - throw UnimplementedError('Web installers only work on Linux and Mac.'); - } - - final io.ProcessResult versionResult = - await io.Process.run('$chromeExecutable', ['--version']); - - if (versionResult.exitCode != 0) { - throw Exception('Failed to locate system Chrome.'); - } - // The output looks like: Google Chrome 79.0.3945.36. - final String output = versionResult.stdout as String; - - print('INFO: chrome version in use $output'); - - // Version number such as 79.0.3945.36. - try { - final String versionAsString = output.trim().split(' ').last; - final String majorVersion = versionAsString.split('.')[0]; - return int.parse(majorVersion); - } catch (e) { - throw Exception( - 'Was expecting a version of the form Google Chrome 79.0.3945.36., ' - 'received $output'); - } -} - -/// Find Google Chrome App on Mac. -Future _findChromeExecutableOnMac() async { - io.Directory chromeDirectory = io.Directory('/Applications') - .listSync() - .whereType() - .firstWhere( - (d) => path.basename(d.path).endsWith('Chrome.app'), - orElse: () => throw Exception('Failed to locate system Chrome'), - ); - - final io.File chromeExecutableDir = io.File( - path.join(chromeDirectory.path, 'Contents', 'MacOS', 'Google Chrome')); - - return chromeExecutableDir.path; -} diff --git a/engine/src/flutter/lib/web_ui/dev/driver_manager.dart b/engine/src/flutter/lib/web_ui/dev/driver_manager.dart deleted file mode 100644 index 61550ff33b3..00000000000 --- a/engine/src/flutter/lib/web_ui/dev/driver_manager.dart +++ /dev/null @@ -1,236 +0,0 @@ -// Copyright 2013 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'dart:io' as io; - -import 'package:path/path.dart' as pathlib; -import 'package:web_driver_installer/chrome_driver_installer.dart'; -import 'package:web_driver_installer/firefox_driver_installer.dart'; -import 'package:web_driver_installer/safari_driver_runner.dart'; -import 'package:yaml/yaml.dart'; - -import 'chrome_installer.dart'; -import 'common.dart'; -import 'environment.dart'; -import 'utils.dart'; - -/// [DriverManager] implementation for Chrome. -/// -/// This manager can be used for both macOS and Linux. -class ChromeDriverManager extends DriverManager { - /// Directory which contains the Chrome's major version. - /// - /// On LUCI we are using the CIPD packages to control Chrome binaries we use. - /// There will be multiple CIPD packages loaded at the same time. Yaml file - /// `driver_version.yaml` contains the version number we want to use. - /// - /// Initialized to the current first to avoid the `Non-nullable` error. - // TODO: https://github.com/flutter/flutter/issues/53179. Local integration - // tests are still using the system Chrome. - late final io.Directory _browserDriverDirWithVersion; - - ChromeDriverManager(String browser) : super(browser) { - final io.File lockFile = io.File(pathlib.join( - environment.webUiRootDir.path, 'dev', 'browser_lock.yaml')); - final YamlMap _configuration = - loadYaml(lockFile.readAsStringSync()) as YamlMap; - final String requiredChromeDriverVersion = - _configuration['required_driver_version']['chrome'] as String; - print('INFO: Major version for Chrome Driver $requiredChromeDriverVersion'); - _browserDriverDirWithVersion = io.Directory(pathlib.join( - environment.webUiDartToolDir.path, - 'drivers', - browser, - requiredChromeDriverVersion, - '${browser}driver-${io.Platform.operatingSystem.toString()}')); - } - - @override - Future _installDriver() async { - if (_browserDriverDirWithVersion.existsSync()) { - _browserDriverDirWithVersion.deleteSync(recursive: true); - } - - _browserDriverDirWithVersion.createSync(recursive: true); - temporaryDirectories.add(_drivers); - - final io.Directory temp = io.Directory.current; - io.Directory.current = _browserDriverDirWithVersion; - - try { - // TODO(nurhan): https://github.com/flutter/flutter/issues/53179 - final String chromeDriverVersion = await queryChromeDriverVersion(); - ChromeDriverInstaller chromeDriverInstaller = - ChromeDriverInstaller.withVersion(chromeDriverVersion); - await chromeDriverInstaller.install(alwaysInstall: true); - } finally { - io.Directory.current = temp; - } - } - - /// Throw an error if driver directory does not exists. - /// - /// Driver should already exist on LUCI as a CIPD package. - @override - Future _verifyDriverForLUCI() { - if (!_browserDriverDirWithVersion.existsSync()) { - throw StateError('Failed to locate Chrome driver on LUCI on path:' - '${_browserDriverDirWithVersion.path}'); - } - return Future.value(); - } - - @override - Future _startDriver() async { - await startProcess('./chromedriver/chromedriver', ['--port=4444'], - workingDirectory: _browserDriverDirWithVersion.path); - print('INFO: Driver started'); - } -} - -/// [DriverManager] implementation for Firefox. -/// -/// This manager can be used for both macOS and Linux. -class FirefoxDriverManager extends DriverManager { - FirefoxDriverManager(String browser) : super(browser); - - FirefoxDriverInstaller firefoxDriverInstaller = - FirefoxDriverInstaller(geckoDriverVersion: getLockedGeckoDriverVersion()); - - @override - Future _installDriver() async { - if (_browserDriverDir.existsSync()) { - _browserDriverDir.deleteSync(recursive: true); - } - - _browserDriverDir.createSync(recursive: true); - temporaryDirectories.add(_drivers); - - final io.Directory temp = io.Directory.current; - io.Directory.current = _browserDriverDir; - - try { - await firefoxDriverInstaller.install(alwaysInstall: false); - } finally { - io.Directory.current = temp; - } - } - - /// Throw an error if driver directory does not exist. - /// - /// Driver should already exist on LUCI as a CIPD package. - @override - Future _verifyDriverForLUCI() { - if (!_browserDriverDir.existsSync()) { - throw StateError('Failed to locate Firefox driver on LUCI on path:' - '${_browserDriverDir.path}'); - } - return Future.value(); - } - - @override - Future _startDriver() async { - await startProcess('./firefoxdriver/geckodriver', ['--port=4444'], - workingDirectory: _browserDriverDir.path); - print('INFO: Driver started'); - } - - /// Get the geckodriver version to be used with [FirefoxDriverInstaller]. - /// - /// For different versions of geckodriver. See: - /// https://github.com/mozilla/geckodriver/releases - static String getLockedGeckoDriverVersion() { - final YamlMap browserLock = BrowserLock.instance.configuration; - String geckoDriverReleaseVersion = browserLock['geckodriver'] as String; - return geckoDriverReleaseVersion; - } -} - -/// [DriverManager] implementation for Safari. -/// -/// This manager is will only be created/used for macOS. -class SafariDriverManager extends DriverManager { - SafariDriverManager(String browser) : super(browser); - - @override - Future _installDriver() { - // No-op. - // macOS comes with Safari Driver installed. - return new Future.value(); - } - - @override - Future _verifyDriverForLUCI() { - // No-op. - // macOS comes with Safari Driver installed. - return Future.value(); - } - - @override - Future _startDriver() async { - final SafariDriverRunner safariDriverRunner = SafariDriverRunner(); - - final io.Process process = - await safariDriverRunner.runDriver(version: 'system'); - - processesToCleanUp.add(process); - } -} - -/// Abstract class for preparing the browser driver before running the integration -/// tests. -abstract class DriverManager { - /// Installation directory for browser's driver. - final io.Directory _browserDriverDir; - - /// This is the parent directory for all drivers. - /// - /// This directory is saved to [temporaryDirectories] and deleted before - /// tests shutdown. - final io.Directory _drivers; - - DriverManager(String browser) - : this._browserDriverDir = io.Directory(pathlib.join( - environment.webUiDartToolDir.path, - 'drivers', - browser, - '${browser}driver-${io.Platform.operatingSystem.toString()}')), - this._drivers = io.Directory( - pathlib.join(environment.webUiDartToolDir.path, 'drivers')); - - Future prepareDriver() async { - if (!isLuci) { - // LUCI installs driver from CIPD, so we skip installing it on LUCI. - await _installDriver(); - } else { - await _verifyDriverForLUCI(); - } - await _startDriver(); - } - - /// Always re-install since driver can change frequently. - /// It usually changes with each the browser version changes. - /// A better solution would be installing the browser and the driver at the - /// same time. - /// TODO(nurhan): https://github.com/flutter/flutter/issues/53179. Partly - // solved. Remaining local integration tests using the locked Chrome version. - Future _installDriver(); - - Future _verifyDriverForLUCI(); - - Future _startDriver(); - - static DriverManager chooseDriver(String browser) { - if (browser == 'chrome') { - return ChromeDriverManager(browser); - } else if (browser == 'firefox') { - return FirefoxDriverManager(browser); - } else if (browser == 'safari' && io.Platform.isMacOS) { - return SafariDriverManager(browser); - } else { - throw StateError('Integration tests are only supported on Firefox, Chrome' - ' and on Safari (running on macOS)'); - } - } -} diff --git a/engine/src/flutter/lib/web_ui/dev/driver_version.yaml b/engine/src/flutter/lib/web_ui/dev/driver_version.yaml deleted file mode 100644 index 7da52450133..00000000000 --- a/engine/src/flutter/lib/web_ui/dev/driver_version.yaml +++ /dev/null @@ -1,15 +0,0 @@ -## Map for driver versions to use for each browser version. -## See: https://chromedriver.chromium.org/downloads -chrome: - 86: '86.0.4240.22' - 85: '85.0.4183.38' - 84: '84.0.4147.30' - 83: '83.0.4103.39' - 81: '81.0.4044.69' - 80: '80.0.3987.106' - 79: '79.0.3945.36' - 78: '78.0.3904.105' - 77: '77.0.3865.40' - 76: '76.0.3809.126' - 75: '75.0.3770.140' - 74: '74.0.3729.6' diff --git a/engine/src/flutter/lib/web_ui/dev/environment.dart b/engine/src/flutter/lib/web_ui/dev/environment.dart index 7c85199860d..db77267a029 100644 --- a/engine/src/flutter/lib/web_ui/dev/environment.dart +++ b/engine/src/flutter/lib/web_ui/dev/environment.dart @@ -29,8 +29,6 @@ class Environment { io.Directory(pathlib.join(hostDebugUnoptDir.path, 'dart-sdk')); final io.Directory webUiRootDir = io.Directory( pathlib.join(engineSrcDir.path, 'flutter', 'lib', 'web_ui')); - final io.Directory integrationTestsDir = io.Directory( - pathlib.join(engineSrcDir.path, 'flutter', 'e2etests', 'web')); for (io.Directory expectedDirectory in [ engineSrcDir, @@ -49,7 +47,6 @@ class Environment { webUiRootDir: webUiRootDir, engineSrcDir: engineSrcDir, engineToolsDir: engineToolsDir, - integrationTestsDir: integrationTestsDir, outDir: outDir, hostDebugUnoptDir: hostDebugUnoptDir, dartSdkDir: dartSdkDir, @@ -61,7 +58,6 @@ class Environment { required this.webUiRootDir, required this.engineSrcDir, required this.engineToolsDir, - required this.integrationTestsDir, required this.outDir, required this.hostDebugUnoptDir, required this.dartSdkDir, @@ -79,9 +75,6 @@ class Environment { /// Path to the engine's "tools" directory. final io.Directory engineToolsDir; - /// Path to the web integration tests. - final io.Directory integrationTestsDir; - /// Path to the engine's "out" directory. /// /// This is where you'll find the ninja output, such as the Dart SDK. diff --git a/engine/src/flutter/lib/web_ui/dev/exceptions.dart b/engine/src/flutter/lib/web_ui/dev/exceptions.dart index 167d1734e65..acb665bf6a8 100644 --- a/engine/src/flutter/lib/web_ui/dev/exceptions.dart +++ b/engine/src/flutter/lib/web_ui/dev/exceptions.dart @@ -11,15 +11,6 @@ class BrowserInstallerException implements Exception { String toString() => message; } -class DriverException implements Exception { - DriverException(this.message); - - final String message; - - @override - String toString() => message; -} - class ToolException implements Exception { ToolException(this.message); diff --git a/engine/src/flutter/lib/web_ui/dev/integration_tests_manager.dart b/engine/src/flutter/lib/web_ui/dev/integration_tests_manager.dart deleted file mode 100644 index 30794f9b9a7..00000000000 --- a/engine/src/flutter/lib/web_ui/dev/integration_tests_manager.dart +++ /dev/null @@ -1,658 +0,0 @@ -// Copyright 2013 The Flutter Authors. All rights reserved. -// Use of this source code is governed by a BSD-style license that can be -// found in the LICENSE file. - -import 'dart:io' as io; -import 'package:args/args.dart'; -import 'package:path/path.dart' as pathlib; - -import 'chrome_installer.dart'; -import 'driver_manager.dart'; -import 'environment.dart'; -import 'exceptions.dart'; -import 'common.dart'; -import 'utils.dart'; - -const String _unsupportedConfigurationWarning = 'WARNING: integration tests ' - 'are only supported on Chrome, Firefox and on Safari (running on macOS)'; - -class IntegrationTestsManager { - final String _browser; - - final bool _useSystemFlutter; - - final DriverManager _driverManager; - - final bool _doUpdateScreenshotGoldens; - - IntegrationTestsManager( - this._browser, this._useSystemFlutter, this._doUpdateScreenshotGoldens) - : _driverManager = DriverManager.chooseDriver(_browser); - - Future runTests() async { - if (validateIfTestsShouldRun()) { - await _driverManager.prepareDriver(); - return await _runTests(); - } else { - return false; - } - } - - Future _runPubGet(String workingDirectory) async { - if (!_useSystemFlutter) { - await _cloneFlutterRepo(); - await _enableWeb(workingDirectory); - } - await runFlutter(workingDirectory, ['pub', 'get'], - useSystemFlutter: _useSystemFlutter); - } - - /// Clone flutter repository, use the youngest commit older than the engine - /// commit. - /// - /// Use engine/src/flutter/.dart_tools to clone the Flutter repo. - /// TODO(nurhan): Use git pull instead if repo exists. - Future _cloneFlutterRepo() async { - // Delete directory if exists. - if (environment.engineDartToolDir.existsSync()) { - environment.engineDartToolDir.deleteSync(recursive: true); - } - environment.engineDartToolDir.createSync(); - - final int exitCode = await runProcess( - environment.cloneFlutterScript.path, - [ - environment.engineDartToolDir.path, - ], - workingDirectory: environment.webUiRootDir.path, - ); - - if (exitCode != 0) { - throw ToolException('ERROR: Failed to clone flutter repo. Exited with ' - 'exit code $exitCode'); - } - } - - Future _enableWeb(String workingDirectory) async { - await runFlutter(workingDirectory, ['config', '--enable-web'], - useSystemFlutter: _useSystemFlutter); - } - - /// Runs all the web tests under e2e_tests/web. - Future _runTests() async { - // Only list the files under e2e_tests/web. - final List entities = - environment.integrationTestsDir.listSync(followLinks: false); - - bool allTestsPassed = true; - for (io.FileSystemEntity e in entities) { - // The tests should be under this directories. - if (e is io.Directory) { - allTestsPassed = allTestsPassed && await _validateAndRunTests(e); - } - } - return allTestsPassed; - } - - /// Run tests in a single directory under: e2e_tests/web. - /// - /// Run `flutter pub get` as the first step. - /// - /// Validate the directory before running the tests. Each directory is - /// expected to be a test project which includes a `pubspec.yaml` file - /// and a `test_driver` directory. - Future _validateAndRunTests(io.Directory directory) async { - _validateTestDirectory(directory); - await _runPubGet(directory.path); - final bool testResults = await _runTestsInDirectory(directory); - return testResults; - } - - int _numberOfPassedTests = 0; - int _numberOfFailedTests = 0; - - Future _runTestsInDirectory(io.Directory directory) async { - final io.Directory testDirectory = - io.Directory(pathlib.join(directory.path, 'test_driver')); - final List entities = testDirectory - .listSync(followLinks: false) - .whereType() - .toList(); - - final List e2eTestsToRun = []; - final List blockedTests = - blockedTestsListsMap[getBlockedTestsListMapKey(_browser)] ?? []; - - // If no target is specified run all the tests. - if (_runAllTestTargets) { - // The following loops over the contents of the directory and saves an - // expected driver file name for each e2e test assuming any dart file - // not ending with `_test.dart` is an e2e test. - // Other files are not considered since developers can add files such as - // README. - for (io.File f in entities) { - final String basename = pathlib.basename(f.path); - if (!basename.contains('_test.dart') && basename.endsWith('.dart')) { - // Do not add the basename if it is in the `blockedTests`. - if (!blockedTests.contains(basename)) { - e2eTestsToRun.add(basename); - } else { - print('INFO: Test $basename is skipped since it is blocked for ' - '${getBlockedTestsListMapKey(_browser)}'); - } - } - } - if (isVerboseLoggingEnabled) { - print( - 'INFO: In project ${directory} ${e2eTestsToRun.length} tests to run.'); - } - } else { - // If a target is specified it will run regardless of if it's blocked or - // not. There will be an info note to warn the developer. - final String targetTest = - IntegrationTestsArgumentParser.instance.testTarget; - final io.File file = - entities.singleWhere((f) => pathlib.basename(f.path) == targetTest); - final String basename = pathlib.basename(file.path); - if (blockedTests.contains(basename) && isVerboseLoggingEnabled) { - print('INFO: Test $basename do not run on CI environments. Please ' - 'remove it from the blocked tests list if you want to enable this ' - 'test on CI.'); - } - e2eTestsToRun.add(basename); - } - - final Set buildModes = _getBuildModes(); - - for (String fileName in e2eTestsToRun) { - await _runTestsTarget(directory, fileName, buildModes); - } - - final int numberOfTestsRun = _numberOfPassedTests + _numberOfFailedTests; - - print('INFO: ${numberOfTestsRun} tests run. ${_numberOfPassedTests} passed ' - 'and ${_numberOfFailedTests} failed.'); - return _numberOfFailedTests == 0; - } - - Future _runTestsTarget( - io.Directory directory, String target, Set buildModes) async { - final Set renderingBackends = _getRenderingBackends(); - for (String renderingBackend in renderingBackends) { - for (String mode in buildModes) { - if (!blockedTestsListsMapForModes[mode]!.contains(target) && - !blockedTestsListsMapForRenderBackends[renderingBackend]! - .contains(target)) { - final bool result = await _runTestsInMode(directory, target, - mode: mode, webRenderer: renderingBackend); - if (result) { - _numberOfPassedTests++; - } else { - _numberOfFailedTests++; - } - } - } - } - } - - Future _runTestsInMode(io.Directory directory, String testName, - {String mode = 'profile', String webRenderer = 'html'}) async { - String executable = - _useSystemFlutter ? 'flutter' : environment.flutterCommand.path; - Map flutterEnvironment = Map(); - if (_doUpdateScreenshotGoldens) { - flutterEnvironment['UPDATE_GOLDENS'] = 'true'; - } - final IntegrationArguments arguments = - IntegrationArguments.fromBrowser(_browser); - final int exitCode = await runProcess( - executable, - arguments.getTestArguments(testName, mode, webRenderer), - workingDirectory: directory.path, - environment: flutterEnvironment, - ); - - if (exitCode != 0) { - final String command = - arguments.getCommandToRun(testName, mode, webRenderer); - io.stderr - .writeln('ERROR: Failed to run test. Exited with exit code $exitCode' - '. To run $testName locally use the following command:' - '\n\n$command'); - return false; - } else { - return true; - } - } - - Set _getRenderingBackends() { - Set renderingBackends; - if (_renderingBackendSelected) { - final String mode = IntegrationTestsArgumentParser.instance.webRenderer; - renderingBackends = {mode}; - } else { - // TODO(nurhan): Enable `auto` when recipe is sharded. - renderingBackends = {'html', 'canvaskit'}; - } - return renderingBackends; - } - - Set _getBuildModes() { - Set buildModes; - if (_buildModeSelected) { - final String mode = IntegrationTestsArgumentParser.instance.buildMode; - if (mode == 'debug' && _browser != 'chrome') { - throw ToolException('Debug mode is only supported for Chrome.'); - } else { - buildModes = {mode}; - } - } else { - buildModes = _browser == 'chrome' - ? {'debug', 'profile', 'release'} - : {'profile', 'release'}; - } - return buildModes; - } - - /// Validate the directory has a `pubspec.yaml` file and a `test_driver` - /// directory. - /// - /// Also check the validity of files under `test_driver` directory calling - /// [_checkE2ETestsValidity] method. - void _validateTestDirectory(io.Directory directory) { - final List entities = - directory.listSync(followLinks: false); - - // Whether the project has the pubspec.yaml file. - bool pubSpecFound = false; - // The test directory 'test_driver'. - io.Directory? testDirectory; - - for (io.FileSystemEntity e in entities) { - // The tests should be under this directories. - final String baseName = pathlib.basename(e.path); - if (e is io.Directory && baseName == 'test_driver') { - testDirectory = e; - } - if (e is io.File && baseName == 'pubspec.yaml') { - pubSpecFound = true; - } - } - if (!pubSpecFound) { - throw StateError('ERROR: pubspec.yaml file not found in the test project ' - 'in the directory ${directory.path}.'); - } - if (testDirectory == null) { - throw StateError( - 'ERROR: test_driver folder not found in the test project.' - 'in the directory ${directory.path}.'); - } else { - _checkE2ETestsValidity(testDirectory); - } - } - - /// Checks if each e2e test file in the directory has a driver test - /// file to run it. - /// - /// Prints informative message to the developer if an error has found. - /// For each e2e test which has name {name}.dart there will be a driver - /// file which drives it. The driver file should be named: - /// {name}_test.dart - void _checkE2ETestsValidity(io.Directory testDirectory) { - final Iterable directories = - testDirectory.listSync(followLinks: false).whereType(); - - if (directories.length > 0) { - throw StateError('${testDirectory.path} directory should not contain ' - 'any sub-directories'); - } - - final Iterable entities = - testDirectory.listSync(followLinks: false).whereType(); - - final Set expectedDriverFileNames = Set(); - final Set foundDriverFileNames = Set(); - int numberOfTests = 0; - - // The following loops over the contents of the directory and saves an - // expected driver file name for each e2e test assuming any file - // not ending with `_test.dart` is an e2e test. - for (io.File f in entities) { - final String basename = pathlib.basename(f.path); - if (basename.contains('_test.dart')) { - // First remove this from expectedSet if not there add to the foundSet. - if (!expectedDriverFileNames.remove(basename)) { - foundDriverFileNames.add(basename); - } - } else if (basename.contains('.dart')) { - // Only run on dart files. - final String e2efileName = pathlib.basenameWithoutExtension(f.path); - final String expectedDriverName = '${e2efileName}_test.dart'; - numberOfTests++; - // First remove this from foundSet if not there add to the expectedSet. - if (!foundDriverFileNames.remove(expectedDriverName)) { - expectedDriverFileNames.add(expectedDriverName); - } - } - } - - if (numberOfTests == 0) { - throw StateError( - 'WARNING: No tests to run in this directory ${testDirectory.path}'); - } - - // TODO(nurhan): In order to reduce the work required from team members, - // remove the need for driver file, by using the same template file. - // Some driver files are missing. - if (expectedDriverFileNames.length > 0) { - for (String expectedDriverName in expectedDriverFileNames) { - print('ERROR: Test driver file named has ${expectedDriverName} ' - 'not found under directory ${testDirectory.path}. Stopping the ' - 'integration tests. Please add ${expectedDriverName}. Check to ' - 'README file on more details on how to set up integration tests.'); - } - throw StateError('Error in test files. Check the logs for ' - 'further instructions'); - } - } - - bool get _buildModeSelected => - !IntegrationTestsArgumentParser.instance.buildMode.isEmpty; - - bool get _renderingBackendSelected => - !IntegrationTestsArgumentParser.instance.webRenderer.isEmpty; - - bool get _runAllTestTargets => - IntegrationTestsArgumentParser.instance.testTarget.isEmpty; - - /// Validate the given `browser`, `platform` combination is suitable for - /// integration tests to run. - bool validateIfTestsShouldRun() { - if (_buildModeSelected) { - final String mode = IntegrationTestsArgumentParser.instance.buildMode; - if (mode == 'debug' && _browser != 'chrome') { - throw ToolException('Debug mode is only supported for Chrome.'); - } - } - - // Chrome tests should run at all Platforms (Linux, macOS, Windows). - // They can also run successfully on CI and local. - if (_browser == 'chrome') { - return true; - } else if (_browser == 'firefox' && - (io.Platform.isLinux || io.Platform.isMacOS)) { - return true; - } else if (_browser == 'safari' && io.Platform.isMacOS && !isLuci) { - return true; - } else { - io.stderr.writeln(_unsupportedConfigurationWarning); - return false; - } - } -} - -/// Interface for collecting arguments to give `flutter drive` to run the -/// integration tests. -abstract class IntegrationArguments { - IntegrationArguments(); - - factory IntegrationArguments.fromBrowser(String browser) { - if (browser == 'chrome') { - return ChromeIntegrationArguments(); - } else if (browser == 'firefox') { - return FirefoxIntegrationArguments(); - } else if (browser == 'safari' && io.Platform.isMacOS) { - return SafariIntegrationArguments(); - } else { - throw StateError(_unsupportedConfigurationWarning); - } - } - - List getTestArguments( - String testName, String mode, String webRenderer); - - String getCommandToRun(String testName, String mode, String webRenderer); -} - -/// Arguments to give `flutter drive` to run the integration tests on Chrome. -class ChromeIntegrationArguments extends IntegrationArguments { - List getTestArguments( - String testName, String mode, String webRenderer) { - return [ - 'drive', - '--target=test_driver/${testName}', - '-d', - 'web-server', - '--$mode', - '--browser-name=chrome', - if (isLuci) '--chrome-binary=${preinstalledChromeExecutable()}', - '--headless', - '--local-engine=host_debug_unopt', - '--web-renderer=$webRenderer', - ]; - } - - String getCommandToRun(String testName, String mode, String webRenderer) { - String statementToRun = 'flutter drive ' - '--target=test_driver/${testName} -d web-server --$mode ' - '--browser-name=chrome --local-engine=host_debug_unopt ' - '--web-renderer=$webRenderer'; - if (isLuci) { - statementToRun = '$statementToRun --chrome-binary=' - '${preinstalledChromeExecutable()}'; - } - return statementToRun; - } -} - -/// Arguments to give `flutter drive` to run the integration tests on Firefox. -class FirefoxIntegrationArguments extends IntegrationArguments { - List getTestArguments( - String testName, String mode, String webRenderer) { - return [ - 'drive', - '--target=test_driver/${testName}', - '-d', - 'web-server', - '--$mode', - '--browser-name=firefox', - '--headless', - '--local-engine=host_debug_unopt', - '--web-renderer=$webRenderer', - ]; - } - - String getCommandToRun(String testName, String mode, String webRenderer) { - final String arguments = - getTestArguments(testName, mode, webRenderer).join(' '); - return 'flutter $arguments'; - } -} - -/// Arguments to give `flutter drive` to run the integration tests on Safari. -class SafariIntegrationArguments extends IntegrationArguments { - SafariIntegrationArguments(); - - List getTestArguments( - String testName, String mode, String webRenderer) { - return [ - 'drive', - '--target=test_driver/${testName}', - '-d', - 'web-server', - '--$mode', - '--browser-name=safari', - '--local-engine=host_debug_unopt', - '--web-renderer=$webRenderer', - ]; - } - - String getCommandToRun(String testName, String mode, String webRenderer) { - final String arguments = - getTestArguments(testName, mode, webRenderer).join(' '); - return 'flutter $arguments'; - } -} - -/// Parses additional options that can be used when running integration tests. -class IntegrationTestsArgumentParser { - static final IntegrationTestsArgumentParser _singletonInstance = - IntegrationTestsArgumentParser._(); - - /// The [IntegrationTestsArgumentParser] singleton. - static IntegrationTestsArgumentParser get instance => _singletonInstance; - - IntegrationTestsArgumentParser._(); - - /// If target name is provided integration tests can run that one test - /// instead of running all the tests. - late final String testTarget; - - /// The build mode to run the integration tests. - /// - /// If not specified, these tests will run using 'debug, profile, release' - /// modes on Chrome and will run using 'profile, release' on other browsers. - /// - /// In order to skip a test for one of the modes, add the test to the - /// `blockedTestsListsMapForModes` list for the relevant compile mode. - late final String buildMode; - - /// Whether to use html, canvaskit or auto for web renderer. - /// - /// If not set all backends will be used one after another for integration - /// tests. If set only the provided option will be used. - late final String webRenderer; - - void populateOptions(ArgParser argParser) { - argParser - ..addOption( - 'target', - defaultsTo: '', - help: 'By default integration tests are run for all the tests under' - 'flutter/e2etests/web directory. If a test name is specified, that ' - 'only that test will run. The test name will be the name of the ' - 'integration test (e2e test) file. For example: ' - 'text_editing_integration.dart or ' - 'profile_diagnostics_integration.dart', - ) - ..addOption('build-mode', - defaultsTo: '', - help: 'Flutter supports three modes when building your app. This ' - 'option sets the build mode for the integration tests. ' - 'By default an integration test will sequentially run on ' - 'multiple modes. All three modes (debug, release, profile) are ' - 'used for Chrome. Only profile, release modes will be used for ' - 'other browsers. In other words, if a build mode is selected ' - 'tests will only be run using that mode. ' - 'See https://flutter.dev/docs/testing/build-modes for more ' - 'details on the build modes.') - ..addOption('web-renderer', - defaultsTo: '', - help: 'By default all three options (`html`, `canvaskit`, `auto`) ' - ' for rendering backends are tested when running integration ' - ' tests. If this option is set only the backend provided by this ' - ' option will be used. `auto`, `canvaskit` and `html`' - ' are the available options. '); - } - - /// Populate results of the arguments passed. - void parseOptions(ArgResults argResults) { - testTarget = argResults['target'] as String; - buildMode = argResults['build-mode'] as String; - if (!buildMode.isEmpty && - buildMode != 'debug' && - buildMode != 'profile' && - buildMode != 'release') { - throw ArgumentError('Unexpected build mode: $buildMode'); - } - webRenderer = argResults['web-renderer'] as String; - if (!webRenderer.isEmpty && - webRenderer != 'html' && - webRenderer != 'canvaskit' && - webRenderer != 'auto') { - throw ArgumentError('Unexpected rendering backend: $webRenderer'); - } - } -} - -/// Prepares a key for the [blackList] map. -/// -/// Uses the browser name and the operating system name. -String getBlockedTestsListMapKey(String browser) => - '${browser}-${io.Platform.operatingSystem}'; - -/// Tests that should be skipped run for a specific platform-browser -/// combination. -/// -/// These tests might be failing or might have been implemented for a specific -/// configuration. -/// -/// For example when adding a tests only intended for mobile browsers, it should -/// be added to [blockedTests] for `chrome-linux`, `safari-macos` and -/// `chrome-macos`. It will work on `chrome-android`, `safari-ios`. -/// -/// Note that integration tests are only running on chrome for now. -const Map> blockedTestsListsMap = >{ - 'chrome-linux': [ - 'target_platform_android_integration.dart', - 'target_platform_ios_integration.dart', - 'target_platform_macos_integration.dart', - ], - 'chrome-macos': [ - 'target_platform_ios_integration.dart', - 'target_platform_android_integration.dart', - ], - 'safari-macos': [ - 'target_platform_ios_integration.dart', - 'target_platform_android_integration.dart', - 'image_loading_integration.dart', - ], - 'firefox-linux': [ - 'target_platform_android_integration.dart', - 'target_platform_ios_integration.dart', - 'target_platform_macos_integration.dart', - ], - 'firefox-macos': [ - 'target_platform_android_integration.dart', - 'target_platform_ios_integration.dart', - ], -}; - -/// Tests blocked for one of the build modes. -/// -/// If a test is not supposed to run for one of the modes also add that test -/// to the corresponding list. -// TODO(nurhan): Remove the failing test after fixing. -const Map> blockedTestsListsMapForModes = - >{ - 'debug': [ - 'treeshaking_integration.dart', - 'text_editing_integration.dart', - 'url_strategy_integration.dart', - ], - 'profile': [], - 'release': [], -}; - -/// Tests blocked for one of the rendering backends. -/// -/// If a test is not suppose to run for one of the backends also add that test -/// to the corresponding list. -// TODO(nurhan): Remove the failing test after fixing. -const Map> blockedTestsListsMapForRenderBackends = - >{ - 'auto': [ - 'image_loading_integration.dart', - 'platform_messages_integration.dart', - 'profile_diagnostics_integration.dart', - 'scroll_wheel_integration.dart', - 'text_editing_integration.dart', - 'treeshaking_integration.dart', - 'url_strategy_integration.dart', - ], - 'html': [], - // This test failed on canvaskit on all three build modes. - 'canvaskit': [ - 'image_loading_integration.dart', - ], -}; diff --git a/engine/src/flutter/lib/web_ui/dev/test_runner.dart b/engine/src/flutter/lib/web_ui/dev/test_runner.dart index f0322de57dc..0ae6487ad76 100644 --- a/engine/src/flutter/lib/web_ui/dev/test_runner.dart +++ b/engine/src/flutter/lib/web_ui/dev/test_runner.dart @@ -38,7 +38,6 @@ import 'environment.dart'; import 'exceptions.dart'; import 'firefox.dart'; import 'firefox_installer.dart'; -import 'integration_tests_manager.dart'; import 'macos_info.dart'; import 'safari_installation.dart'; import 'safari_ios.dart'; @@ -58,18 +57,6 @@ List failedShards = []; /// Whether all test shards succeeded. bool get allShardsPassed => failedShards.isEmpty; -/// The type of tests requested by the tool user. -enum TestTypesRequested { - /// For running the unit tests only. - unit, - - /// For running the integration tests only. - integration, - - /// For running both unit and integration tests. - all, -} - /// Command-line argument parsers that parse browser-specific options. final List _browserArgParsers = [ ChromeArgParser.instance, @@ -116,13 +103,6 @@ class TestCommand extends Command with ArgUtils { help: 'felt test command runs the unit tests and the integration tests ' 'at the same time. If this flag is set, only run the unit tests.', ) - ..addFlag( - 'integration-tests-only', - defaultsTo: false, - help: 'felt test command runs the unit tests and the integration tests ' - 'at the same time. If this flag is set, only run the integration ' - 'tests.', - ) ..addFlag('use-system-flutter', defaultsTo: false, help: @@ -170,7 +150,6 @@ class TestCommand extends Command with ArgUtils { browserArgParser.populateOptions(argParser); } GeneralTestsArgumentParser.instance.populateOptions(argParser); - IntegrationTestsArgumentParser.instance.populateOptions(argParser); } @override @@ -193,25 +172,6 @@ class TestCommand extends Command with ArgUtils { /// repeat them. bool _testPreparationReady = false; - /// Check the flags to see what type of tests are requested. - TestTypesRequested get testType { - if (boolArg('unit-tests-only')! && boolArg('integration-tests-only')!) { - throw ArgumentError('Conflicting arguments: unit-tests-only and ' - 'integration-tests-only are both set'); - } else if (boolArg('unit-tests-only')!) { - print('Running the unit tests only'); - return TestTypesRequested.unit; - } else if (boolArg('integration-tests-only')!) { - if (!isChrome && !isSafariOnMacOS && !isFirefox) { - throw UnimplementedError( - 'Integration tests are only available on Chrome Desktop for now'); - } - return TestTypesRequested.integration; - } else { - return TestTypesRequested.all; - } - } - @override Future run() async { for (BrowserArgParser browserArgParser in _browserArgParsers) { @@ -277,46 +237,7 @@ class TestCommand extends Command with ArgUtils { return message.toString(); } - Future runTests() async { - try { - switch (testType) { - case TestTypesRequested.unit: - return runUnitTests(); - case TestTypesRequested.integration: - return runIntegrationTests(); - case TestTypesRequested.all: - if (runAllTests && isIntegrationTestsAvailable) { - bool unitTestResult = await runUnitTests(); - bool integrationTestResult = await runIntegrationTests(); - if (integrationTestResult != unitTestResult) { - print( - 'Tests run. Integration tests passed: $integrationTestResult ' - 'unit tests passed: $unitTestResult'); - } - return integrationTestResult && unitTestResult; - } else { - return await runUnitTests(); - } - } - } on TestFailureException { - return true; - } - } - - Future runIntegrationTests() async { - // Parse additional arguments specific for integration testing. - IntegrationTestsArgumentParser.instance.parseOptions(argResults!); - await _prepare(); - final bool result = await IntegrationTestsManager( - browser, useSystemFlutter, doUpdateScreenshotGoldens) - .runTests(); - if (!result) { - failedShards.add('Integration tests'); - } - return result; - } - - Future runUnitTests() async { + Future runUnitTests() async { _copyTestFontsIntoWebUi(); await _buildHostPage(); await _prepare(); @@ -327,7 +248,6 @@ class TestCommand extends Command with ArgUtils { } else { await _runSpecificTests(targetFiles); } - return true; } /// Preparations before running the tests such as booting simulators or @@ -887,6 +807,6 @@ class TestRunnerStep implements PipelineStep { @override Future run() async { - await testCommand.runTests(); + await testCommand.runUnitTests(); } } diff --git a/engine/src/flutter/lib/web_ui/pubspec.yaml b/engine/src/flutter/lib/web_ui/pubspec.yaml index b9d0e711a9a..64046a79823 100644 --- a/engine/src/flutter/lib/web_ui/pubspec.yaml +++ b/engine/src/flutter/lib/web_ui/pubspec.yaml @@ -28,8 +28,3 @@ dev_dependencies: url: git://github.com/flutter/web_installers.git path: packages/simulators/ ref: 4a7b0a2c84b8993bf4d19030218de38c18838c26 - web_driver_installer: - git: - url: git://github.com/flutter/web_installers.git - path: packages/web_drivers/ - ref: 4a7b0a2c84b8993bf4d19030218de38c18838c26 diff --git a/engine/src/flutter/tools/licenses/lib/main.dart b/engine/src/flutter/tools/licenses/lib/main.dart index 4a96ab499f2..b05c45a965e 100644 --- a/engine/src/flutter/tools/licenses/lib/main.dart +++ b/engine/src/flutter/tools/licenses/lib/main.dart @@ -2122,9 +2122,6 @@ class _RepositoryFlutterDirectory extends _RepositoryDirectory { && entry.name != 'build' && entry.name != 'ci' && entry.name != 'flutter_frontend_server' - // The directory that contains end to end tests. - // Should be excluded from the licence checks. - && entry.name != 'e2etests' // None of the web_sdk code is linked into Flutter apps. It's only used // by engine tests and tools. && entry.name != 'web_sdk' diff --git a/engine/src/flutter/web_sdk/web_test_utils/lib/environment.dart b/engine/src/flutter/web_sdk/web_test_utils/lib/environment.dart index 5cc9eb8a6bf..1ba54b6b474 100644 --- a/engine/src/flutter/web_sdk/web_test_utils/lib/environment.dart +++ b/engine/src/flutter/web_sdk/web_test_utils/lib/environment.dart @@ -48,8 +48,6 @@ class Environment { io.Directory(pathlib.join(hostDebugUnoptDir.path, 'dart-sdk')); final io.Directory webUiRootDir = io.Directory( pathlib.join(engineSrcDir.path, 'flutter', 'lib', 'web_ui')); - final io.Directory integrationTestsDir = io.Directory( - pathlib.join(engineSrcDir.path, 'flutter', 'e2etests', 'web')); for (final io.Directory expectedDirectory in [ engineSrcDir, @@ -68,7 +66,6 @@ class Environment { webUiRootDir: webUiRootDir, engineSrcDir: engineSrcDir, engineToolsDir: engineToolsDir, - integrationTestsDir: integrationTestsDir, outDir: outDir, hostDebugUnoptDir: hostDebugUnoptDir, dartSdkDir: dartSdkDir, @@ -80,7 +77,6 @@ class Environment { required this.webUiRootDir, required this.engineSrcDir, required this.engineToolsDir, - required this.integrationTestsDir, required this.outDir, required this.hostDebugUnoptDir, required this.dartSdkDir, @@ -98,9 +94,6 @@ class Environment { /// Path to the engine's "tools" directory. final io.Directory engineToolsDir; - /// Path to the web integration tests. - final io.Directory integrationTestsDir; - /// Path to the engine's "out" directory. /// /// This is where you'll find the ninja output, such as the Dart SDK.