mirror of
https://github.com/flutter/flutter.git
synced 2026-02-20 02:29:02 +08:00
386 lines
14 KiB
Dart
386 lines
14 KiB
Dart
// Copyright 2013 The Flutter Authors. All rights reserved.
|
|
// Use of this source code is governed by a BSD-style license that can be
|
|
// found in the LICENSE file.
|
|
|
|
import 'dart:io' as io;
|
|
import 'package:path/path.dart' as pathlib;
|
|
import 'package:web_driver_installer/chrome_driver_installer.dart';
|
|
|
|
import 'chrome_installer.dart';
|
|
import 'environment.dart';
|
|
import 'exceptions.dart';
|
|
import 'common.dart';
|
|
import 'utils.dart';
|
|
|
|
class IntegrationTestsManager {
|
|
final String _browser;
|
|
|
|
/// Installation directory for browser's driver.
|
|
///
|
|
/// Always re-install since driver can change frequently.
|
|
/// It usually changes with each the browser version changes.
|
|
/// A better solution would be installing the browser and the driver at the
|
|
/// same time.
|
|
// TODO(nurhan): https://github.com/flutter/flutter/issues/53179. Partly
|
|
// solved. Remaining local integration tests using the locked Chrome version.
|
|
final io.Directory _browserDriverDir;
|
|
|
|
/// This is the parent directory for all drivers.
|
|
///
|
|
/// This directory is saved to [temporaryDirectories] and deleted before
|
|
/// tests shutdown.
|
|
final io.Directory _drivers;
|
|
|
|
final bool _useSystemFlutter;
|
|
|
|
IntegrationTestsManager(this._browser, this._useSystemFlutter)
|
|
: this._browserDriverDir = io.Directory(pathlib.join(
|
|
environment.webUiDartToolDir.path,
|
|
'drivers',
|
|
_browser,
|
|
'${_browser}driver-${io.Platform.operatingSystem.toString()}')),
|
|
this._drivers = io.Directory(
|
|
pathlib.join(environment.webUiDartToolDir.path, 'drivers'));
|
|
|
|
Future<bool> runTests() async {
|
|
if (_browser != 'chrome') {
|
|
print('WARNING: integration tests are only supported on chrome for now');
|
|
return false;
|
|
} else {
|
|
if (!isLuci) {
|
|
// LUCI installs driver from CIPD, so we skip installing it on LUCI.
|
|
await _prepareDriver();
|
|
} else {
|
|
await _verifyDriverForLUCI();
|
|
}
|
|
await _startDriver(_browserDriverDir.path);
|
|
// TODO(nurhan): https://github.com/flutter/flutter/issues/52987
|
|
return await _runTests();
|
|
}
|
|
}
|
|
|
|
Future<void> _runPubGet(String workingDirectory) async {
|
|
if (!_useSystemFlutter) {
|
|
await _cloneFlutterRepo();
|
|
await _enableWeb(workingDirectory);
|
|
}
|
|
await runFlutter(workingDirectory, <String>['pub', 'get'],
|
|
useSystemFlutter: _useSystemFlutter);
|
|
}
|
|
|
|
/// Clone flutter repository, use the youngest commit older than the engine
|
|
/// commit.
|
|
///
|
|
/// Use engine/src/flutter/.dart_tools to clone the Flutter repo.
|
|
/// TODO(nurhan): Use git pull instead if repo exists.
|
|
Future<void> _cloneFlutterRepo() async {
|
|
// Delete directory if exists.
|
|
if (environment.engineDartToolDir.existsSync()) {
|
|
environment.engineDartToolDir.deleteSync(recursive: true);
|
|
}
|
|
environment.engineDartToolDir.createSync();
|
|
|
|
final int exitCode = await runProcess(
|
|
environment.cloneFlutterScript.path,
|
|
<String>[
|
|
environment.engineDartToolDir.path,
|
|
],
|
|
workingDirectory: environment.webUiRootDir.path,
|
|
);
|
|
|
|
if (exitCode != 0) {
|
|
throw ToolException('ERROR: Failed to clone flutter repo. Exited with '
|
|
'exit code $exitCode');
|
|
}
|
|
}
|
|
|
|
Future<void> _enableWeb(String workingDirectory) async {
|
|
await runFlutter(workingDirectory, <String>['config', '--enable-web'],
|
|
useSystemFlutter: _useSystemFlutter);
|
|
}
|
|
|
|
/// Driver should already exist on LUCI as a CIPD package.
|
|
///
|
|
/// Throw an error if directory does not exists.
|
|
void _verifyDriverForLUCI() {
|
|
if (!_browserDriverDir.existsSync()) {
|
|
throw StateError('Failed to locate Chrome driver on LUCI on path:'
|
|
'${_browserDriverDir.path}');
|
|
}
|
|
}
|
|
|
|
void _startDriver(String workingDirectory) async {
|
|
await startProcess('./chromedriver/chromedriver', ['--port=4444'],
|
|
workingDirectory: workingDirectory);
|
|
print('INFO: Driver started');
|
|
}
|
|
|
|
void _prepareDriver() async {
|
|
if (_browserDriverDir.existsSync()) {
|
|
_browserDriverDir.deleteSync(recursive: true);
|
|
}
|
|
|
|
_browserDriverDir.createSync(recursive: true);
|
|
temporaryDirectories.add(_drivers);
|
|
|
|
io.Directory temp = io.Directory.current;
|
|
io.Directory.current = _browserDriverDir;
|
|
|
|
// TODO(nurhan): https://github.com/flutter/flutter/issues/53179
|
|
final String chromeDriverVersion = await queryChromeDriverVersion();
|
|
ChromeDriverInstaller chromeDriverInstaller =
|
|
ChromeDriverInstaller.withVersion(chromeDriverVersion);
|
|
await chromeDriverInstaller.install(alwaysInstall: true);
|
|
io.Directory.current = temp;
|
|
}
|
|
|
|
/// Runs all the web tests under e2e_tests/web.
|
|
Future<bool> _runTests() async {
|
|
// Only list the files under e2e_tests/web.
|
|
final List<io.FileSystemEntity> entities =
|
|
environment.integrationTestsDir.listSync(followLinks: false);
|
|
|
|
bool allTestsPassed = true;
|
|
for (io.FileSystemEntity e in entities) {
|
|
// The tests should be under this directories.
|
|
if (e is io.Directory) {
|
|
allTestsPassed = allTestsPassed && await _validateAndRunTests(e);
|
|
}
|
|
}
|
|
return allTestsPassed;
|
|
}
|
|
|
|
/// Run tests in a single directory under: e2e_tests/web.
|
|
///
|
|
/// Run `flutter pub get` as the first step.
|
|
///
|
|
/// Validate the directory before running the tests. Each directory is
|
|
/// expected to be a test project which includes a `pubspec.yaml` file
|
|
/// and a `test_driver` directory.
|
|
Future<bool> _validateAndRunTests(io.Directory directory) async {
|
|
_validateTestDirectory(directory);
|
|
await _runPubGet(directory.path);
|
|
final bool testResults = await _runTestsInDirectory(directory);
|
|
return testResults;
|
|
}
|
|
|
|
Future<bool> _runTestsInDirectory(io.Directory directory) async {
|
|
final io.Directory testDirectory =
|
|
io.Directory(pathlib.join(directory.path, 'test_driver'));
|
|
final List<io.File> entities = testDirectory
|
|
.listSync(followLinks: false)
|
|
.whereType<io.File>()
|
|
.toList();
|
|
|
|
final List<String> e2eTestsToRun = List<String>();
|
|
final List<String> blockedTests =
|
|
blockedTestsListsMap[getBlockedTestsListMapKey(_browser)] ?? <String>[];
|
|
|
|
// The following loops over the contents of the directory and saves an
|
|
// expected driver file name for each e2e test assuming any dart file
|
|
// not ending with `_test.dart` is an e2e test.
|
|
// Other files are not considered since developers can add files such as
|
|
// README.
|
|
for (io.File f in entities) {
|
|
final String basename = pathlib.basename(f.path);
|
|
if (!basename.contains('_test.dart') && basename.endsWith('.dart')) {
|
|
// Do not add the basename if it is in the `blockedTests`.
|
|
if (!blockedTests.contains(basename)) {
|
|
e2eTestsToRun.add(basename);
|
|
} else {
|
|
print('INFO: Test $basename is skipped since it is blocked for '
|
|
'${getBlockedTestsListMapKey(_browser)}');
|
|
}
|
|
}
|
|
}
|
|
|
|
print(
|
|
'INFO: In project ${directory} ${e2eTestsToRun.length} tests to run.');
|
|
|
|
int numberOfPassedTests = 0;
|
|
int numberOfFailedTests = 0;
|
|
for (String fileName in e2eTestsToRun) {
|
|
final bool testResults =
|
|
await _runTestsInProfileMode(directory, fileName);
|
|
if (testResults) {
|
|
numberOfPassedTests++;
|
|
} else {
|
|
numberOfFailedTests++;
|
|
}
|
|
}
|
|
final int numberOfTestsRun = numberOfPassedTests + numberOfFailedTests;
|
|
|
|
print('INFO: ${numberOfTestsRun} tests run. ${numberOfPassedTests} passed '
|
|
'and ${numberOfFailedTests} failed.');
|
|
return numberOfFailedTests == 0;
|
|
}
|
|
|
|
Future<bool> _runTestsInProfileMode(
|
|
io.Directory directory, String testName) async {
|
|
final String executable =
|
|
_useSystemFlutter ? 'flutter' : environment.flutterCommand.path;
|
|
final int exitCode = await runProcess(
|
|
executable,
|
|
<String>[
|
|
'drive',
|
|
'--target=test_driver/${testName}',
|
|
'-d',
|
|
'web-server',
|
|
'--profile',
|
|
'--browser-name=$_browser',
|
|
if (isLuci) '--chrome-binary=${preinstalledChromeExecutable()}',
|
|
if (isLuci) '--headless',
|
|
'--local-engine=host_debug_unopt',
|
|
],
|
|
workingDirectory: directory.path,
|
|
);
|
|
|
|
if (exitCode != 0) {
|
|
String statementToRun = 'flutter drive '
|
|
'--target=test_driver/${testName} -d web-server --profile '
|
|
'--browser-name=$_browser --local-engine=host_debug_unopt';
|
|
if (isLuci) {
|
|
statementToRun = '$statementToRun --chrome-binary='
|
|
'${preinstalledChromeExecutable()}';
|
|
}
|
|
io.stderr
|
|
.writeln('ERROR: Failed to run test. Exited with exit code $exitCode'
|
|
'. Statement to run $testName locally use the following '
|
|
'command:\n\n$statementToRun');
|
|
return false;
|
|
} else {
|
|
return true;
|
|
}
|
|
}
|
|
|
|
/// Validate the directory has a `pubspec.yaml` file and a `test_driver`
|
|
/// directory.
|
|
///
|
|
/// Also check the validity of files under `test_driver` directory calling
|
|
/// [_checkE2ETestsValidity] method.
|
|
void _validateTestDirectory(io.Directory directory) {
|
|
final List<io.FileSystemEntity> entities =
|
|
directory.listSync(followLinks: false);
|
|
|
|
// Whether the project has the pubspec.yaml file.
|
|
bool pubSpecFound = false;
|
|
// The test directory 'test_driver'.
|
|
io.Directory testDirectory = null;
|
|
|
|
for (io.FileSystemEntity e in entities) {
|
|
// The tests should be under this directories.
|
|
final String baseName = pathlib.basename(e.path);
|
|
if (e is io.Directory && baseName == 'test_driver') {
|
|
testDirectory = e;
|
|
}
|
|
if (e is io.File && baseName == 'pubspec.yaml') {
|
|
pubSpecFound = true;
|
|
}
|
|
}
|
|
if (!pubSpecFound) {
|
|
throw StateError('ERROR: pubspec.yaml file not found in the test project '
|
|
'in the directory ${directory.path}.');
|
|
}
|
|
if (testDirectory == null) {
|
|
throw StateError(
|
|
'ERROR: test_driver folder not found in the test project.'
|
|
'in the directory ${directory.path}.');
|
|
} else {
|
|
_checkE2ETestsValidity(testDirectory);
|
|
}
|
|
}
|
|
|
|
/// Checks if each e2e test file in the directory has a driver test
|
|
/// file to run it.
|
|
///
|
|
/// Prints informative message to the developer if an error has found.
|
|
/// For each e2e test which has name {name}.dart there will be a driver
|
|
/// file which drives it. The driver file should be named:
|
|
/// {name}_test.dart
|
|
void _checkE2ETestsValidity(io.Directory testDirectory) {
|
|
final Iterable<io.Directory> directories =
|
|
testDirectory.listSync(followLinks: false).whereType<io.Directory>();
|
|
|
|
if (directories.length > 0) {
|
|
throw StateError('${testDirectory.path} directory should not contain '
|
|
'any sub-directories');
|
|
}
|
|
|
|
final Iterable<io.File> entities =
|
|
testDirectory.listSync(followLinks: false).whereType<io.File>();
|
|
|
|
final Set<String> expectedDriverFileNames = Set<String>();
|
|
final Set<String> foundDriverFileNames = Set<String>();
|
|
int numberOfTests = 0;
|
|
|
|
// The following loops over the contents of the directory and saves an
|
|
// expected driver file name for each e2e test assuming any file
|
|
// not ending with `_test.dart` is an e2e test.
|
|
for (io.File f in entities) {
|
|
final String basename = pathlib.basename(f.path);
|
|
if (basename.contains('_test.dart')) {
|
|
// First remove this from expectedSet if not there add to the foundSet.
|
|
if (!expectedDriverFileNames.remove(basename)) {
|
|
foundDriverFileNames.add(basename);
|
|
}
|
|
} else if (basename.contains('.dart')) {
|
|
// Only run on dart files.
|
|
final String e2efileName = pathlib.basenameWithoutExtension(f.path);
|
|
final String expectedDriverName = '${e2efileName}_test.dart';
|
|
numberOfTests++;
|
|
// First remove this from foundSet if not there add to the expectedSet.
|
|
if (!foundDriverFileNames.remove(expectedDriverName)) {
|
|
expectedDriverFileNames.add(expectedDriverName);
|
|
}
|
|
}
|
|
}
|
|
|
|
if (numberOfTests == 0) {
|
|
throw StateError(
|
|
'WARNING: No tests to run in this directory ${testDirectory.path}');
|
|
}
|
|
|
|
// TODO(nurhan): In order to reduce the work required from team members,
|
|
// remove the need for driver file, by using the same template file.
|
|
// Some driver files are missing.
|
|
if (expectedDriverFileNames.length > 0) {
|
|
for (String expectedDriverName in expectedDriverFileNames) {
|
|
print('ERROR: Test driver file named has ${expectedDriverName} '
|
|
'not found under directory ${testDirectory.path}. Stopping the '
|
|
'integration tests. Please add ${expectedDriverName}. Check to '
|
|
'README file on more details on how to setup integration tests.');
|
|
}
|
|
throw StateError('Error in test files. Check the logs for '
|
|
'further instructions');
|
|
}
|
|
}
|
|
}
|
|
|
|
/// Prepares a key for the [blockedTests] map.
|
|
///
|
|
/// Uses the browser name and the operating system name.
|
|
String getBlockedTestsListMapKey(String browser) =>
|
|
'${browser}-${io.Platform.operatingSystem}';
|
|
|
|
/// Tests that should be skipped run for a specific platform-browser
|
|
/// combination.
|
|
///
|
|
/// These tests might be failing or might have been implemented for a specific
|
|
/// configuration.
|
|
///
|
|
/// For example when adding a tests only intended for mobile browsers, it should
|
|
/// be added to [blockedTests] for `chrome-linux`, `safari-macos` and
|
|
/// `chrome-macos`. It will work on `chrome-android`, `safari-ios`.
|
|
///
|
|
/// Note that integration tests are only running on chrome for now.
|
|
const Map<String, List<String>> blockedTestsListsMap = <String, List<String>>{
|
|
'chrome-linux': [
|
|
'target_platform_ios_e2e.dart',
|
|
'target_platform_macos_e2e.dart',
|
|
],
|
|
'chrome-macos': [
|
|
'target_platform_ios_e2e.dart',
|
|
'target_platform_android_e2e.dart',
|
|
],
|
|
};
|