DEV/flutter_flutter
1
0
Fork 0
mirror of https://github.com/flutter/flutter.git synced 2026-02-20 02:29:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

Refactor args parsing/environment constructor for scenario_app (flutter/engine#50980)

Browse Source 
This moves the ever-growing amount of options and defaults into it's own
class(es).

The test runner itself has no tests (yet), but this shim will make
writing tests easier. I tried to make no other real changes to how the
runner functions in this PR.
This commit is contained in:
Matan Lurey 2024-02-26 15:29:32 -08:00 committed by GitHub
parent 9eca58ab36
commit 71f9dec343
5 changed files with 423 additions and 188 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

53
engine/src/flutter/testing/scenario_app/bin/README.md
View File

@ -24,23 +24,40 @@ dart bin/android_integration_tests.dart --smoke-test dev.flutter.scenarios.Engin
## Additional arguments
- `--verbose`: Print additional information about the test run.
```txt
-v, --verbose Enable verbose logging
-h, --help Print usage information
--[no-]enable-impeller Whether to enable Impeller as the graphics backend. If true, the
test runner will use --impeller-backend if set, otherwise the
default backend will be used. To explicitly run with the Skia
backend, set this to false (--no-enable-impeller).
--impeller-backend The graphics backend to use when --enable-impeller is true. Unlike
the similar option when launching an app, there is no fallback;
that is, either Vulkan or OpenGLES must be specified.
[vulkan (default), opengles]
--logs-dir Path to a directory where logs and screenshots are stored.
--out-dir=<path/to/out/android_variant> Path to a out/{variant} directory where the APKs are built.
Defaults to the latest updated out/ directory that starts with
"android_" if the current working directory is within the engine
repository.
--smoke-test=<package.ClassName> Fully qualified class name of a single test to run. For example try
"dev.flutter.scenarios.EngineLaunchE2ETest" or
"dev.flutter.scenariosui.ExternalTextureTests".
--output-contents-golden=<path/to/golden.txt> Path to a file that contains the expected filenames of golden
files. If the current working directory is within the engine
repository, defaults to
./testing/scenario_app/android/expected_golden_output.txt.
```
- `--adb`: The path to the `adb` tool. Defaults to
`third_party/android_tools/sdk/platform-tools/adb`.
## Advanced usage
- `--out-dir`: The directory containing the build artifacts. Defaults to the
last updated build directory in `out/` that starts with `android_`.
- `--logs-dir`: The directory to store logs and screenshots. Defaults to
`FLUTTER_LOGS_DIR` if set, or `out/.../scenario_app/logs` otherwise.
- `--use-skia-gold`: Use Skia Gold to compare screenshots. Defaults to true
when running on CI, and false otherwise (i.e. when running locally). If
set to true, `isSkiaGoldClientAvailable` must be true.
- `--enable-impeller`: Enable Impeller for the Android app. Defaults to
false, which means that the app will use Skia as the graphics backend.
- `--impeller-backend`: The Impeller backend to use for the Android app.
Defaults to 'vulkan'. Only used when `--enable-impeller` is set to true.
```txt
--[no-]use-skia-gold Whether to use Skia Gold to compare screenshots. Defaults to true
on CI and false otherwise.
--adb=<path/to/adb> Path to the Android Debug Bridge (adb) executable. If the current
working directory is within the engine repository, defaults to
./third_party/android_tools/sdk/platform-tools/adb.
--ndk-stack=<path/to/ndk-stack> Path to the NDK stack tool. Defaults to the checked-in version in
third_party/android_tools if the current working directory is
within the engine repository on a supported platform.
```

231
engine/src/flutter/testing/scenario_app/bin/run_android_tests.dart
View File

@ -7,7 +7,6 @@ import 'dart:convert';
import 'dart:io';
import 'dart:typed_data';
import 'package:args/args.dart';
import 'package:dir_contents_diff/dir_contents_diff.dart' show dirContentsDiff;
import 'package:engine_repo_tools/engine_repo_tools.dart';
import 'package:path/path.dart';
@ -15,180 +14,73 @@ import 'package:process/process.dart';
import 'package:skia_gold_client/skia_gold_client.dart';
import 'utils/adb_logcat_filtering.dart';
import 'utils/environment.dart';
import 'utils/logs.dart';
import 'utils/options.dart';
import 'utils/process_manager_extension.dart';
import 'utils/screenshot_transformer.dart';
void _withTemporaryCwd(String path, void Function() callback) {
final String originalCwd = Directory.current.path;
Directory.current = Directory(path).parent.path;
try {
callback();
} finally {
Directory.current = originalCwd;
}
}
// If you update the arguments, update the documentation in the README.md file.
void main(List<String> args) async {
final Engine? engine = Engine.tryFindWithin();
final ArgParser parser = ArgParser()
..addFlag(
'help',
help: 'Prints usage information',
negatable: false,
)
..addFlag(
'verbose',
help: 'Prints verbose output',
negatable: false,
)
..addOption(
'adb',
help: 'Path to the adb tool',
defaultsTo: engine != null
? join(
engine.srcDir.path,
'third_party',
'android_tools',
'sdk',
'platform-tools',
'adb',
)
: null,
)
..addOption(
'ndk-stack',
help: 'Path to the ndk-stack tool',
defaultsTo: engine != null
? join(
engine.srcDir.path,
'third_party',
'android_tools',
'ndk',
'prebuilt',
() {
if (Platform.isLinux) {
return 'linux-x86_64';
} else if (Platform.isMacOS) {
return 'darwin-x86_64';
} else if (Platform.isWindows) {
return 'windows-x86_64';
} else {
throw UnsupportedError('Unsupported platform: ${Platform.operatingSystem}');
}
}(),
'bin',
'ndk-stack',
)
: null,
)
..addOption(
'out-dir',
help: 'Out directory',
defaultsTo: engine
?.outputs()
.where((Output o) => basename(o.path.path).startsWith('android_'))
.firstOrNull
?.path
.path,
)
..addOption(
'smoke-test',
help: 'Runs a single test to verify the setup',
)
..addFlag(
'use-skia-gold',
help: 'Use Skia Gold to compare screenshots.',
defaultsTo: isLuciEnv,
)
..addFlag(
'enable-impeller',
help: 'Enable Impeller for the Android app.',
)
..addOption(
'output-contents-golden',
help: 'Path to a file that contains the expected filenames of golden files.',
defaultsTo: engine != null
? join(
engine.srcDir.path,
'flutter',
'testing',
'scenario_app',
'android',
'expected_golden_output.txt',
)
: null,
)
..addOption(
'impeller-backend',
help: 'The Impeller backend to use for the Android app.',
allowed: <String>['vulkan', 'opengles'],
defaultsTo: 'vulkan',
)
..addOption(
'logs-dir',
help: 'The directory to store the logs and screenshots. Defaults to '
'the value of the FLUTTER_LOGS_DIR environment variable, if set, '
'otherwise it defaults to a path within out-dir.',
defaultsTo: Platform.environment['FLUTTER_LOGS_DIR'],
// Get some basic environment information to guide the rest of the program.
final Environment environment = Environment(
isCi: Platform.environment['LUCI_CONTEXT'] != null,
showVerbose: Options.showVerbose(args),
logsDir: Platform.environment['FLUTTER_LOGS_DIR'],
);
// Determine if the CWD is within an engine checkout.
final Engine? localEngineDir = Engine.tryFindWithin();
// Show usage if requested.
if (Options.showUsage(args)) {
stdout.writeln(Options.usage(
environment: environment,
localEngineDir: localEngineDir,
));
return;
}
// Parse the command line arguments.
final Options options;
try {
options = Options.parse(
args,
environment: environment,
localEngine: localEngineDir,
);
} on FormatException catch (error) {
stderr.writeln(error);
stderr.writeln(Options.usage(
environment: environment,
localEngineDir: localEngineDir,
));
exitCode = 1;
return;
}
runZonedGuarded(
() async {
final ArgResults results = parser.parse(args);
if (results['help'] as bool) {
stdout.writeln(parser.usage);
return;
}
if (results['out-dir'] == null) {
panic(<String>['--out-dir is required']);
}
if (results['adb'] == null) {
panic(<String>['--adb is required']);
}
final bool verbose = results['verbose'] as bool;
final Directory outDir = Directory(results['out-dir'] as String);
final File adb = File(results['adb'] as String);
final bool useSkiaGold = results['use-skia-gold'] as bool;
final String? smokeTest = results['smoke-test'] as String?;
final bool enableImpeller = results['enable-impeller'] as bool;
final String? contentsGolden = results['output-contents-golden'] as String?;
final _ImpellerBackend? impellerBackend = _ImpellerBackend.tryParse(results['impeller-backend'] as String?);
if (enableImpeller && impellerBackend == null) {
panic(<String>[
'invalid graphics-backend',
results['impeller-backend'] as String? ?? '<null>'
]);
}
final Directory logsDir = Directory(results['logs-dir'] as String? ?? join(outDir.path, 'scenario_app', 'logs'));
final String? ndkStack = results['ndk-stack'] as String?;
if (ndkStack == null) {
panic(<String>['--ndk-stack is required']);
}
await _run(
verbose: verbose,
outDir: outDir,
adb: adb,
smokeTestFullPath: smokeTest,
useSkiaGold: useSkiaGold,
enableImpeller: enableImpeller,
impellerBackend: impellerBackend,
logsDir: logsDir,
contentsGolden: contentsGolden,
ndkStack: ndkStack,
verbose: options.verbose,
outDir: Directory(options.outDir),
adb: File(options.adb),
smokeTestFullPath: options.smokeTest,
useSkiaGold: options.useSkiaGold,
enableImpeller: options.enableImpeller,
impellerBackend: _ImpellerBackend.tryParse(options.impellerBackend),
logsDir: Directory(options.logsDir),
contentsGolden: options.outputContentsGolden,
ndkStack: options.ndkStack,
);
exit(0);
},
(Object error, StackTrace stackTrace) {
if (error is! Panic) {
stderr.writeln(error);
stderr.writeln('Unhandled error: $error');
stderr.writeln(stackTrace);
}
exit(1);
exitCode = 1;
},
);
}
@ -222,18 +114,6 @@ Future<void> _run({
required String ndkStack,
}) async {
const ProcessManager pm = LocalProcessManager();
if (!outDir.existsSync()) {
panic(<String>[
'out-dir does not exist: $outDir',
'make sure to build the selected engine variant'
]);
}
if (!adb.existsSync()) {
panic(<String>['cannot find adb: $adb', 'make sure to run gclient sync']);
}
final String scenarioAppPath = join(outDir.path, 'scenario_app');
final String logcatPath = join(logsDir.path, 'logcat.txt');
@ -521,7 +401,7 @@ Future<void> _run({
// TODO(matanlurey): Resolve this in a better way. On CI this file always exists.
File(join(screenshotPath, 'noop.txt')).writeAsStringSync('');
// TODO(gaaclarke): We should move this into dir_contents_diff.
_withTemporaryCwd(contentsGolden, () {
_withTemporaryCwd(dirname(contentsGolden), () {
final int exitCode = dirContentsDiff(basename(contentsGolden), screenshotPath);
if (exitCode != 0) {
panic(<String>['Output contents incorrect.']);
@ -531,3 +411,14 @@ Future<void> _run({
}
}
}
void _withTemporaryCwd(String path, void Function() callback) {
final String originalCwd = Directory.current.path;
Directory.current = Directory(path).path;
try {
callback();
} finally {
Directory.current = originalCwd;
}
}

45
engine/src/flutter/testing/scenario_app/bin/utils/environment.dart Normal file
View File

@ -0,0 +1,45 @@
// Copyright 2013 The Flutter Authors. All rights reserved.
// Use of this source code is governed by a BSD-style license that can be
// found in the LICENSE file.
import 'package:meta/meta.dart';
/// An overridable collection of values provided by the environment.
@immutable
final class Environment {
/// Creates a new environment from the given values.
const Environment({
required this.isCi,
required this.showVerbose,
required this.logsDir,
});
/// Whether the current program is running on a CI environment.
///
/// Useful for determining if certain features should be enabled or disabled
/// based on the environment, or to add safety checks (for example, using
/// confusing or ambiguous flags).
final bool isCi;
/// Whether the user has requested verbose logging and program output.
final bool showVerbose;
/// What directory to store logs and screenshots in.
final String? logsDir;
@override
bool operator ==(Object o) {
return o is Environment &&
o.isCi == isCi &&
o.showVerbose == showVerbose &&
o.logsDir == logsDir;
}
@override
int get hashCode => Object.hash(isCi, showVerbose, logsDir);
@override
String toString() {
return 'Environment(isCi: $isCi, showVerbose: $showVerbose, logsDir: $logsDir)';
}
}

281
engine/src/flutter/testing/scenario_app/bin/utils/options.dart Normal file
View File

@ -0,0 +1,281 @@
import 'dart:io' as io;
import 'package:args/args.dart';
import 'package:engine_repo_tools/engine_repo_tools.dart';
import 'package:path/path.dart' as p;
import 'environment.dart';
/// Command line options and parser for the Android `scenario_app` test runner.
extension type const Options._(ArgResults _args) {
/// Parses the command line [args] into a set of options.
///
/// Throws a [FormatException] if command line arguments are invalid.
factory Options.parse(
List<String> args, {
required Environment environment,
required Engine? localEngine,
}) {
final ArgResults results = _parser(environment, localEngine).parse(args);
final Options options = Options._(results);
// The 'adb' tool must exist.
if (results['adb'] == null) {
throw const FormatException('The --adb option must be set.');
} else if (!io.File(options.adb).existsSync()) {
throw FormatException(
'The adb tool does not exist at ${options.adb}.',
);
}
// The 'ndk-stack' tool must exist.
if (results['ndk-stack'] == null) {
throw const FormatException('The --ndk-stack option must be set.');
} else if (!io.File(options.ndkStack).existsSync()) {
throw FormatException(
'The ndk-stack tool does not exist at ${options.ndkStack}.',
);
}
// The 'out-dir' must exist.
if (results['out-dir'] == null) {
throw const FormatException('The --out-dir option must be set.');
} else if (!io.Directory(options.outDir).existsSync()) {
throw FormatException(
'The out directory does not exist at ${options.outDir}.',
);
}
return options;
}
/// Whether usage information should be shown based on command line [args].
///
/// This is a shortcut that can be used to determine if the usage information
/// before parsing the remaining command line arguments. For example:
///
/// ```
/// void main(List<String> args) {
/// if (Options.showUsage(args)) {
/// stdout.writeln(Options.usage);
/// return;
/// }
/// final options = Options.parse(args);
/// // ...
/// }
/// ```
static bool showUsage(List<String> args) {
// If any of the arguments are '--help' or -'h'.
return args.isNotEmpty && args.any((String arg) {
return arg == '--help' || arg == '-h';
});
}
/// Whether verbose logging should be enabled based on command line [args].
///
/// This is a shortcut that can be used to determine if verbose logging should
/// be enabled before parsing the remaining command line arguments. For
/// example:
///
/// ```
/// void main(List<String> args) {
/// final bool verbose = Options.showVerbose(args);
/// // ...
/// }
/// ```
static bool showVerbose(List<String> args) {
// If any of the arguments are '--verbose' or -'v'.
return args.isNotEmpty && args.any((String arg) {
return arg == '--verbose' || arg == '-v';
});
}
/// Returns usage information for the `scenario_app` test runner.
///
/// If [verbose] is `true`, then additional options are shown.
static String usage({
required Environment environment,
required Engine? localEngineDir,
}) {
return _parser(environment, localEngineDir).usage;
}
/// Parses the command line [args] into a set of options.
///
/// Unlike [_miniParser], this parser includes all options.
static ArgParser _parser(Environment environment, Engine? localEngine) {
final bool hideUnusualOptions = !environment.showVerbose;
return ArgParser(usageLineLength: 120)
..addFlag(
'verbose',
abbr: 'v',
help: 'Enable verbose logging',
negatable: false,
)
..addFlag(
'help',
abbr: 'h',
help: 'Print usage information',
negatable: false,
)
..addFlag(
'use-skia-gold',
help:
'Whether to use Skia Gold to compare screenshots. Defaults to true '
'on CI and false otherwise.',
defaultsTo: environment.isCi,
hide: hideUnusualOptions,
)
..addFlag(
'enable-impeller',
help:
'Whether to enable Impeller as the graphics backend. If true, the '
'test runner will use --impeller-backend if set, otherwise the '
'default backend will be used. To explicitly run with the Skia '
'backend, set this to false (--no-enable-impeller).',
)
..addOption(
'impeller-backend',
help: 'The graphics backend to use when --enable-impeller is true. '
'Unlike the similar option when launching an app, there is no '
'fallback; that is, either Vulkan or OpenGLES must be specified. ',
allowed: <String>['vulkan', 'opengles'],
defaultsTo: 'vulkan',
)
..addOption(
'logs-dir',
help: 'Path to a directory where logs and screenshots are stored.',
defaultsTo: environment.logsDir,
)
..addOption(
'adb',
help: 'Path to the Android Debug Bridge (adb) executable. '
'If the current working directory is within the engine repository, '
'defaults to ./third_party/android_tools/sdk/platform-tools/adb.',
defaultsTo: localEngine != null
? p.join(
localEngine.srcDir.path,
'third_party',
'android_tools',
'sdk',
'platform-tools',
'adb',
)
: null,
valueHelp: 'path/to/adb',
hide: hideUnusualOptions,
)
..addOption(
'ndk-stack',
help:
'Path to the NDK stack tool. Defaults to the checked-in version in '
'third_party/android_tools if the current working directory is '
'within the engine repository on a supported platform.',
defaultsTo: localEngine != null &&
(io.Platform.isLinux ||
io.Platform.isMacOS ||
io.Platform.isWindows)
? p.join(
localEngine.srcDir.path,
'third_party',
'android_tools',
'ndk',
'prebuilt',
() {
if (io.Platform.isLinux) {
return 'linux-x86_64';
} else if (io.Platform.isMacOS) {
return 'darwin-x86_64';
} else if (io.Platform.isWindows) {
return 'windows-x86_64';
} else {
// Unreachable.
throw UnsupportedError(
'Unsupported platform: ${io.Platform.operatingSystem}',
);
}
}(),
'bin',
'ndk-stack',
)
: null,
valueHelp: 'path/to/ndk-stack',
hide: hideUnusualOptions,
)
..addOption(
'out-dir',
help: 'Path to a out/{variant} directory where the APKs are built. '
'Defaults to the latest updated out/ directory that starts with '
'"android_" if the current working directory is within the engine '
'repository.',
defaultsTo: environment.isCi ? null : localEngine
?.outputs()
.where((Output o) => p.basename(o.path.path).startsWith('android_'))
.firstOrNull
?.path
.path,
mandatory: environment.isCi,
valueHelp: 'path/to/out/android_variant',
)
..addOption(
'smoke-test',
help: 'Fully qualified class name of a single test to run. For example '
'try "dev.flutter.scenarios.EngineLaunchE2ETest" or '
'"dev.flutter.scenariosui.ExternalTextureTests".',
valueHelp: 'package.ClassName',
)
..addOption(
'output-contents-golden',
help: 'Path to a file that contains the expected filenames of golden '
'files. If the current working directory is within the engine '
'repository, defaults to ./testing/scenario_app/android/'
'expected_golden_output.txt.',
defaultsTo: localEngine != null
? p.join(
localEngine.flutterDir.path,
'testing',
'scenario_app',
'android',
'expected_golden_output.txt',
)
: null,
valueHelp: 'path/to/golden.txt',
);
}
/// Whether verbose logging should be enabled.
bool get verbose => _args['verbose'] as bool;
/// Whether usage information should be shown.
bool get help => _args['help'] as bool;
/// Whether to use Skia Gold to compare screenshots.
bool get useSkiaGold => _args['use-skia-gold'] as bool;
/// Whether to enable Impeller as the graphics backend.
bool get enableImpeller => _args['enable-impeller'] as bool;
/// The graphics backend to use when --enable-impeller is true.
String get impellerBackend => _args['impeller-backend'] as String;
/// Path to a directory where logs and screenshots are stored.
String get logsDir {
final String? logsDir = _args['logs-dir'] as String?;
return logsDir ?? p.join(outDir, 'logs');
}
/// Path to the Android Debug Bridge (adb) executable.
String get adb => _args['adb'] as String;
/// Path to the NDK stack tool.
String get ndkStack => _args['ndk-stack'] as String;
/// Path to a out/{variant} directory where the APKs are built.
String get outDir => _args['out-dir'] as String;
/// Fully qualified class name of a single test to run.
String? get smokeTest => _args['smoke-test'] as String?;
/// Path to a file that contains the expected filenames of golden files.
String? get outputContentsGolden => _args['output-contents-golden'] as String;
}

1
engine/src/flutter/testing/scenario_app/pubspec.yaml
View File

@ -18,6 +18,7 @@ dependencies:
args: any
dir_contents_diff: any
engine_repo_tools: any
meta: any
path: any
process: any
sky_engine: any