mirror of
https://github.com/flutter/flutter.git
synced 2026-02-20 02:29:02 +08:00
ed30d77ab9
`//flutter/testing` now contains a lot of utilities used by other test targets. This includes stuff like working with render targets that use either OpenGL or Metal, fixtures for interacting with the Dart VM, test assertion predicates, etc.. However, these utilities themselves are not tested as part of a standalone test suite. Instead, only the test targets that include it exercise these utilities. Since these are no longer trivial, a new test target has been added that tests the testing utilities directly.
357 lines
12 KiB
Python
Executable File
357 lines
12 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# Copyright 2013 The Flutter Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
"""
|
|
A top level harness to run all unit-tests in a specific engine build.
|
|
"""
|
|
|
|
import argparse
|
|
import glob
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
|
|
buildroot_dir = os.path.abspath(os.path.join(os.path.realpath(__file__), '..', '..', '..'))
|
|
out_dir = os.path.join(buildroot_dir, 'out')
|
|
golden_dir = os.path.join(buildroot_dir, 'flutter', 'testing', 'resources')
|
|
fonts_dir = os.path.join(buildroot_dir, 'flutter', 'third_party', 'txt', 'third_party', 'fonts')
|
|
roboto_font_path = os.path.join(fonts_dir, 'Roboto-Regular.ttf')
|
|
dart_tests_dir = os.path.join(buildroot_dir, 'flutter', 'testing', 'dart',)
|
|
|
|
fml_unittests_filter = '--gtest_filter=-*TimeSensitiveTest*:*GpuThreadMerger*'
|
|
|
|
def RunCmd(cmd, **kwargs):
|
|
print(subprocess.check_output(cmd, **kwargs))
|
|
|
|
def IsMac():
|
|
return sys.platform == 'darwin'
|
|
|
|
|
|
def IsLinux():
|
|
return sys.platform.startswith('linux')
|
|
|
|
|
|
def IsWindows():
|
|
return sys.platform.startswith(('cygwin', 'win'))
|
|
|
|
|
|
def ExecutableSuffix():
|
|
return '.exe' if IsWindows() else ''
|
|
|
|
def FindExecutablePath(path):
|
|
if os.path.exists(path):
|
|
return path
|
|
|
|
if IsWindows():
|
|
exe_path = path + '.exe'
|
|
if os.path.exists(exe_path):
|
|
return exe_path
|
|
|
|
bat_path = path + '.bat'
|
|
if os.path.exists(bat_path):
|
|
return bat_path
|
|
|
|
raise Exception('Executable %s does not exist!' % path)
|
|
|
|
|
|
def RunEngineExecutable(build_dir, executable_name, filter, flags=[], cwd=buildroot_dir):
|
|
if filter is not None and executable_name not in filter:
|
|
print('Skipping %s due to filter.' % executable_name)
|
|
return
|
|
|
|
executable = FindExecutablePath(os.path.join(build_dir, executable_name))
|
|
|
|
print('Running %s in %s' % (executable_name, cwd))
|
|
test_command = [ executable ] + flags
|
|
print(' '.join(test_command))
|
|
RunCmd(test_command, cwd=cwd)
|
|
|
|
|
|
def RunCCTests(build_dir, filter):
|
|
print("Running Engine Unit-tests.")
|
|
|
|
shuffle_flags = [
|
|
"--gtest_shuffle",
|
|
"--gtest_repeat=2",
|
|
]
|
|
|
|
RunEngineExecutable(build_dir, 'client_wrapper_glfw_unittests', filter, shuffle_flags)
|
|
|
|
RunEngineExecutable(build_dir, 'client_wrapper_unittests', filter, shuffle_flags)
|
|
|
|
# https://github.com/flutter/flutter/issues/36294
|
|
if not IsWindows():
|
|
RunEngineExecutable(build_dir, 'embedder_unittests', filter, shuffle_flags)
|
|
else:
|
|
RunEngineExecutable(build_dir, 'flutter_windows_unittests', filter, shuffle_flags)
|
|
|
|
RunEngineExecutable(build_dir, 'client_wrapper_windows_unittests', filter, shuffle_flags)
|
|
|
|
flow_flags = ['--gtest_filter=-PerformanceOverlayLayer.Gold']
|
|
if IsLinux():
|
|
flow_flags = [
|
|
'--golden-dir=%s' % golden_dir,
|
|
'--font-file=%s' % roboto_font_path,
|
|
]
|
|
RunEngineExecutable(build_dir, 'flow_unittests', filter, flow_flags + shuffle_flags)
|
|
|
|
RunEngineExecutable(build_dir, 'fml_unittests', filter, [ fml_unittests_filter ] + shuffle_flags)
|
|
|
|
RunEngineExecutable(build_dir, 'runtime_unittests', filter, shuffle_flags)
|
|
|
|
# https://github.com/flutter/flutter/issues/36295
|
|
if not IsWindows():
|
|
RunEngineExecutable(build_dir, 'shell_unittests', filter, shuffle_flags)
|
|
|
|
RunEngineExecutable(build_dir, 'ui_unittests', filter, shuffle_flags)
|
|
|
|
RunEngineExecutable(build_dir, 'testing_unittests', filter, shuffle_flags)
|
|
|
|
# These unit-tests are Objective-C and can only run on Darwin.
|
|
if IsMac():
|
|
RunEngineExecutable(build_dir, 'flutter_channels_unittests', filter, shuffle_flags)
|
|
|
|
# https://github.com/flutter/flutter/issues/36296
|
|
if IsLinux():
|
|
RunEngineExecutable(build_dir, 'txt_unittests', filter, shuffle_flags)
|
|
|
|
|
|
def RunEngineBenchmarks(build_dir, filter):
|
|
print("Running Engine Benchmarks.")
|
|
|
|
RunEngineExecutable(build_dir, 'shell_benchmarks', filter)
|
|
|
|
RunEngineExecutable(build_dir, 'fml_benchmarks', filter)
|
|
|
|
if IsLinux():
|
|
RunEngineExecutable(build_dir, 'txt_benchmarks', filter)
|
|
|
|
|
|
|
|
def SnapshotTest(build_dir, dart_file, kernel_file_output, verbose_dart_snapshot):
|
|
print("Generating snapshot for test %s" % dart_file)
|
|
|
|
dart = os.path.join(build_dir, 'dart-sdk', 'bin', 'dart')
|
|
frontend_server = os.path.join(build_dir, 'gen', 'frontend_server.dart.snapshot')
|
|
flutter_patched_sdk = os.path.join(build_dir, 'flutter_patched_sdk')
|
|
test_packages = os.path.join(dart_tests_dir, '.packages')
|
|
|
|
assert os.path.exists(dart)
|
|
assert os.path.exists(frontend_server)
|
|
assert os.path.exists(flutter_patched_sdk)
|
|
assert os.path.exists(test_packages)
|
|
|
|
snapshot_command = [
|
|
dart,
|
|
frontend_server,
|
|
'--sdk-root',
|
|
flutter_patched_sdk,
|
|
'--incremental',
|
|
'--target=flutter',
|
|
'--packages',
|
|
test_packages,
|
|
'--output-dill',
|
|
kernel_file_output,
|
|
dart_file
|
|
]
|
|
|
|
if verbose_dart_snapshot:
|
|
RunCmd(snapshot_command, cwd=buildroot_dir)
|
|
else:
|
|
subprocess.check_output(snapshot_command, cwd=buildroot_dir)
|
|
assert os.path.exists(kernel_file_output)
|
|
|
|
|
|
def RunDartTest(build_dir, dart_file, verbose_dart_snapshot, multithreaded):
|
|
kernel_file_name = os.path.basename(dart_file) + '.kernel.dill'
|
|
kernel_file_output = os.path.join(out_dir, kernel_file_name)
|
|
|
|
SnapshotTest(build_dir, dart_file, kernel_file_output, verbose_dart_snapshot)
|
|
|
|
command_args = [
|
|
'--disable-observatory',
|
|
'--use-test-fonts',
|
|
kernel_file_output
|
|
]
|
|
|
|
if multithreaded:
|
|
threading = 'multithreaded'
|
|
command_args.insert(0, '--force-multithreading')
|
|
else:
|
|
threading = 'single-threaded'
|
|
|
|
print("Running test '%s' using 'flutter_tester' (%s)" % (kernel_file_name, threading))
|
|
RunEngineExecutable(build_dir, 'flutter_tester', None, command_args)
|
|
|
|
def RunPubGet(build_dir, directory):
|
|
print("Running 'pub get' in the tests directory %s" % dart_tests_dir)
|
|
|
|
pub_get_command = [
|
|
os.path.join(build_dir, 'dart-sdk', 'bin', 'pub'),
|
|
'get'
|
|
]
|
|
RunCmd(pub_get_command, cwd=directory)
|
|
|
|
|
|
def EnsureDebugUnoptSkyPackagesAreBuilt():
|
|
variant_out_dir = os.path.join(out_dir, 'host_debug_unopt')
|
|
|
|
ninja_command = [
|
|
'autoninja',
|
|
'-C',
|
|
variant_out_dir,
|
|
'flutter/sky/packages'
|
|
]
|
|
|
|
# Attempt running Ninja if the out directory exists.
|
|
# We don't want to blow away any custom GN args the caller may have already set.
|
|
if os.path.exists(variant_out_dir):
|
|
RunCmd(ninja_command, cwd=buildroot_dir)
|
|
return
|
|
|
|
gn_command = [
|
|
os.path.join(buildroot_dir, 'flutter', 'tools', 'gn'),
|
|
'--runtime-mode',
|
|
'debug',
|
|
'--unopt',
|
|
'--no-lto',
|
|
]
|
|
|
|
RunCmd(gn_command, cwd=buildroot_dir)
|
|
RunCmd(ninja_command, cwd=buildroot_dir)
|
|
|
|
def EnsureJavaTestsAreBuilt(android_out_dir):
|
|
ninja_command = [
|
|
'autoninja',
|
|
'-C',
|
|
android_out_dir,
|
|
'flutter/shell/platform/android:robolectric_tests'
|
|
]
|
|
|
|
# Attempt running Ninja if the out directory exists.
|
|
# We don't want to blow away any custom GN args the caller may have already set.
|
|
if os.path.exists(android_out_dir):
|
|
RunCmd(ninja_command, cwd=buildroot_dir)
|
|
return
|
|
|
|
# Otherwise prepare the directory first, then build the test.
|
|
gn_command = [
|
|
os.path.join(buildroot_dir, 'flutter', 'tools', 'gn'),
|
|
'--android',
|
|
'--unoptimized',
|
|
'--runtime-mode=debug',
|
|
'--no-lto',
|
|
]
|
|
RunCmd(gn_command, cwd=buildroot_dir)
|
|
RunCmd(ninja_command, cwd=buildroot_dir)
|
|
|
|
def AssertExpectedJavaVersion():
|
|
EXPECTED_VERSION = '1.8'
|
|
# `java -version` is output to stderr. https://bugs.java.com/bugdatabase/view_bug.do?bug_id=4380614
|
|
version_output = subprocess.check_output(['java', '-version'], stderr=subprocess.STDOUT)
|
|
match = bool(re.compile('version "%s' % EXPECTED_VERSION).search(version_output))
|
|
message = "JUnit tests need to be run with Java %s. Check the `java -version` on your PATH." % EXPECTED_VERSION
|
|
assert match, message
|
|
|
|
def RunJavaTests(filter, android_variant='android_debug_unopt'):
|
|
AssertExpectedJavaVersion()
|
|
android_out_dir = os.path.join(out_dir, android_variant)
|
|
EnsureJavaTestsAreBuilt(android_out_dir)
|
|
|
|
robolectric_dir = os.path.join(buildroot_dir, 'third_party', 'robolectric', 'lib')
|
|
classpath = map(str, [
|
|
os.path.join(buildroot_dir, 'third_party', 'android_tools', 'sdk', 'platforms', 'android-29', 'android.jar'),
|
|
os.path.join(robolectric_dir, '*'), # Wildcard for all jars in the directory
|
|
os.path.join(android_out_dir, 'flutter.jar'),
|
|
os.path.join(android_out_dir, 'robolectric_tests.jar')
|
|
])
|
|
|
|
test_class = filter if filter else 'io.flutter.FlutterTestSuite'
|
|
command = [
|
|
'java',
|
|
'-Drobolectric.offline=true',
|
|
'-Drobolectric.dependency.dir=' + robolectric_dir,
|
|
'-classpath', ':'.join(classpath),
|
|
'-Drobolectric.logging=stdout',
|
|
'org.junit.runner.JUnitCore',
|
|
test_class
|
|
]
|
|
|
|
RunCmd(command)
|
|
|
|
def RunDartTests(build_dir, filter, verbose_dart_snapshot):
|
|
# This one is a bit messy. The pubspec.yaml at flutter/testing/dart/pubspec.yaml
|
|
# has dependencies that are hardcoded to point to the sky packages at host_debug_unopt/
|
|
# Before running Dart tests, make sure to run just that target (NOT the whole engine)
|
|
EnsureDebugUnoptSkyPackagesAreBuilt();
|
|
|
|
# Now that we have the Sky packages at the hardcoded location, run `pub get`.
|
|
RunEngineExecutable(build_dir, os.path.join('dart-sdk', 'bin', 'pub'), None, flags=['get'], cwd=dart_tests_dir)
|
|
|
|
dart_tests = glob.glob('%s/*.dart' % dart_tests_dir)
|
|
|
|
for dart_test_file in dart_tests:
|
|
if filter is not None and os.path.basename(dart_test_file) not in filter:
|
|
print("Skipping %s due to filter." % dart_test_file)
|
|
else:
|
|
print("Testing dart file %s" % dart_test_file)
|
|
RunDartTest(build_dir, dart_test_file, verbose_dart_snapshot, True)
|
|
RunDartTest(build_dir, dart_test_file, verbose_dart_snapshot, False)
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser();
|
|
|
|
parser.add_argument('--variant', dest='variant', action='store',
|
|
default='host_debug_unopt', help='The engine build variant to run the tests for.');
|
|
parser.add_argument('--type', type=str, default='all')
|
|
parser.add_argument('--engine-filter', type=str, default='',
|
|
help='A list of engine test executables to run.')
|
|
parser.add_argument('--dart-filter', type=str, default='',
|
|
help='A list of Dart test scripts to run.')
|
|
parser.add_argument('--java-filter', type=str, default='',
|
|
help='A single Java test class to run.')
|
|
parser.add_argument('--android-variant', dest='android_variant', action='store',
|
|
default='android_debug_unopt',
|
|
help='The engine build variant to run java tests for')
|
|
parser.add_argument('--verbose-dart-snapshot', dest='verbose_dart_snapshot', action='store_true',
|
|
default=False, help='Show extra dart snapshot logging.')
|
|
|
|
args = parser.parse_args()
|
|
|
|
if args.type == 'all':
|
|
types = ['engine', 'dart', 'benchmarks', 'java']
|
|
else:
|
|
types = args.type.split(',')
|
|
|
|
build_dir = os.path.join(out_dir, args.variant)
|
|
if args.type != 'java':
|
|
assert os.path.exists(build_dir), 'Build variant directory %s does not exist!' % build_dir
|
|
|
|
engine_filter = args.engine_filter.split(',') if args.engine_filter else None
|
|
if 'engine' in types:
|
|
RunCCTests(build_dir, engine_filter)
|
|
|
|
if 'dart' in types:
|
|
assert not IsWindows(), "Dart tests can't be run on windows. https://github.com/flutter/flutter/issues/36301."
|
|
dart_filter = args.dart_filter.split(',') if args.dart_filter else None
|
|
RunDartTests(build_dir, dart_filter, args.verbose_dart_snapshot)
|
|
|
|
if 'java' in types:
|
|
assert not IsWindows(), "Android engine files can't be compiled on Windows."
|
|
java_filter = args.java_filter
|
|
if ',' in java_filter or '*' in java_filter:
|
|
print('Can only filter JUnit4 tests by single entire class name, eg "io.flutter.SmokeTest". Ignoring filter=' + java_filter)
|
|
java_filter = None
|
|
RunJavaTests(java_filter, args.android_variant)
|
|
|
|
# https://github.com/flutter/flutter/issues/36300
|
|
if 'benchmarks' in types and not IsWindows():
|
|
RunEngineBenchmarks(build_dir, engine_filter)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|