DEV/flutter_flutter
1
0
Fork 0
mirror of https://github.com/flutter/flutter.git synced 2026-02-20 02:29:02 +08:00
Code Issues Packages Projects Releases Wiki Activity
flutter_flutter/engine/src/flutter/testing/run_tests.py
Márk Tolmács f41c1644d8 Enabling pre-push checks on Windows (flutter/engine#36123) 
Re-submit the changes to enable windows pre-push checks.

This patch changes how `ci/bin/format.dart` generate diffs from `diff` and `patch` commands to `git diff` and `git apply` in order to have a common method for these operations on all platforms. Windows installations don't have diff and patch commands available by default and many implementations which provide such commands work differently than the UN*X tools. Git however works consistently across all platforms.

Additionally, this patch also changes the python executable in some of the pre-push components affected by this to `vpython3` to continue the effort started at flutter/flutter#108474 and I also removed the `--no-sound-null-safety` parameter in the ci/format.sh, ci/format.bat files

NOTE: Since the original patch caused some issues, I suggest that this should be tested more carefully before it is merged.

### Issues fixed by this PR
* flutter/flutter#108122
* flutter/flutter#107920
* flutter/flutter#86506
* flutter/flutter#106615

### [flutter/tests] repo impact
None.

writing and running engine tests.

[C++, Objective-C, Java style guides]: https://github.com/flutter/engine/blob/main/CONTRIBUTING.md#style
2023-06-21 17:20:06 +00:00

1287 lines
39 KiB
Python
Executable File
Raw Blame History

#!/usr/bin/env python3
#
# Copyright 2013 The Flutter Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
"""
A top level harness to run all unit-tests in a specific engine build.
"""
from pathlib import Path
import argparse
import errno
import glob
import multiprocessing
import os
import re
import shutil
import subprocess
import sys
import tempfile
import time
import typing
import xvfb
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
BUILDROOT_DIR = os.path.abspath(
os.path.join(os.path.realpath(__file__), '..', '..', '..')
)
OUT_DIR = os.path.join(BUILDROOT_DIR, 'out')
GOLDEN_DIR = os.path.join(BUILDROOT_DIR, 'flutter', 'testing', 'resources')
FONTS_DIR = os.path.join(
BUILDROOT_DIR, 'flutter', 'third_party', 'txt', 'third_party', 'fonts'
)
ROBOTO_FONT_PATH = os.path.join(FONTS_DIR, 'Roboto-Regular.ttf')
FONT_SUBSET_DIR = os.path.join(BUILDROOT_DIR, 'flutter', 'tools', 'font-subset')
FML_UNITTESTS_FILTER = '--gtest_filter=-*TimeSensitiveTest*'
ENCODING = 'UTF-8'
def print_divider(char='='):
print('\n')
for _ in range(4):
print(''.join([char for _ in range(80)]))
print('\n')
def is_asan(build_dir):
with open(os.path.join(build_dir, 'args.gn')) as args:
if 'is_asan = true' in args.read():
return True
return False
def run_cmd(
cmd: typing.List[str],
forbidden_output: typing.List[str] = None,
expect_failure: bool = False,
env: typing.Dict[str, str] = None,
allowed_failure_output: typing.List[str] = None,
**kwargs
) -> None:
if forbidden_output is None:
forbidden_output = []
if allowed_failure_output is None:
allowed_failure_output = []
command_string = ' '.join(cmd)
print_divider('>')
print(f'Running command "{command_string}"')
start_time = time.time()
process = subprocess.Popen(
cmd,
stdout=subprocess.PIPE,
stderr=subprocess.STDOUT,
env=env,
universal_newlines=True,
**kwargs
)
output = ''
for line in iter(process.stdout.readline, ''):
output += line
sys.stdout.write(line)
sys.stdout.flush()
process.wait()
end_time = time.time()
if process.returncode != 0 and not expect_failure:
print_divider('!')
print(
f'Failed Command:\n\n{command_string}\n\nExit Code: {process.returncode}\n'
)
print_divider('!')
allowed_failure = False
for allowed_string in allowed_failure_output:
if allowed_string in output:
allowed_failure = True
if not allowed_failure:
raise RuntimeError(
f'Command "{command_string}" exited with code {process.returncode}.'
)
for forbidden_string in forbidden_output:
if forbidden_string in output:
raise RuntimeError(
f'command "{command_string}" contained forbidden string {forbidden_string}'
)
print_divider('<')
print(
f'Command run successfully in {end_time - start_time:.2f} seconds: {command_string}'
)
def is_mac():
return sys.platform == 'darwin'
def is_aarm64():
assert is_mac()
output = subprocess.check_output(['sysctl', 'machdep.cpu'])
text = output.decode('utf-8')
aarm64 = text.find('Apple') >= 0
if not aarm64:
assert text.find('GenuineIntel') >= 0
return aarm64
def is_linux():
return sys.platform.startswith('linux')
def is_windows():
return sys.platform.startswith(('cygwin', 'win'))
def executable_suffix():
return '.exe' if is_windows() else ''
def find_executable_path(path):
if os.path.exists(path):
return path
if is_windows():
exe_path = path + '.exe'
if os.path.exists(exe_path):
return exe_path
bat_path = path + '.bat'
if os.path.exists(bat_path):
return bat_path
raise Exception('Executable %s does not exist!' % path)
def build_engine_executable_command(
build_dir, executable_name, flags=None, coverage=False, gtest=False
):
if flags is None:
flags = []
unstripped_exe = os.path.join(build_dir, 'exe.unstripped', executable_name)
# We cannot run the unstripped binaries directly when coverage is enabled.
if is_linux() and os.path.exists(unstripped_exe) and not coverage:
# Use unstripped executables in order to get better symbolized crash
# stack traces on Linux.
executable = unstripped_exe
else:
executable = find_executable_path(os.path.join(build_dir, executable_name))
coverage_script = os.path.join(
BUILDROOT_DIR, 'flutter', 'build', 'generate_coverage.py'
)
if coverage:
coverage_flags = [
'-t', executable, '-o',
os.path.join(build_dir, 'coverage', executable_name), '-f', 'html'
]
updated_flags = ['--args=%s' % ' '.join(flags)]
test_command = [coverage_script] + coverage_flags + updated_flags
else:
test_command = [executable] + flags
if gtest:
gtest_parallel = os.path.join(
BUILDROOT_DIR, 'third_party', 'gtest-parallel', 'gtest-parallel'
)
test_command = ['python3', gtest_parallel] + test_command
return test_command
def run_engine_executable( # pylint: disable=too-many-arguments
build_dir,
executable_name,
executable_filter,
flags=None,
cwd=BUILDROOT_DIR,
forbidden_output=None,
allowed_failure_output=None,
expect_failure=False,
coverage=False,
extra_env=None,
gtest=False,
):
if executable_filter is not None and executable_name not in executable_filter:
print('Skipping %s due to filter.' % executable_name)
return
if flags is None:
flags = []
if forbidden_output is None:
forbidden_output = []
if allowed_failure_output is None:
allowed_failure_output = []
if extra_env is None:
extra_env = {}
unstripped_exe = os.path.join(build_dir, 'exe.unstripped', executable_name)
env = os.environ.copy()
if is_linux():
env['LD_LIBRARY_PATH'] = build_dir
env['VK_DRIVER_FILES'] = os.path.join(build_dir, 'vk_swiftshader_icd.json')
if os.path.exists(unstripped_exe):
try:
os.symlink(
os.path.join(build_dir, 'lib.unstripped', 'libvulkan.so.1'),
os.path.join(build_dir, 'exe.unstripped', 'libvulkan.so.1')
)
except OSError as err:
if err.errno == errno.EEXIST:
pass
else:
raise
elif is_mac():
env['DYLD_LIBRARY_PATH'] = build_dir
else:
env['PATH'] = build_dir + ':' + env['PATH']
print('Running %s in %s' % (executable_name, cwd))
test_command = build_engine_executable_command(
build_dir,
executable_name,
flags=flags,
coverage=coverage,
gtest=gtest,
)
env['FLUTTER_BUILD_DIRECTORY'] = build_dir
for key, value in extra_env.items():
env[key] = value
try:
run_cmd(
test_command,
cwd=cwd,
forbidden_output=forbidden_output,
expect_failure=expect_failure,
env=env,
allowed_failure_output=allowed_failure_output,
)
except:
# The LUCI environment may provide a variable containing a directory path
# for additional output files that will be uploaded to cloud storage.
# If the command generated a core dump, then run a script to analyze
# the dump and output a report that will be uploaded.
luci_test_outputs_path = os.environ.get('FLUTTER_TEST_OUTPUTS_DIR')
core_path = os.path.join(cwd, 'core')
if luci_test_outputs_path and os.path.exists(core_path) and os.path.exists(
unstripped_exe):
dump_path = os.path.join(
luci_test_outputs_path, '%s_%s.txt' % (executable_name, sys.platform)
)
print('Writing core dump analysis to %s' % dump_path)
subprocess.call([
os.path.join(
BUILDROOT_DIR, 'flutter', 'testing', 'analyze_core_dump.sh'
),
BUILDROOT_DIR,
unstripped_exe,
core_path,
dump_path,
])
os.unlink(core_path)
raise
class EngineExecutableTask(): # pylint: disable=too-many-instance-attributes
def __init__( # pylint: disable=too-many-arguments
self,
build_dir,
executable_name,
executable_filter,
flags=None,
cwd=BUILDROOT_DIR,
forbidden_output=None,
allowed_failure_output=None,
expect_failure=False,
coverage=False,
extra_env=None,
):
self.build_dir = build_dir
self.executable_name = executable_name
self.executable_filter = executable_filter
self.flags = flags
self.cwd = cwd
self.forbidden_output = forbidden_output
self.allowed_failure_output = allowed_failure_output
self.expect_failure = expect_failure
self.coverage = coverage
self.extra_env = extra_env
def __call__(self, *args):
run_engine_executable(
self.build_dir,
self.executable_name,
self.executable_filter,
flags=self.flags,
cwd=self.cwd,
forbidden_output=self.forbidden_output,
allowed_failure_output=self.allowed_failure_output,
expect_failure=self.expect_failure,
coverage=self.coverage,
extra_env=self.extra_env,
)
def __str__(self):
command = build_engine_executable_command(
self.build_dir,
self.executable_name,
flags=self.flags,
coverage=self.coverage
)
return ' '.join(command)
shuffle_flags = [
'--gtest_repeat=2',
'--gtest_shuffle',
]
def run_cc_tests(build_dir, executable_filter, coverage, capture_core_dump):
print('Running Engine Unit-tests.')
if capture_core_dump and is_linux():
import resource # pylint: disable=import-outside-toplevel
resource.setrlimit(
resource.RLIMIT_CORE, (resource.RLIM_INFINITY, resource.RLIM_INFINITY)
)
repeat_flags = [
'--repeat=2',
]
def make_test(name, flags=None, extra_env=None):
if flags is None:
flags = repeat_flags
if extra_env is None:
extra_env = {}
return (name, flags, extra_env)
unittests = [
make_test('client_wrapper_glfw_unittests'),
make_test('client_wrapper_unittests'),
make_test('common_cpp_core_unittests'),
make_test('common_cpp_unittests'),
make_test('dart_plugin_registrant_unittests'),
make_test('display_list_rendertests'),
make_test('display_list_unittests'),
make_test('embedder_a11y_unittests'),
make_test('embedder_proctable_unittests'),
make_test('embedder_unittests'),
make_test('fml_unittests', flags=[FML_UNITTESTS_FILTER] + repeat_flags),
make_test('no_dart_plugin_registrant_unittests'),
make_test('runtime_unittests'),
make_test('testing_unittests'),
make_test('tonic_unittests'),
# The image release unit test can take a while on slow machines.
make_test('ui_unittests', flags=repeat_flags + ['--timeout=90']),
]
if not is_windows():
unittests += [
# https://github.com/google/googletest/issues/2490
make_test('android_external_view_embedder_unittests'),
make_test('jni_unittests'),
make_test('platform_view_android_delegate_unittests'),
# https://github.com/flutter/flutter/issues/36295
make_test('shell_unittests'),
]
if is_windows():
unittests += [
# The accessibility library only supports Mac and Windows.
make_test('accessibility_unittests'),
make_test('client_wrapper_windows_unittests'),
make_test('flutter_windows_unittests'),
]
# These unit-tests are Objective-C and can only run on Darwin.
if is_mac():
unittests += [
# The accessibility library only supports Mac and Windows.
make_test('accessibility_unittests'),
make_test('flutter_channels_unittests'),
make_test('spring_animation_unittests'),
]
if is_linux():
flow_flags = [
'--golden-dir=%s' % GOLDEN_DIR,
'--font-file=%s' % ROBOTO_FONT_PATH,
]
icu_flags = [
'--icu-data-file-path=%s' % os.path.join(build_dir, 'icudtl.dat')
]
unittests += [
make_test('flow_unittests', flags=repeat_flags + ['--'] + flow_flags),
make_test('flutter_glfw_unittests'),
make_test(
'flutter_linux_unittests', extra_env={'G_DEBUG': 'fatal-criticals'}
),
# https://github.com/flutter/flutter/issues/36296
make_test('txt_unittests', flags=repeat_flags + ['--'] + icu_flags),
]
else:
flow_flags = ['--gtest_filter=-PerformanceOverlayLayer.Gold']
unittests += [
make_test('flow_unittests', flags=repeat_flags + flow_flags),
]
for test, flags, extra_env in unittests:
run_engine_executable(
build_dir,
test,
executable_filter,
flags,
coverage=coverage,
extra_env=extra_env,
gtest=True
)
if is_mac():
# flutter_desktop_darwin_unittests uses global state that isn't handled
# correctly by gtest-parallel.
# https://github.com/flutter/flutter/issues/104789
if not os.path.basename(build_dir).startswith('host_debug'):
# Test is disabled for flaking in debug runs:
# https://github.com/flutter/flutter/issues/127441
run_engine_executable(
build_dir,
'flutter_desktop_darwin_unittests',
executable_filter,
shuffle_flags,
coverage=coverage
)
extra_env = {
# pylint: disable=line-too-long
# See https://developer.apple.com/documentation/metal/diagnosing_metal_programming_issues_early?language=objc
'MTL_SHADER_VALIDATION': '1', # Enables all shader validation tests.
'MTL_SHADER_VALIDATION_GLOBAL_MEMORY':
'1', # Validates accesses to device and constant memory.
'MTL_SHADER_VALIDATION_THREADGROUP_MEMORY':
'1', # Validates accesses to threadgroup memory.
'MTL_SHADER_VALIDATION_TEXTURE_USAGE':
'1', # Validates that texture references are not nil.
# Note: built from //third_party/swiftshader
'VK_ICD_FILENAMES': os.path.join(build_dir, 'vk_swiftshader_icd.json'),
# Note: built from //third_party/vulkan_validation_layers:vulkan_gen_json_files
# and //third_party/vulkan_validation_layers.
'VK_LAYER_PATH': os.path.join(build_dir, 'vulkan-data'),
'VK_INSTANCE_LAYERS': 'VK_LAYER_KHRONOS_validation',
}
if is_aarm64():
extra_env.update({
'METAL_DEBUG_ERROR_MODE': '0', # Enables metal validation.
'METAL_DEVICE_WRAPPER_TYPE': '1', # Enables metal validation.
})
# Impeller tests are only supported on macOS for now.
run_engine_executable(
build_dir,
'impeller_unittests',
executable_filter,
shuffle_flags + ['--enable_vulkan_validation'],
coverage=coverage,
extra_env=extra_env,
# TODO(117122): Remove this allowlist.
# https://github.com/flutter/flutter/issues/114872
allowed_failure_output=[
'[MTLCompiler createVertexStageAndLinkPipelineWithFragment:',
'[MTLCompiler pipelineStateWithVariant:',
]
)
def run_engine_benchmarks(build_dir, executable_filter):
print('Running Engine Benchmarks.')
icu_flags = [
'--icu-data-file-path=%s' % os.path.join(build_dir, 'icudtl.dat')
]
run_engine_executable(
build_dir, 'shell_benchmarks', executable_filter, icu_flags
)
run_engine_executable(
build_dir, 'fml_benchmarks', executable_filter, icu_flags
)
run_engine_executable(
build_dir, 'ui_benchmarks', executable_filter, icu_flags
)
run_engine_executable(
build_dir, 'display_list_builder_benchmarks', executable_filter, icu_flags
)
run_engine_executable(
build_dir, 'geometry_benchmarks', executable_filter, icu_flags
)
if is_linux():
run_engine_executable(
build_dir, 'txt_benchmarks', executable_filter, icu_flags
)
def gather_dart_test(
build_dir,
dart_file,
multithreaded,
enable_observatory=False,
expect_failure=False,
):
kernel_file_name = os.path.basename(dart_file) + '.dill'
kernel_file_output = os.path.join(build_dir, 'gen', kernel_file_name)
error_message = "%s doesn't exist. Please run the build that populates %s" % (
kernel_file_output, build_dir
)
assert os.path.isfile(kernel_file_output), error_message
command_args = []
if not enable_observatory:
command_args.append('--disable-observatory')
dart_file_contents = open(dart_file, 'r')
custom_options = re.findall(
'// FlutterTesterOptions=(.*)', dart_file_contents.read()
)
dart_file_contents.close()
command_args.extend(custom_options)
command_args += [
'--use-test-fonts',
'--icu-data-file-path=%s' % os.path.join(build_dir, 'icudtl.dat'),
'--flutter-assets-dir=%s' %
os.path.join(build_dir, 'gen', 'flutter', 'lib', 'ui', 'assets'),
'--disable-asset-fonts',
kernel_file_output,
]
if multithreaded:
threading = 'multithreaded'
command_args.insert(0, '--force-multithreading')
else:
threading = 'single-threaded'
tester_name = 'flutter_tester'
print(
"Running test '%s' using '%s' (%s)" %
(kernel_file_name, tester_name, threading)
)
forbidden_output = [] if 'unopt' in build_dir or expect_failure else [
'[ERROR'
]
return EngineExecutableTask(
build_dir,
tester_name,
None,
command_args,
forbidden_output=forbidden_output,
expect_failure=expect_failure,
)
def ensure_ios_tests_are_built(ios_out_dir):
"""Builds the engine variant and the test dylib containing the XCTests"""
tmp_out_dir = os.path.join(OUT_DIR, ios_out_dir)
ios_test_lib = os.path.join(tmp_out_dir, 'libios_test_flutter.dylib')
message = []
message.append(
'gn --ios --unoptimized --runtime-mode=debug --no-lto --simulator'
)
message.append('ninja -C %s ios_test_flutter' % ios_out_dir)
final_message = "%s or %s doesn't exist. Please run the following commands: \n%s" % (
ios_out_dir, ios_test_lib, '\n'.join(message)
)
assert os.path.exists(tmp_out_dir
) and os.path.exists(ios_test_lib), final_message
ios_test_lib_time = os.path.getmtime(ios_test_lib)
flutter_dylib = os.path.join(tmp_out_dir, 'libFlutter.dylib')
flutter_dylib_time = os.path.getmtime(flutter_dylib)
final_message = '%s is older than %s. Please run the following commands: \n%s' % (
ios_test_lib, flutter_dylib, '\n'.join(message)
)
assert flutter_dylib_time <= ios_test_lib_time, final_message
def assert_expected_xcode_version():
"""Checks that the user has a version of Xcode installed"""
version_output = subprocess.check_output(['xcodebuild', '-version'])
# TODO ricardoamador: remove this check when python 2 is deprecated.
version_output = version_output if isinstance(
version_output, str
) else version_output.decode(ENCODING)
version_output = version_output.strip()
match = re.match(r'Xcode (\d+)', version_output)
message = 'Xcode must be installed to run the iOS embedding unit tests'
assert match, message
def java_home():
script_path = os.path.dirname(os.path.realpath(__file__))
if is_mac():
return os.path.join(
script_path, '..', '..', 'third_party', 'java', 'openjdk', 'Contents',
'Home'
)
return os.path.join(script_path, '..', '..', 'third_party', 'java', 'openjdk')
def java_bin():
return os.path.join(
java_home(), 'bin', 'java.exe' if is_windows() else 'java'
)
def run_java_tests(executable_filter, android_variant='android_debug_unopt'):
"""Runs the Java JUnit unit tests for the Android embedding"""
test_runner_dir = os.path.join(
BUILDROOT_DIR, 'flutter', 'shell', 'platform', 'android', 'test_runner'
)
gradle_bin = os.path.join(
BUILDROOT_DIR, 'third_party', 'gradle', 'bin',
'gradle.bat' if is_windows() else 'gradle'
)
flutter_jar = os.path.join(OUT_DIR, android_variant, 'flutter.jar')
android_home = os.path.join(
BUILDROOT_DIR, 'third_party', 'android_tools', 'sdk'
)
build_dir = os.path.join(
OUT_DIR, android_variant, 'robolectric_tests', 'build'
)
gradle_cache_dir = os.path.join(
OUT_DIR, android_variant, 'robolectric_tests', '.gradle'
)
test_class = executable_filter if executable_filter else '*'
command = [
gradle_bin,
'-Pflutter_jar=%s' % flutter_jar,
'-Pbuild_dir=%s' % build_dir,
'testDebugUnitTest',
'--tests=%s' % test_class,
'--rerun-tasks',
'--no-daemon',
'--project-cache-dir=%s' % gradle_cache_dir,
'--gradle-user-home=%s' % gradle_cache_dir,
]
env = dict(os.environ, ANDROID_HOME=android_home, JAVA_HOME=java_home())
run_cmd(command, cwd=test_runner_dir, env=env)
def run_android_tests(android_variant='android_debug_unopt', adb_path=None):
test_runner_name = 'flutter_shell_native_unittests'
tests_path = os.path.join(OUT_DIR, android_variant, test_runner_name)
remote_path = '/data/local/tmp'
remote_tests_path = os.path.join(remote_path, test_runner_name)
if adb_path is None:
adb_path = 'adb'
run_cmd([adb_path, 'push', tests_path, remote_path], cwd=BUILDROOT_DIR)
run_cmd([adb_path, 'shell', remote_tests_path])
systrace_test = os.path.join(
BUILDROOT_DIR, 'flutter', 'testing', 'android_systrace_test.py'
)
scenario_apk = os.path.join(
OUT_DIR, android_variant, 'firebase_apks', 'scenario_app.apk'
)
run_cmd([
systrace_test, '--adb-path', adb_path, '--apk-path', scenario_apk,
'--package-name', 'dev.flutter.scenarios', '--activity-name',
'.PlatformViewsActivity'
])
def run_objc_tests(ios_variant='ios_debug_sim_unopt', test_filter=None):
"""Runs Objective-C XCTest unit tests for the iOS embedding"""
assert_expected_xcode_version()
ios_out_dir = os.path.join(OUT_DIR, ios_variant)
ensure_ios_tests_are_built(ios_out_dir)
new_simulator_name = 'IosUnitTestsSimulator'
# Delete simulators with this name in case any were leaked
# from another test run.
delete_simulator(new_simulator_name)
create_simulator = [
'xcrun '
'simctl '
'create '
'%s com.apple.CoreSimulator.SimDeviceType.iPhone-11' % new_simulator_name
]
run_cmd(create_simulator, shell=True)
try:
ios_unit_test_dir = os.path.join(
BUILDROOT_DIR, 'flutter', 'testing', 'ios', 'IosUnitTests'
)
with tempfile.TemporaryDirectory(suffix='ios_embedding_xcresult'
) as result_bundle_temp:
result_bundle_path = os.path.join(result_bundle_temp, 'ios_embedding')
# Avoid using xcpretty unless the following can be addressed:
# - Make sure all relevant failure output is printed on a failure.
# - Make sure that a failing exit code is set for CI.
# See https://github.com/flutter/flutter/issues/63742
test_command = [
'xcodebuild '
'-sdk iphonesimulator '
'-scheme IosUnitTests '
'-resultBundlePath ' + result_bundle_path + ' '
'-destination name=' + new_simulator_name + ' '
'test '
'FLUTTER_ENGINE=' + ios_variant
]
if test_filter is not None:
test_command[0] = test_command[0] + ' -only-testing:%s' % test_filter
try:
run_cmd(test_command, cwd=ios_unit_test_dir, shell=True)
except:
# The LUCI environment may provide a variable containing a directory path
# for additional output files that will be uploaded to cloud storage.
# Upload the xcresult when the tests fail.
luci_test_outputs_path = os.environ.get('FLUTTER_TEST_OUTPUTS_DIR')
xcresult_bundle = os.path.join(
result_bundle_temp, 'ios_embedding.xcresult'
)
if luci_test_outputs_path and os.path.exists(xcresult_bundle):
dump_path = os.path.join(
luci_test_outputs_path, 'ios_embedding.xcresult'
)
# xcresults contain many little files. Archive the bundle before upload.
shutil.make_archive(dump_path, 'zip', root_dir=xcresult_bundle)
raise
finally:
delete_simulator(new_simulator_name)
def delete_simulator(simulator_name):
# Will delete all simulators with this name.
command = [
'xcrun',
'simctl',
'delete',
simulator_name,
]
# Let this fail if the simulator was never created.
run_cmd(command, expect_failure=True)
def gather_dart_tests(build_dir, test_filter):
dart_tests_dir = os.path.join(
BUILDROOT_DIR,
'flutter',
'testing',
'dart',
)
# Now that we have the Sky packages at the hardcoded location, run `dart pub get`.
run_engine_executable(
build_dir,
os.path.join('dart-sdk', 'bin', 'dart'),
None,
flags=['pub', 'get', '--offline'],
cwd=dart_tests_dir,
)
dart_observatory_tests = glob.glob(
'%s/observatory/*_test.dart' % dart_tests_dir
)
dart_tests = glob.glob('%s/*_test.dart' % dart_tests_dir)
if 'release' not in build_dir:
for dart_test_file in dart_observatory_tests:
if test_filter is not None and os.path.basename(dart_test_file
) not in test_filter:
print("Skipping '%s' due to filter." % dart_test_file)
else:
print(
"Gathering dart test '%s' with observatory enabled" % dart_test_file
)
yield gather_dart_test(build_dir, dart_test_file, True, True)
yield gather_dart_test(build_dir, dart_test_file, False, True)
for dart_test_file in dart_tests:
if test_filter is not None and os.path.basename(dart_test_file
) not in test_filter:
print("Skipping '%s' due to filter." % dart_test_file)
else:
print("Gathering dart test '%s'" % dart_test_file)
yield gather_dart_test(build_dir, dart_test_file, True)
yield gather_dart_test(build_dir, dart_test_file, False)
def gather_dart_smoke_test(build_dir):
smoke_test = os.path.join(
BUILDROOT_DIR, 'flutter', 'testing', 'smoke_test_failure',
'fail_test.dart'
)
yield gather_dart_test(build_dir, smoke_test, True, expect_failure=True)
yield gather_dart_test(build_dir, smoke_test, False, expect_failure=True)
def gather_front_end_server_tests(build_dir):
test_dir = os.path.join(BUILDROOT_DIR, 'flutter', 'flutter_frontend_server')
dart_tests = glob.glob('%s/test/*_test.dart' % test_dir)
for dart_test_file in dart_tests:
opts = [
'--disable-dart-dev', dart_test_file, build_dir,
os.path.join(build_dir, 'gen', 'frontend_server.dart.snapshot'),
os.path.join(build_dir, 'flutter_patched_sdk')
]
yield EngineExecutableTask(
build_dir,
os.path.join('dart-sdk', 'bin', 'dart'),
None,
flags=opts,
cwd=test_dir
)
def gather_path_ops_tests(build_dir):
# TODO(dnfield): https://github.com/flutter/flutter/issues/107321
if is_asan(build_dir):
return
test_dir = os.path.join(
BUILDROOT_DIR, 'flutter', 'tools', 'path_ops', 'dart', 'test'
)
opts = ['--disable-dart-dev', os.path.join(test_dir, 'path_ops_test.dart')]
yield EngineExecutableTask(
build_dir,
os.path.join('dart-sdk', 'bin', 'dart'),
None,
flags=opts,
cwd=test_dir
)
def gather_const_finder_tests(build_dir):
test_dir = os.path.join(
BUILDROOT_DIR, 'flutter', 'tools', 'const_finder', 'test'
)
opts = [
'--disable-dart-dev',
os.path.join(test_dir, 'const_finder_test.dart'),
os.path.join(build_dir, 'gen', 'frontend_server.dart.snapshot'),
os.path.join(build_dir, 'flutter_patched_sdk'),
os.path.join(build_dir, 'dart-sdk', 'lib', 'libraries.json')
]
yield EngineExecutableTask(
build_dir,
os.path.join('dart-sdk', 'bin', 'dart'),
None,
flags=opts,
cwd=test_dir
)
def gather_litetest_tests(build_dir):
test_dir = os.path.join(BUILDROOT_DIR, 'flutter', 'testing', 'litetest')
dart_tests = glob.glob('%s/test/*_test.dart' % test_dir)
for dart_test_file in dart_tests:
opts = ['--disable-dart-dev', dart_test_file]
yield EngineExecutableTask(
build_dir,
os.path.join('dart-sdk', 'bin', 'dart'),
None,
flags=opts,
cwd=test_dir
)
def run_benchmark_tests(build_dir):
test_dir = os.path.join(BUILDROOT_DIR, 'flutter', 'testing', 'benchmark')
dart_tests = glob.glob('%s/test/*_test.dart' % test_dir)
for dart_test_file in dart_tests:
opts = ['--disable-dart-dev', dart_test_file]
run_engine_executable(
build_dir,
os.path.join('dart-sdk', 'bin', 'dart'),
None,
flags=opts,
cwd=test_dir
)
def gather_githooks_tests(build_dir):
test_dir = os.path.join(BUILDROOT_DIR, 'flutter', 'tools', 'githooks')
dart_tests = glob.glob('%s/test/*_test.dart' % test_dir)
for dart_test_file in dart_tests:
opts = ['--disable-dart-dev', dart_test_file]
yield EngineExecutableTask(
build_dir,
os.path.join('dart-sdk', 'bin', 'dart'),
None,
flags=opts,
cwd=test_dir
)
def gather_clang_tidy_tests(build_dir):
test_dir = os.path.join(BUILDROOT_DIR, 'flutter', 'tools', 'clang_tidy')
dart_tests = glob.glob('%s/test/*_test.dart' % test_dir)
for dart_test_file in dart_tests:
opts = [
'--disable-dart-dev', dart_test_file,
os.path.join(build_dir, 'compile_commands.json'),
os.path.join(BUILDROOT_DIR, 'flutter')
]
yield EngineExecutableTask(
build_dir,
os.path.join('dart-sdk', 'bin', 'dart'),
None,
flags=opts,
cwd=test_dir
)
def gather_api_consistency_tests(build_dir):
test_dir = os.path.join(BUILDROOT_DIR, 'flutter', 'tools', 'api_check')
dart_tests = glob.glob('%s/test/*_test.dart' % test_dir)
for dart_test_file in dart_tests:
opts = [
'--disable-dart-dev', dart_test_file,
os.path.join(BUILDROOT_DIR, 'flutter')
]
yield EngineExecutableTask(
build_dir,
os.path.join('dart-sdk', 'bin', 'dart'),
None,
flags=opts,
cwd=test_dir
)
def gather_ci_tests(build_dir):
test_dir = os.path.join(BUILDROOT_DIR, 'flutter', 'ci')
dart_tests = glob.glob('%s/test/*_test.dart' % test_dir)
run_engine_executable(
build_dir,
os.path.join('dart-sdk', 'bin', 'dart'),
None,
flags=['pub', 'get', '--offline'],
cwd=test_dir,
)
for dart_test_file in dart_tests:
opts = [
'--disable-dart-dev', dart_test_file,
os.path.join(BUILDROOT_DIR, 'flutter')
]
yield EngineExecutableTask(
build_dir,
os.path.join('dart-sdk', 'bin', 'dart'),
None,
flags=opts,
cwd=test_dir
)
def run_engine_tasks_in_parallel(tasks):
# Work around a bug in Python.
#
# The multiprocessing package relies on the win32 WaitForMultipleObjects()
# call, which supports waiting on a maximum of MAXIMUM_WAIT_OBJECTS (defined
# by Windows to be 64) handles, processes in this case. To avoid hitting
# this, we limit ourselves to 60 handles (since there are a couple extra
# processes launched for the queue reader and thread wakeup reader).
#
# See: https://bugs.python.org/issue26903
max_processes = multiprocessing.cpu_count()
if sys.platform.startswith(('cygwin', 'win')) and max_processes > 60:
max_processes = 60
pool = multiprocessing.Pool(processes=max_processes)
async_results = [(t, pool.apply_async(t, ())) for t in tasks]
failures = []
for task, async_result in async_results:
try:
async_result.get()
except Exception as exn: # pylint: disable=broad-except
failures += [(task, exn)]
if len(failures) > 0:
print('The following commands failed:')
for task, exn in failures:
print('%s\n%s\n' % (str(task), str(exn)))
raise Exception()
class DirectoryChange():
"""
A scoped change in the CWD.
"""
old_cwd: str = ''
new_cwd: str = ''
def __init__(self, new_cwd: str):
self.new_cwd = new_cwd
def __enter__(self):
self.old_cwd = os.getcwd()
os.chdir(self.new_cwd)
def __exit__(self, exception_type, exception_value, exception_traceback):
os.chdir(self.old_cwd)
def run_impeller_golden_tests(build_dir: str):
"""
Executes the impeller golden image tests from in the `variant` build.
"""
tests_path: str = os.path.join(build_dir, 'impeller_golden_tests')
if not os.path.exists(tests_path):
raise Exception(
'Cannot find the "impeller_golden_tests" executable in "%s". You may need to build it.'
% (build_dir)
)
harvester_path: Path = Path(SCRIPT_DIR).parent.joinpath('impeller').joinpath(
'golden_tests_harvester'
)
with tempfile.TemporaryDirectory(prefix='impeller_golden') as temp_dir:
run_cmd([tests_path, '--working_dir=%s' % temp_dir])
with DirectoryChange(harvester_path):
run_cmd(['dart', 'pub', 'get'])
bin_path = Path('.').joinpath('bin'
).joinpath('golden_tests_harvester.dart')
run_cmd(['dart', 'run', str(bin_path), temp_dir])
def main():
parser = argparse.ArgumentParser(
description="""
In order to learn the details of running tests in the engine, please consult the
Flutter Wiki page on the subject: https://github.com/flutter/flutter/wiki/Testing-the-engine
"""
)
all_types = [
'engine',
'dart',
'benchmarks',
'java',
'android',
'objc',
'font-subset',
'impeller-golden',
]
parser.add_argument(
'--variant',
dest='variant',
action='store',
default='host_debug_unopt',
help='The engine build variant to run the tests for.'
)
parser.add_argument(
'--type',
type=str,
default='all',
help='A list of test types, default is "all" (equivalent to "%s")' %
(','.join(all_types))
)
parser.add_argument(
'--engine-filter',
type=str,
default='',
help='A list of engine test executables to run.'
)
parser.add_argument(
'--dart-filter',
type=str,
default='',
help='A list of Dart test scripts to run.'
)
parser.add_argument(
'--java-filter',
type=str,
default='',
help='A single Java test class to run (example: "io.flutter.SmokeTest")'
)
parser.add_argument(
'--android-variant',
dest='android_variant',
action='store',
default='android_debug_unopt',
help='The engine build variant to run java or android tests for'
)
parser.add_argument(
'--ios-variant',
dest='ios_variant',
action='store',
default='ios_debug_sim_unopt',
help='The engine build variant to run objective-c tests for'
)
parser.add_argument(
'--verbose-dart-snapshot',
dest='verbose_dart_snapshot',
action='store_true',
default=False,
help='Show extra dart snapshot logging.'
)
parser.add_argument(
'--objc-filter',
type=str,
default=None,
help=(
'Filter parameter for which objc tests to run '
'(example: "IosUnitTestsTests/SemanticsObjectTest/testShouldTriggerAnnouncement")'
)
)
parser.add_argument(
'--coverage',
action='store_true',
default=None,
help='Generate coverage reports for each unit test framework run.'
)
parser.add_argument(
'--engine-capture-core-dump',
dest='engine_capture_core_dump',
action='store_true',
default=False,
help='Capture core dumps from crashes of engine tests.'
)
parser.add_argument(
'--use-sanitizer-suppressions',
dest='sanitizer_suppressions',
action='store_true',
default=False,
help='Provide the sanitizer suppressions lists to the via environment to the tests.'
)
parser.add_argument(
'--adb-path',
dest='adb_path',
action='store',
default=None,
help='Provide the path of adb used for android tests. By default it looks on $PATH.'
)
args = parser.parse_args()
if args.type == 'all':
types = all_types
else:
types = args.type.split(',')
build_dir = os.path.join(OUT_DIR, args.variant)
if args.type != 'java' and args.type != 'android':
assert os.path.exists(
build_dir
), 'Build variant directory %s does not exist!' % build_dir
if args.sanitizer_suppressions:
assert is_linux() or is_mac(
), 'The sanitizer suppressions flag is only supported on Linux and Mac.'
file_dir = os.path.dirname(os.path.abspath(__file__))
command = [
'env', '-i', 'bash', '-c',
'source {}/sanitizer_suppressions.sh >/dev/null && env'
.format(file_dir)
]
process = subprocess.Popen(command, stdout=subprocess.PIPE)
for line in process.stdout:
key, _, value = line.decode('utf8').strip().partition('=')
os.environ[key] = value
process.communicate() # Avoid pipe deadlock while waiting for termination.
engine_filter = args.engine_filter.split(',') if args.engine_filter else None
if 'engine' in types:
run_cc_tests(
build_dir, engine_filter, args.coverage, args.engine_capture_core_dump
)
# Use this type to exclusively run impeller vulkan tests.
if 'impeller-vulkan' in types:
build_name = args.variant
try:
xvfb.start_virtual_x(build_name, build_dir)
run_engine_executable(
build_dir,
'impeller_unittests',
engine_filter,
shuffle_flags + ['--gtest_filter=-'
'*/OpenGLES:'],
coverage=args.coverage
)
finally:
xvfb.stop_virtual_x(build_name)
if 'dart' in types:
dart_filter = args.dart_filter.split(',') if args.dart_filter else None
tasks = list(gather_dart_smoke_test(build_dir))
tasks += list(gather_litetest_tests(build_dir))
tasks += list(gather_githooks_tests(build_dir))
tasks += list(gather_clang_tidy_tests(build_dir))
tasks += list(gather_api_consistency_tests(build_dir))
tasks += list(gather_path_ops_tests(build_dir))
tasks += list(gather_const_finder_tests(build_dir))
tasks += list(gather_front_end_server_tests(build_dir))
tasks += list(gather_ci_tests(build_dir))
tasks += list(gather_dart_tests(build_dir, dart_filter))
run_engine_tasks_in_parallel(tasks)
if 'java' in types:
assert not is_windows(
), "Android engine files can't be compiled on Windows."
java_filter = args.java_filter
if ',' in java_filter or '*' in java_filter:
print(
'Can only filter JUnit4 tests by single entire class name, '
'eg "io.flutter.SmokeTest". Ignoring filter=' + java_filter
)
java_filter = None
run_java_tests(java_filter, args.android_variant)
if 'android' in types:
assert not is_windows(
), "Android engine files can't be compiled on Windows."
run_android_tests(args.android_variant, args.adb_path)
if 'objc' in types:
assert is_mac(), 'iOS embedding tests can only be run on macOS.'
run_objc_tests(args.ios_variant, args.objc_filter)
# https://github.com/flutter/flutter/issues/36300
if 'benchmarks' in types and not is_windows():
run_benchmark_tests(build_dir)
run_engine_benchmarks(build_dir, engine_filter)
variants_to_skip = ['host_release', 'host_profile']
if ('engine' in types or
'font-subset' in types) and args.variant not in variants_to_skip:
run_cmd(['python3', 'test.py'], cwd=FONT_SUBSET_DIR)
if 'impeller-golden' in types:
run_impeller_golden_tests(build_dir)
if __name__ == '__main__':
sys.exit(main())
Reference in New Issue View Git Blame Copy Permalink