mirror of
https://github.com/flutter/flutter.git
synced 2026-02-20 02:29:02 +08:00
0a9ea0d319
* The outputs of all commands are not printed and not just commands that fail. * The stdout and stderr are now printed in order. * Clear dividers mark logs from specific subprocesses or errors. * The test whose run failed should now be exactly on top of the error message and code.
398 lines
13 KiB
Python
Executable File
398 lines
13 KiB
Python
Executable File
#!/usr/bin/env python
|
|
# Copyright 2013 The Flutter Authors. All rights reserved.
|
|
# Use of this source code is governed by a BSD-style license that can be
|
|
# found in the LICENSE file.
|
|
|
|
"""
|
|
A top level harness to run all unit-tests in a specific engine build.
|
|
"""
|
|
|
|
import argparse
|
|
import glob
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
buildroot_dir = os.path.abspath(os.path.join(os.path.realpath(__file__), '..', '..', '..'))
|
|
out_dir = os.path.join(buildroot_dir, 'out')
|
|
golden_dir = os.path.join(buildroot_dir, 'flutter', 'testing', 'resources')
|
|
fonts_dir = os.path.join(buildroot_dir, 'flutter', 'third_party', 'txt', 'third_party', 'fonts')
|
|
roboto_font_path = os.path.join(fonts_dir, 'Roboto-Regular.ttf')
|
|
dart_tests_dir = os.path.join(buildroot_dir, 'flutter', 'testing', 'dart',)
|
|
font_subset_dir = os.path.join(buildroot_dir, 'flutter', 'tools', 'font-subset')
|
|
|
|
fml_unittests_filter = '--gtest_filter=-*TimeSensitiveTest*'
|
|
|
|
def PrintDivider(char='='):
|
|
print '\n'
|
|
for _ in xrange(4):
|
|
print(''.join([char for _ in xrange(80)]))
|
|
print '\n'
|
|
|
|
def RunCmd(cmd, **kwargs):
|
|
command_string = ' '.join(cmd)
|
|
|
|
PrintDivider('>')
|
|
print 'Running command "%s"' % command_string
|
|
|
|
start_time = time.time()
|
|
process = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, **kwargs)
|
|
(output, _) = process.communicate()
|
|
end_time = time.time()
|
|
|
|
# Print the result no matter what.
|
|
for line in output.splitlines():
|
|
print line
|
|
|
|
if process.returncode != 0:
|
|
PrintDivider('!')
|
|
raise Exception('Command "%s" exited with code %d' % (command_string, process.returncode))
|
|
|
|
PrintDivider('<')
|
|
print 'Command run successfully in %.2f seconds: %s' % (end_time - start_time, command_string)
|
|
|
|
|
|
def IsMac():
|
|
return sys.platform == 'darwin'
|
|
|
|
|
|
def IsLinux():
|
|
return sys.platform.startswith('linux')
|
|
|
|
|
|
def IsWindows():
|
|
return sys.platform.startswith(('cygwin', 'win'))
|
|
|
|
|
|
def ExecutableSuffix():
|
|
return '.exe' if IsWindows() else ''
|
|
|
|
def FindExecutablePath(path):
|
|
if os.path.exists(path):
|
|
return path
|
|
|
|
if IsWindows():
|
|
exe_path = path + '.exe'
|
|
if os.path.exists(exe_path):
|
|
return exe_path
|
|
|
|
bat_path = path + '.bat'
|
|
if os.path.exists(bat_path):
|
|
return bat_path
|
|
|
|
raise Exception('Executable %s does not exist!' % path)
|
|
|
|
|
|
def RunEngineExecutable(build_dir, executable_name, filter, flags=[], cwd=buildroot_dir):
|
|
if filter is not None and executable_name not in filter:
|
|
print('Skipping %s due to filter.' % executable_name)
|
|
return
|
|
|
|
executable = FindExecutablePath(os.path.join(build_dir, executable_name))
|
|
|
|
print('Running %s in %s' % (executable_name, cwd))
|
|
test_command = [ executable ] + flags
|
|
print(' '.join(test_command))
|
|
RunCmd(test_command, cwd=cwd)
|
|
|
|
|
|
def RunCCTests(build_dir, filter):
|
|
print("Running Engine Unit-tests.")
|
|
|
|
shuffle_flags = [
|
|
"--gtest_shuffle",
|
|
"--gtest_repeat=2",
|
|
]
|
|
|
|
RunEngineExecutable(build_dir, 'client_wrapper_glfw_unittests', filter, shuffle_flags)
|
|
|
|
RunEngineExecutable(build_dir, 'client_wrapper_unittests', filter, shuffle_flags)
|
|
|
|
# https://github.com/flutter/flutter/issues/36294
|
|
if not IsWindows():
|
|
RunEngineExecutable(build_dir, 'embedder_unittests', filter, shuffle_flags)
|
|
else:
|
|
RunEngineExecutable(build_dir, 'flutter_windows_unittests', filter, shuffle_flags)
|
|
|
|
RunEngineExecutable(build_dir, 'client_wrapper_windows_unittests', filter, shuffle_flags)
|
|
|
|
flow_flags = ['--gtest_filter=-PerformanceOverlayLayer.Gold']
|
|
if IsLinux():
|
|
flow_flags = [
|
|
'--golden-dir=%s' % golden_dir,
|
|
'--font-file=%s' % roboto_font_path,
|
|
]
|
|
RunEngineExecutable(build_dir, 'flow_unittests', filter, flow_flags + shuffle_flags)
|
|
|
|
# TODO(44614): Re-enable after https://github.com/flutter/flutter/issues/44614 has been addressed.
|
|
# RunEngineExecutable(build_dir, 'fml_unittests', filter, [ fml_unittests_filter ] + shuffle_flags)
|
|
|
|
RunEngineExecutable(build_dir, 'runtime_unittests', filter, shuffle_flags)
|
|
|
|
# https://github.com/flutter/flutter/issues/36295
|
|
if not IsWindows():
|
|
RunEngineExecutable(build_dir, 'shell_unittests', filter, shuffle_flags)
|
|
|
|
RunEngineExecutable(build_dir, 'ui_unittests', filter, shuffle_flags)
|
|
|
|
RunEngineExecutable(build_dir, 'testing_unittests', filter, shuffle_flags)
|
|
|
|
# These unit-tests are Objective-C and can only run on Darwin.
|
|
if IsMac():
|
|
RunEngineExecutable(build_dir, 'flutter_channels_unittests', filter, shuffle_flags)
|
|
|
|
# https://github.com/flutter/flutter/issues/36296
|
|
if IsLinux():
|
|
RunEngineExecutable(build_dir, 'txt_unittests', filter, shuffle_flags)
|
|
|
|
|
|
def RunEngineBenchmarks(build_dir, filter):
|
|
print("Running Engine Benchmarks.")
|
|
|
|
RunEngineExecutable(build_dir, 'shell_benchmarks', filter)
|
|
|
|
RunEngineExecutable(build_dir, 'fml_benchmarks', filter)
|
|
|
|
if IsLinux():
|
|
RunEngineExecutable(build_dir, 'txt_benchmarks', filter)
|
|
|
|
|
|
|
|
def SnapshotTest(build_dir, dart_file, kernel_file_output, verbose_dart_snapshot):
|
|
print("Generating snapshot for test %s" % dart_file)
|
|
|
|
dart = os.path.join(build_dir, 'dart-sdk', 'bin', 'dart')
|
|
frontend_server = os.path.join(build_dir, 'gen', 'frontend_server.dart.snapshot')
|
|
flutter_patched_sdk = os.path.join(build_dir, 'flutter_patched_sdk')
|
|
test_packages = os.path.join(dart_tests_dir, '.packages')
|
|
|
|
assert os.path.exists(dart)
|
|
assert os.path.exists(frontend_server)
|
|
assert os.path.exists(flutter_patched_sdk)
|
|
assert os.path.exists(test_packages)
|
|
|
|
snapshot_command = [
|
|
dart,
|
|
frontend_server,
|
|
'--sdk-root',
|
|
flutter_patched_sdk,
|
|
'--incremental',
|
|
'--target=flutter',
|
|
'--packages',
|
|
test_packages,
|
|
'--output-dill',
|
|
kernel_file_output,
|
|
dart_file
|
|
]
|
|
|
|
if verbose_dart_snapshot:
|
|
RunCmd(snapshot_command, cwd=buildroot_dir)
|
|
else:
|
|
subprocess.check_output(snapshot_command, cwd=buildroot_dir)
|
|
assert os.path.exists(kernel_file_output)
|
|
|
|
|
|
def RunDartTest(build_dir, dart_file, verbose_dart_snapshot, multithreaded):
|
|
kernel_file_name = os.path.basename(dart_file) + '.kernel.dill'
|
|
kernel_file_output = os.path.join(out_dir, kernel_file_name)
|
|
|
|
SnapshotTest(build_dir, dart_file, kernel_file_output, verbose_dart_snapshot)
|
|
|
|
command_args = [
|
|
'--disable-observatory',
|
|
'--use-test-fonts',
|
|
kernel_file_output
|
|
]
|
|
|
|
if multithreaded:
|
|
threading = 'multithreaded'
|
|
command_args.insert(0, '--force-multithreading')
|
|
else:
|
|
threading = 'single-threaded'
|
|
|
|
print("Running test '%s' using 'flutter_tester' (%s)" % (kernel_file_name, threading))
|
|
RunEngineExecutable(build_dir, 'flutter_tester', None, command_args)
|
|
|
|
def RunPubGet(build_dir, directory):
|
|
print("Running 'pub get' in the tests directory %s" % dart_tests_dir)
|
|
|
|
pub_get_command = [
|
|
os.path.join(build_dir, 'dart-sdk', 'bin', 'pub'),
|
|
'get'
|
|
]
|
|
RunCmd(pub_get_command, cwd=directory)
|
|
|
|
|
|
def EnsureDebugUnoptSkyPackagesAreBuilt():
|
|
variant_out_dir = os.path.join(out_dir, 'host_debug_unopt')
|
|
|
|
ninja_command = [
|
|
'autoninja',
|
|
'-C',
|
|
variant_out_dir,
|
|
'flutter/sky/packages'
|
|
]
|
|
|
|
# Attempt running Ninja if the out directory exists.
|
|
# We don't want to blow away any custom GN args the caller may have already set.
|
|
if os.path.exists(variant_out_dir):
|
|
RunCmd(ninja_command, cwd=buildroot_dir)
|
|
return
|
|
|
|
gn_command = [
|
|
os.path.join(buildroot_dir, 'flutter', 'tools', 'gn'),
|
|
'--runtime-mode',
|
|
'debug',
|
|
'--unopt',
|
|
'--no-lto',
|
|
]
|
|
|
|
RunCmd(gn_command, cwd=buildroot_dir)
|
|
RunCmd(ninja_command, cwd=buildroot_dir)
|
|
|
|
def EnsureJavaTestsAreBuilt(android_out_dir):
|
|
ninja_command = [
|
|
'autoninja',
|
|
'-C',
|
|
android_out_dir,
|
|
'flutter/shell/platform/android:robolectric_tests'
|
|
]
|
|
|
|
# Attempt running Ninja if the out directory exists.
|
|
# We don't want to blow away any custom GN args the caller may have already set.
|
|
if os.path.exists(android_out_dir):
|
|
RunCmd(ninja_command, cwd=buildroot_dir)
|
|
return
|
|
|
|
# Otherwise prepare the directory first, then build the test.
|
|
gn_command = [
|
|
os.path.join(buildroot_dir, 'flutter', 'tools', 'gn'),
|
|
'--android',
|
|
'--unoptimized',
|
|
'--runtime-mode=debug',
|
|
'--no-lto',
|
|
]
|
|
RunCmd(gn_command, cwd=buildroot_dir)
|
|
RunCmd(ninja_command, cwd=buildroot_dir)
|
|
|
|
def AssertExpectedJavaVersion():
|
|
EXPECTED_VERSION = '1.8'
|
|
# `java -version` is output to stderr. https://bugs.java.com/bugdatabase/view_bug.do?bug_id=4380614
|
|
version_output = subprocess.check_output(['java', '-version'], stderr=subprocess.STDOUT)
|
|
match = bool(re.compile('version "%s' % EXPECTED_VERSION).search(version_output))
|
|
message = "JUnit tests need to be run with Java %s. Check the `java -version` on your PATH." % EXPECTED_VERSION
|
|
assert match, message
|
|
|
|
def RunJavaTests(filter, android_variant='android_debug_unopt'):
|
|
AssertExpectedJavaVersion()
|
|
android_out_dir = os.path.join(out_dir, android_variant)
|
|
EnsureJavaTestsAreBuilt(android_out_dir)
|
|
|
|
robolectric_dir = os.path.join(buildroot_dir, 'third_party', 'robolectric', 'lib')
|
|
classpath = map(str, [
|
|
os.path.join(buildroot_dir, 'third_party', 'android_tools', 'sdk', 'platforms', 'android-29', 'android.jar'),
|
|
os.path.join(robolectric_dir, '*'), # Wildcard for all jars in the directory
|
|
os.path.join(android_out_dir, 'flutter.jar'),
|
|
os.path.join(android_out_dir, 'robolectric_tests.jar')
|
|
])
|
|
|
|
test_class = filter if filter else 'io.flutter.FlutterTestSuite'
|
|
command = [
|
|
'java',
|
|
'-Drobolectric.offline=true',
|
|
'-Drobolectric.dependency.dir=' + robolectric_dir,
|
|
'-classpath', ':'.join(classpath),
|
|
'-Drobolectric.logging=stdout',
|
|
'org.junit.runner.JUnitCore',
|
|
test_class
|
|
]
|
|
|
|
RunCmd(command)
|
|
|
|
def RunDartTests(build_dir, filter, verbose_dart_snapshot):
|
|
# This one is a bit messy. The pubspec.yaml at flutter/testing/dart/pubspec.yaml
|
|
# has dependencies that are hardcoded to point to the sky packages at host_debug_unopt/
|
|
# Before running Dart tests, make sure to run just that target (NOT the whole engine)
|
|
EnsureDebugUnoptSkyPackagesAreBuilt();
|
|
|
|
# Now that we have the Sky packages at the hardcoded location, run `pub get`.
|
|
RunEngineExecutable(build_dir, os.path.join('dart-sdk', 'bin', 'pub'), None, flags=['get'], cwd=dart_tests_dir)
|
|
|
|
dart_tests = glob.glob('%s/*.dart' % dart_tests_dir)
|
|
|
|
for dart_test_file in dart_tests:
|
|
if filter is not None and os.path.basename(dart_test_file) not in filter:
|
|
print("Skipping %s due to filter." % dart_test_file)
|
|
else:
|
|
print("Testing dart file %s" % dart_test_file)
|
|
RunDartTest(build_dir, dart_test_file, verbose_dart_snapshot, True)
|
|
RunDartTest(build_dir, dart_test_file, verbose_dart_snapshot, False)
|
|
|
|
def RunConstFinderTests(build_dir):
|
|
test_dir = os.path.join(buildroot_dir, 'flutter', 'tools', 'const_finder', 'test')
|
|
opts = [
|
|
os.path.join(test_dir, 'const_finder_test.dart'),
|
|
os.path.join(build_dir, 'gen', 'frontend_server.dart.snapshot'),
|
|
os.path.join(build_dir, 'flutter_patched_sdk')]
|
|
RunEngineExecutable(build_dir, os.path.join('dart-sdk', 'bin', 'dart'), None, flags=opts, cwd=test_dir)
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser()
|
|
|
|
parser.add_argument('--variant', dest='variant', action='store',
|
|
default='host_debug_unopt', help='The engine build variant to run the tests for.');
|
|
parser.add_argument('--type', type=str, default='all')
|
|
parser.add_argument('--engine-filter', type=str, default='',
|
|
help='A list of engine test executables to run.')
|
|
parser.add_argument('--dart-filter', type=str, default='',
|
|
help='A list of Dart test scripts to run.')
|
|
parser.add_argument('--java-filter', type=str, default='',
|
|
help='A single Java test class to run.')
|
|
parser.add_argument('--android-variant', dest='android_variant', action='store',
|
|
default='android_debug_unopt',
|
|
help='The engine build variant to run java tests for')
|
|
parser.add_argument('--verbose-dart-snapshot', dest='verbose_dart_snapshot', action='store_true',
|
|
default=False, help='Show extra dart snapshot logging.')
|
|
|
|
args = parser.parse_args()
|
|
|
|
if args.type == 'all':
|
|
types = ['engine', 'dart', 'benchmarks', 'java', 'font-subset']
|
|
else:
|
|
types = args.type.split(',')
|
|
|
|
build_dir = os.path.join(out_dir, args.variant)
|
|
if args.type != 'java':
|
|
assert os.path.exists(build_dir), 'Build variant directory %s does not exist!' % build_dir
|
|
|
|
engine_filter = args.engine_filter.split(',') if args.engine_filter else None
|
|
if 'engine' in types:
|
|
RunCCTests(build_dir, engine_filter)
|
|
|
|
if 'dart' in types:
|
|
assert not IsWindows(), "Dart tests can't be run on windows. https://github.com/flutter/flutter/issues/36301."
|
|
dart_filter = args.dart_filter.split(',') if args.dart_filter else None
|
|
RunDartTests(build_dir, dart_filter, args.verbose_dart_snapshot)
|
|
RunConstFinderTests(build_dir)
|
|
|
|
if 'java' in types:
|
|
assert not IsWindows(), "Android engine files can't be compiled on Windows."
|
|
java_filter = args.java_filter
|
|
if ',' in java_filter or '*' in java_filter:
|
|
print('Can only filter JUnit4 tests by single entire class name, eg "io.flutter.SmokeTest". Ignoring filter=' + java_filter)
|
|
java_filter = None
|
|
RunJavaTests(java_filter, args.android_variant)
|
|
|
|
# https://github.com/flutter/flutter/issues/36300
|
|
if 'benchmarks' in types and not IsWindows():
|
|
RunEngineBenchmarks(build_dir, engine_filter)
|
|
|
|
if 'engine' in types or 'font-subset' in types:
|
|
RunCmd(['python', 'test.py'], cwd=font_subset_dir)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
sys.exit(main())
|