mirror of
https://github.com/flutter/flutter.git
synced 2026-02-20 02:29:02 +08:00
Convert run_tests to python, allow running on Mac/Windows and allow filters for tests. (#9818)
Sample usage: To run only the embedder_unittests in the engine with the profile variant, the command would be ``` ./flutter/testing/run_tests.py --variant host_profile_unopt --type engine --filter embedder_unittests `` To run only the geometry in Dart with the debug variant, the command would be ``` ./flutter/testing/run_tests.py --variant host_debug_unopt --type dart --filter geometry_test `` Without any argument, the behavior is identical to `run_tests.sh`. In a subsequent patch, I will enable running unit-tests on Windows in the tryjobs. The lack of compatibility of the shell script on Windows made it so that we never ran any Windows unit-tests in the tryjobs.
This commit is contained in:
Chinmay Garde
GitHub
parent
dd937e0c4d
commit
a59b6a6e38
223
testing/run_tests.py
Executable file
223
testing/run_tests.py
Executable file
@ -0,0 +1,223 @@
|
||||
#!/usr/bin/env python
|
||||
# Copyright 2013 The Flutter Authors. All rights reserved.
|
||||
# Use of this source code is governed by a BSD-style license that can be
|
||||
# found in the LICENSE file.
|
||||
|
||||
"""
|
||||
A top level harness to run all unit-tests in a specific engine build.
|
||||
"""
|
||||
|
||||
import sys
|
||||
import os
|
||||
import argparse
|
||||
import glob
|
||||
import subprocess
|
||||
|
||||
buildroot_dir = os.path.abspath(os.path.join(os.path.realpath(__file__), '..', '..', '..'))
|
||||
out_dir = os.path.join(buildroot_dir, 'out')
|
||||
fonts_dir = os.path.join(buildroot_dir, 'flutter', 'third_party', 'txt', 'third_party', 'fonts')
|
||||
dart_tests_dir = os.path.join(buildroot_dir, 'flutter', 'testing', 'dart',)
|
||||
|
||||
fonts_dir_flag = '--font-directory=%s' % fonts_dir
|
||||
time_sensitve_test_flag = '--gtest_filter="-*TimeSensitiveTest*"'
|
||||
|
||||
def IsMac():
|
||||
sys.platform == 'darwin'
|
||||
|
||||
|
||||
def IsLinux():
|
||||
sys.platform.startswith('linux')
|
||||
|
||||
|
||||
def IsWindows():
|
||||
sys.platform.startswith(('cygwin', 'win'))
|
||||
|
||||
|
||||
def RunEngineExecutable(build_dir, executable_name, filter, flags=[], cwd=buildroot_dir):
|
||||
if not filter in executable_name:
|
||||
print 'Skipping %s due to filter.' % executable_name
|
||||
return
|
||||
|
||||
print 'Running %s in %s' % (executable_name, cwd)
|
||||
|
||||
executable = os.path.join(build_dir, executable_name)
|
||||
assert os.path.exists(executable), '%s does not exist!' % executable
|
||||
|
||||
test_command = [ executable ] + flags
|
||||
print ' '.join(test_command)
|
||||
|
||||
subprocess.check_call(test_command, cwd=cwd)
|
||||
|
||||
|
||||
def RunCCTests(build_dir, filter):
|
||||
print "Running Engine Unit-tests."
|
||||
|
||||
RunEngineExecutable(build_dir, 'client_wrapper_glfw_unittests', filter)
|
||||
|
||||
RunEngineExecutable(build_dir, 'client_wrapper_unittests', filter)
|
||||
|
||||
RunEngineExecutable(build_dir, 'embedder_unittests', filter)
|
||||
|
||||
RunEngineExecutable(build_dir, 'flow_unittests', filter)
|
||||
|
||||
RunEngineExecutable(build_dir, 'fml_unittests', filter, [ time_sensitve_test_flag ])
|
||||
|
||||
RunEngineExecutable(build_dir, 'runtime_unittests', filter)
|
||||
|
||||
RunEngineExecutable(build_dir, 'shell_unittests', filter)
|
||||
|
||||
RunEngineExecutable(build_dir, 'ui_unittests', filter)
|
||||
|
||||
if IsMac():
|
||||
RunEngineExecutable(build_dir, 'flutter_channels_unittests', filter)
|
||||
|
||||
if IsLinux():
|
||||
RunEngineExecutable(build_dir, 'txt_unittests', filter, [ fonts_dir_flag ])
|
||||
|
||||
|
||||
def RunEngineBenchmarks(build_dir, filter):
|
||||
print "Running Engine Benchmarks."
|
||||
|
||||
RunEngineExecutable(build_dir, 'shell_benchmarks', filter)
|
||||
|
||||
RunEngineExecutable(build_dir, 'fml_benchmarks', filter)
|
||||
|
||||
if IsLinux():
|
||||
RunEngineExecutable(build_dir, 'txt_benchmarks', filter, [ fonts_dir_flag ])
|
||||
|
||||
|
||||
|
||||
def SnapshotTest(build_dir, dart_file, kernel_file_output):
|
||||
print "Generating snapshot for test %s" % dart_file
|
||||
|
||||
dart = os.path.join(build_dir, 'dart-sdk', 'bin', 'dart')
|
||||
frontend_server = os.path.join(build_dir, 'gen', 'frontend_server.dart.snapshot')
|
||||
flutter_patched_sdk = os.path.join(build_dir, 'flutter_patched_sdk')
|
||||
test_packages = os.path.join(dart_tests_dir, '.packages')
|
||||
|
||||
assert os.path.exists(dart)
|
||||
assert os.path.exists(frontend_server)
|
||||
assert os.path.exists(flutter_patched_sdk)
|
||||
assert os.path.exists(test_packages)
|
||||
|
||||
snapshot_command = [
|
||||
dart,
|
||||
frontend_server,
|
||||
'--sdk-root',
|
||||
flutter_patched_sdk,
|
||||
'--incremental',
|
||||
'--strong',
|
||||
'--target=flutter',
|
||||
'--packages',
|
||||
test_packages,
|
||||
'--output-dill',
|
||||
kernel_file_output,
|
||||
dart_file
|
||||
]
|
||||
|
||||
subprocess.check_call(snapshot_command, cwd=buildroot_dir)
|
||||
assert os.path.exists(kernel_file_output)
|
||||
|
||||
|
||||
def RunDartTest(build_dir, dart_file, filter):
|
||||
kernel_file_name = os.path.basename(dart_file) + '.kernel.dill'
|
||||
kernel_file_output = os.path.join(out_dir, kernel_file_name)
|
||||
|
||||
SnapshotTest(build_dir, dart_file, kernel_file_output)
|
||||
|
||||
print "Running test '%s' using 'flutter_tester'" % kernel_file_name
|
||||
RunEngineExecutable(build_dir, 'flutter_tester', filter, [
|
||||
'--disable-observatory',
|
||||
'--use-test-fonts',
|
||||
kernel_file_output
|
||||
])
|
||||
|
||||
def RunPubGet(build_dir, directory):
|
||||
print "Running 'pub get' in the tests directory %s" % dart_tests_dir
|
||||
|
||||
pub_get_command = [
|
||||
os.path.join(build_dir, 'dart-sdk', 'bin', 'pub'),
|
||||
'get'
|
||||
]
|
||||
subprocess.check_call(pub_get_command, cwd=directory)
|
||||
|
||||
|
||||
def EnsureDebugUnoptSkyPackagesAreBuilt():
|
||||
variant_out_dir = os.path.join(out_dir, 'host_debug_unopt')
|
||||
|
||||
ninja_command = [
|
||||
'autoninja',
|
||||
'-C',
|
||||
variant_out_dir,
|
||||
'flutter/sky/packages'
|
||||
]
|
||||
|
||||
# Attempt running Ninja if the out directory exists.
|
||||
# We don't want to blow away any custom GN args the caller may have already set.
|
||||
if os.path.exists(variant_out_dir):
|
||||
subprocess.check_call(ninja_command, cwd=buildroot_dir)
|
||||
return
|
||||
|
||||
gn_command = [
|
||||
os.path.join(buildroot_dir, 'flutter', 'tools', 'gn'),
|
||||
'--runtime-mode',
|
||||
'debug',
|
||||
'--unopt',
|
||||
'--no-lto',
|
||||
]
|
||||
|
||||
subprocess.check_call(gn_command, cwd=buildroot_dir)
|
||||
subprocess.check_call(ninja_command, cwd=buildroot_dir)
|
||||
|
||||
def RunDartTests(build_dir, filter):
|
||||
# This one is a bit messy. The pubspec.yaml at flutter/testing/dart/pubspec.yaml
|
||||
# has dependencies that are hardcoded to point to the sky packages at host_debug_unopt/
|
||||
# Before running Dart tests, make sure to run just that target (NOT the whole engine)
|
||||
EnsureDebugUnoptSkyPackagesAreBuilt();
|
||||
|
||||
# Now that we have the Sky packages at the hardcoded location, run `pub get`.
|
||||
RunPubGet(build_dir, dart_tests_dir)
|
||||
|
||||
dart_tests = glob.glob('%s/*.dart' % dart_tests_dir)
|
||||
|
||||
for dart_test_file in dart_tests:
|
||||
if filter in os.path.basename(dart_test_file):
|
||||
print "Testing dart file %s" % dart_test_file
|
||||
RunDartTest(build_dir, dart_test_file, filter)
|
||||
else:
|
||||
print "Skipping %s due to filter." % dart_test_file
|
||||
|
||||
|
||||
def RunTests(build_dir, filter, run_engine_tests, run_dart_tests, run_benchmarks):
|
||||
if run_engine_tests:
|
||||
RunCCTests(build_dir, filter)
|
||||
|
||||
if run_dart_tests:
|
||||
RunDartTests(build_dir, filter)
|
||||
|
||||
if run_benchmarks:
|
||||
RunEngineBenchmarks(build_dir, filter)
|
||||
|
||||
|
||||
def main():
|
||||
parser = argparse.ArgumentParser();
|
||||
|
||||
parser.add_argument('--variant', dest='variant', action='store',
|
||||
default='host_debug_unopt', help='The engine build variant to run the tests for.');
|
||||
parser.add_argument('--type', type=str, choices=['all', 'engine', 'dart', 'benchmarks'], default='all')
|
||||
parser.add_argument('--filter', type=str, default='',
|
||||
help='The file name filter to use to select specific tests to run.')
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
run_engine_tests = args.type in ['engine', 'all']
|
||||
run_dart_tests = args.type in ['dart', 'all']
|
||||
run_benchmarks = args.type in ['benchmarks', 'all']
|
||||
|
||||
build_dir = os.path.join(out_dir, args.variant)
|
||||
assert os.path.exists(build_dir), 'Build variant directory %s does not exist!' % build_dir
|
||||
RunTests(build_dir, args.filter, run_engine_tests, run_dart_tests, run_benchmarks)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
sys.exit(main())
|
||||
@ -2,102 +2,7 @@
|
||||
|
||||
set -o pipefail -e;
|
||||
|
||||
BUILDROOT_DIR="$(pwd)"
|
||||
if [[ "$BUILDROOT_DIR" != */src ]]; then
|
||||
if [[ "$BUILDROOT_DIR" != */src/* ]]; then
|
||||
echo "Unable to determine build root. Exiting."
|
||||
exit 1
|
||||
fi
|
||||
BUILDROOT_DIR="${BUILDROOT_DIR%/src/*}/src"
|
||||
fi
|
||||
echo "Using build root: $BUILDROOT_DIR"
|
||||
BUILD_VARIANT="${1:-host_debug_unopt}"
|
||||
CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
|
||||
|
||||
OUT_DIR="$BUILDROOT_DIR/out"
|
||||
HOST_DIR="$OUT_DIR/${1:-host_debug_unopt}"
|
||||
|
||||
# Check a Dart SDK has been built.
|
||||
if [[ ! -d "$HOST_DIR/dart-sdk" ]]; then
|
||||
echo "Built Dart SDK not found at $HOST_DIR/dart-sdk. Exiting."
|
||||
exit 1
|
||||
fi
|
||||
|
||||
# Switch to buildroot dir. Some tests assume paths relative to buildroot.
|
||||
cd "$BUILDROOT_DIR"
|
||||
|
||||
echo "Running embedder_unittests..."
|
||||
"$HOST_DIR/embedder_unittests"
|
||||
|
||||
echo "Running flow_unittests..."
|
||||
"$HOST_DIR/flow_unittests"
|
||||
|
||||
echo "Running fml_unittests..."
|
||||
"$HOST_DIR/fml_unittests" --gtest_filter="-*TimeSensitiveTest*"
|
||||
|
||||
echo "Running fml_benchmarks..."
|
||||
"$HOST_DIR/fml_benchmarks"
|
||||
|
||||
echo "Running runtime_unittests..."
|
||||
"$HOST_DIR/runtime_unittests"
|
||||
|
||||
echo "Running shell_unittests..."
|
||||
"$HOST_DIR/shell_unittests"
|
||||
|
||||
echo "Running shell_benchmarks..."
|
||||
"$HOST_DIR/shell_benchmarks"
|
||||
|
||||
echo "Running client_wrapper_unittests..."
|
||||
"$HOST_DIR/client_wrapper_unittests"
|
||||
|
||||
echo "Running client_wrapper_glfw_unittests..."
|
||||
"$HOST_DIR/client_wrapper_glfw_unittests"
|
||||
|
||||
echo "Running txt_unittests..."
|
||||
"$HOST_DIR/txt_unittests" --font-directory="$BUILDROOT_DIR/flutter/third_party/txt/third_party/fonts"
|
||||
|
||||
echo "Running ui_unittests..."
|
||||
"$HOST_DIR/ui_unittests"
|
||||
|
||||
echo "Running txt_benchmarks..."
|
||||
"$HOST_DIR/txt_benchmarks" --font-directory="$BUILDROOT_DIR/flutter/third_party/txt/third_party/fonts"
|
||||
|
||||
# Build flutter/sky/packages.
|
||||
#
|
||||
# flutter/testing/dart/pubspec.yaml contains harcoded path deps to
|
||||
# host_debug_unopt packages.
|
||||
"$BUILDROOT_DIR/flutter/tools/gn" --unoptimized
|
||||
ninja -C $OUT_DIR/host_debug_unopt flutter/sky/packages
|
||||
|
||||
# Fetch Dart test dependencies.
|
||||
pushd "$BUILDROOT_DIR/flutter/testing/dart"
|
||||
"$HOST_DIR/dart-sdk/bin/pub" get
|
||||
popd
|
||||
|
||||
"$HOST_DIR/dart" --version
|
||||
|
||||
run_test () {
|
||||
"$HOST_DIR/dart" $HOST_DIR/gen/frontend_server.dart.snapshot \
|
||||
--sdk-root $HOST_DIR/flutter_patched_sdk \
|
||||
--incremental \
|
||||
--strong \
|
||||
--target=flutter \
|
||||
--packages flutter/testing/dart/.packages \
|
||||
--output-dill $HOST_DIR/engine_test.dill \
|
||||
$1
|
||||
|
||||
"$HOST_DIR/flutter_tester" \
|
||||
--disable-observatory \
|
||||
--use-test-fonts \
|
||||
"$HOST_DIR/engine_test.dill"
|
||||
}
|
||||
|
||||
# Verify that a failing test returns a failure code.
|
||||
! run_test "$BUILDROOT_DIR/flutter/testing/smoke_test_failure/fail_test.dart"
|
||||
|
||||
for TEST_SCRIPT in "$BUILDROOT_DIR"/flutter/testing/dart/*.dart; do
|
||||
run_test "$TEST_SCRIPT"
|
||||
done
|
||||
|
||||
pushd flutter
|
||||
ci/test.sh
|
||||
popd
|
||||
exit 0
|
||||
python "${CURRENT_DIR}/run_tests.py" --variant "${BUILD_VARIANT}"
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user