Updates to the engine test runner script (flutter/engine#9934)

* Use separate filters for engine executables and Dart test scripts
* Do not run the benchmarks on Cirrus

engine/src/flutter/testing/run_tests.py

@@ -53,15 +53,15 @@ def FindExecutablePath(path):
 
 
 def RunEngineExecutable(build_dir, executable_name, filter, flags=[], cwd=buildroot_dir):
-  if not filter in executable_name:
+  if filter is not None and executable_name not in filter:
     print 'Skipping %s due to filter.' % executable_name
     return
 
   executable = FindExecutablePath(os.path.join(build_dir, executable_name))
-  
+
   print 'Running %s in %s' % (executable_name, cwd)
   test_command = [ executable ] + flags
-  print ' '.join(test_command) 
+  print ' '.join(test_command)
   subprocess.check_call(test_command, cwd=cwd)
 
 
@@ -101,7 +101,7 @@ def RunEngineBenchmarks(build_dir, filter):
   print "Running Engine Benchmarks."
 
   RunEngineExecutable(build_dir, 'shell_benchmarks', filter)
-  
+
   RunEngineExecutable(build_dir, 'fml_benchmarks', filter)
 
   if IsLinux():
@@ -121,7 +121,7 @@ def SnapshotTest(build_dir, dart_file, kernel_file_output):
   assert os.path.exists(frontend_server)
   assert os.path.exists(flutter_patched_sdk)
   assert os.path.exists(test_packages)
-  
+
   snapshot_command = [
     dart,
     frontend_server,
@@ -136,19 +136,19 @@ def SnapshotTest(build_dir, dart_file, kernel_file_output):
     kernel_file_output,
     dart_file
   ]
-  
+
   subprocess.check_call(snapshot_command, cwd=buildroot_dir)
   assert os.path.exists(kernel_file_output)
 
 
-def RunDartTest(build_dir, dart_file, filter):
+def RunDartTest(build_dir, dart_file):
   kernel_file_name = os.path.basename(dart_file) + '.kernel.dill'
   kernel_file_output = os.path.join(out_dir, kernel_file_name)
-  
+
   SnapshotTest(build_dir, dart_file, kernel_file_output)
 
   print "Running test '%s' using 'flutter_tester'" % kernel_file_name
-  RunEngineExecutable(build_dir, 'flutter_tester', filter, [
+  RunEngineExecutable(build_dir, 'flutter_tester', None, [
     '--disable-observatory',
     '--use-test-fonts',
     kernel_file_output
@@ -187,7 +187,7 @@ def EnsureDebugUnoptSkyPackagesAreBuilt():
     '--unopt',
     '--no-lto',
   ]
-  
+
   subprocess.check_call(gn_command, cwd=buildroot_dir)
   subprocess.check_call(ninja_command, cwd=buildroot_dir)
 
@@ -198,49 +198,50 @@ def RunDartTests(build_dir, filter):
   EnsureDebugUnoptSkyPackagesAreBuilt();
 
   # Now that we have the Sky packages at the hardcoded location, run `pub get`.
-  RunEngineExecutable(build_dir, os.path.join('dart-sdk', 'bin', 'pub'), '', flags=['get'], cwd=dart_tests_dir)
+  RunEngineExecutable(build_dir, os.path.join('dart-sdk', 'bin', 'pub'), None, flags=['get'], cwd=dart_tests_dir)
 
   dart_tests = glob.glob('%s/*.dart' % dart_tests_dir)
 
   for dart_test_file in dart_tests:
-    if filter in os.path.basename(dart_test_file):
-      print "Testing dart file %s" % dart_test_file
-      RunDartTest(build_dir, dart_test_file, filter)
-    else:
+    if filter is not None and os.path.basename(dart_test_file) not in filter:
       print "Skipping %s due to filter." % dart_test_file
-
-
-def RunTests(build_dir, filter, run_engine_tests, run_dart_tests, run_benchmarks):
-  if run_engine_tests:
-    RunCCTests(build_dir, filter)
-
-  # https://github.com/flutter/flutter/issues/36301
-  if run_dart_tests and not IsWindows():
-    RunDartTests(build_dir, filter)
-
-  # https://github.com/flutter/flutter/issues/36300
-  if run_benchmarks and not IsWindows():
-    RunEngineBenchmarks(build_dir, filter)
-
+    else:
+      print "Testing dart file %s" % dart_test_file
+      RunDartTest(build_dir, dart_test_file)
 
 def main():
   parser = argparse.ArgumentParser();
 
-  parser.add_argument('--variant', dest='variant', action='store', 
+  parser.add_argument('--variant', dest='variant', action='store',
       default='host_debug_unopt', help='The engine build variant to run the tests for.');
-  parser.add_argument('--type', type=str, choices=['all', 'engine', 'dart', 'benchmarks'], default='all')
-  parser.add_argument('--filter', type=str, default='',
-      help='The file name filter to use to select specific tests to run.')
+  parser.add_argument('--type', type=str, default='all')
+  parser.add_argument('--engine-filter', type=str, default='',
+      help='A list of engine test executables to run.')
+  parser.add_argument('--dart-filter', type=str, default='',
+      help='A list of Dart test scripts to run.')
 
   args = parser.parse_args()
-  
-  run_engine_tests = args.type in ['engine', 'all']
-  run_dart_tests = args.type in ['dart', 'all']
-  run_benchmarks = args.type in ['benchmarks', 'all']
+
+  if args.type == 'all':
+    types = ['engine', 'dart', 'benchmarks']
+  else:
+    types = args.type.split(',')
 
   build_dir = os.path.join(out_dir, args.variant)
   assert os.path.exists(build_dir), 'Build variant directory %s does not exist!' % build_dir
-  RunTests(build_dir, args.filter, run_engine_tests, run_dart_tests, run_benchmarks)
+
+  engine_filter = args.engine_filter.split(',') if args.engine_filter else None
+  if 'engine' in types:
+    RunCCTests(build_dir, engine_filter)
+
+  # https://github.com/flutter/flutter/issues/36301
+  if 'dart' in types and not IsWindows():
+    dart_filter = args.dart_filter.split(',') if args.dart_filter else None
+    RunDartTests(build_dir, dart_filter)
+
+  # https://github.com/flutter/flutter/issues/36300
+  if 'benchmarks' in types and not IsWindows():
+    RunEngineBenchmarks(build_dir, engine_filter)
 
 
 if __name__ == '__main__':

engine/src/flutter/testing/run_tests.sh

@@ -5,4 +5,4 @@ set -o pipefail -e;
 BUILD_VARIANT="${1:-host_debug_unopt}"
 CURRENT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
 
-python "${CURRENT_DIR}/run_tests.py" --variant "${BUILD_VARIANT}"
+python "${CURRENT_DIR}/run_tests.py" --variant="${BUILD_VARIANT}" --type=engine,dart