diff --git a/engine/src/flutter/build/concurrent_jobs.gni b/engine/src/flutter/build/concurrent_jobs.gni deleted file mode 100644 index a930b255682..00000000000 --- a/engine/src/flutter/build/concurrent_jobs.gni +++ /dev/null @@ -1,12 +0,0 @@ -# Copyright 2013 The Flutter Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -_script = "//flutter/build/get_concurrent_jobs.py" -_args = [ - "--reserve-memory=1GB", - "--memory-per-job", - "dart=1GB", -] - -concurrent_jobs = exec_script(_script, _args, "json", [ _script ]) diff --git a/engine/src/flutter/build/dart/BUILD.gn b/engine/src/flutter/build/dart/BUILD.gn index a8692ab08b1..4c4d57d0696 100644 --- a/engine/src/flutter/build/dart/BUILD.gn +++ b/engine/src/flutter/build/dart/BUILD.gn @@ -2,7 +2,6 @@ # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -import("//flutter/build/concurrent_jobs.gni") import("//flutter/build/dart/dart.gni") import("//flutter/common/config.gni") @@ -12,7 +11,7 @@ declare_args() { # Maximum number of Dart processes to run in parallel. # # To avoid out-of-memory errors we explicitly reduce the number of jobs. - concurrent_dart_jobs = concurrent_jobs.dart + concurrent_dart_jobs = 1 } pool("dart_pool") { diff --git a/engine/src/flutter/build/get_concurrent_jobs.py b/engine/src/flutter/build/get_concurrent_jobs.py deleted file mode 100755 index 3559073c66d..00000000000 --- a/engine/src/flutter/build/get_concurrent_jobs.py +++ /dev/null @@ -1,130 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2019 The Fuchsia Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -# This script computes the number of concurrent jobs that can run in the -# build as a function of the machine. It accepts a set of key value pairs -# given by repeated --memory-per-job arguments. For example: -# -# $ get_concurrent_jobs.py --memory-per-job dart=1GB -# -# The result is a json map printed to stdout that gives the number of -# concurrent jobs allowed of each kind. For example: -# -# {"dart": 8} -# -# Some memory can be held out of the calculation with the --reserve-memory flag. - -import argparse -import ctypes -import json -import multiprocessing -import os -import re -import subprocess -import sys - -UNITS = {'B': 1, 'KB': 2**10, 'MB': 2**20, 'GB': 2**30, 'TB': 2**40} - - -# pylint: disable=line-too-long -# See https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/nf-sysinfoapi-globalmemorystatusex -# and https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/ns-sysinfoapi-memorystatusex -# pylint: enable=line-too-long -class MEMORYSTATUSEX(ctypes.Structure): - _fields_ = [ - ('dwLength', ctypes.c_ulong), - ('dwMemoryLoad', ctypes.c_ulong), - ('ullTotalPhys', ctypes.c_ulonglong), - ('ullAvailPhys', ctypes.c_ulonglong), - ('ullTotalPageFile', ctypes.c_ulonglong), - ('ullAvailPageFile', ctypes.c_ulonglong), - ('ullTotalVirtual', ctypes.c_ulonglong), - ('ullAvailVirtual', ctypes.c_ulonglong), - ('sullAvailExtendedVirtual', ctypes.c_ulonglong), - ] - - -def get_total_memory(): - if sys.platform in ('win32', 'cygwin'): - stat = MEMORYSTATUSEX(dwLength=ctypes.sizeof(MEMORYSTATUSEX)) - success = ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat)) - return stat.ullTotalPhys if success else 0 - if sys.platform.startswith('linux'): - if os.path.exists('/proc/meminfo'): - with open('/proc/meminfo') as meminfo: - memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB') - for line in meminfo: - match = memtotal_re.match(line) - if match: - return float(match.group(1)) * 2**10 - if sys.platform == 'darwin': - try: - return int(subprocess.check_output(['sysctl', '-n', 'hw.memsize'])) - except: # pylint: disable=bare-except - return 0 - return 0 - - -def parse_size(string): - i = next(i for (i, c) in enumerate(string) if not c.isdigit()) - number = string[:i].strip() - unit = string[i:].strip() - return int(float(number) * UNITS[unit]) - - -class ParseSizeAction(argparse.Action): - - def __call__(self, parser, args, values, option_string=None): - sizes = getattr(args, self.dest, []) - for value in values: - (k, val) = value.split('=', 1) - sizes.append((k, parse_size(val))) - setattr(args, self.dest, sizes) - - -def main(): - parser = argparse.ArgumentParser() - parser.add_argument( - '--memory-per-job', - action=ParseSizeAction, - default=[], - nargs='*', - help='Key value pairings (dart=1GB) giving an estimate of the amount of ' - 'memory needed for the class of job.' - ) - parser.add_argument( - '--reserve-memory', - type=parse_size, - default=0, - help='The amount of memory to be held out of the amount for jobs to use.' - ) - args = parser.parse_args() - - total_memory = get_total_memory() - - # Ensure the total memory used in the calculation below is at least 0 - mem_total_bytes = max(0, total_memory - args.reserve_memory) - - # Ensure the number of cpus used in the calculation below is at least 1 - try: - cpu_cap = multiprocessing.cpu_count() - except: # pylint: disable=bare-except - cpu_cap = 1 - - concurrent_jobs = {} - for job, memory_per_job in args.memory_per_job: - # Calculate the number of jobs that will fit in memory. Ensure the - # value is at least 1. - num_concurrent_jobs = int(max(1, mem_total_bytes / memory_per_job)) - # Cap the number of jobs by the number of cpus available. - concurrent_jobs[job] = min(num_concurrent_jobs, cpu_cap) - - print(json.dumps(concurrent_jobs)) - - return 0 - - -if __name__ == '__main__': - sys.exit(main()) diff --git a/engine/src/flutter/ci/builders/linux_unopt.json b/engine/src/flutter/ci/builders/linux_unopt.json index 2be76e18116..0000048c7e0 100644 --- a/engine/src/flutter/ci/builders/linux_unopt.json +++ b/engine/src/flutter/ci/builders/linux_unopt.json @@ -94,6 +94,11 @@ { "name": "Check build configs", "script": "flutter/ci/check_build_configs.sh" + }, + { + "name": "Tests of tools/gn", + "language": "python3", + "script": "flutter/tools/gn_test.py" } ] }, diff --git a/engine/src/flutter/ci/builders/mac_unopt.json b/engine/src/flutter/ci/builders/mac_unopt.json index c6674574489..dfae57a324e 100644 --- a/engine/src/flutter/ci/builders/mac_unopt.json +++ b/engine/src/flutter/ci/builders/mac_unopt.json @@ -44,6 +44,11 @@ "dart,dart-host,engine", "--engine-capture-core-dump" ] + }, + { + "name": "Tests of tools/gn", + "language": "python3", + "script": "flutter/tools/gn_test.py" } ] }, diff --git a/engine/src/flutter/ci/builders/standalone/windows_unopt.json b/engine/src/flutter/ci/builders/standalone/windows_unopt.json index cf3a5a36ee8..3af977a0ef9 100644 --- a/engine/src/flutter/ci/builders/standalone/windows_unopt.json +++ b/engine/src/flutter/ci/builders/standalone/windows_unopt.json @@ -37,6 +37,11 @@ "engine", "--engine-capture-core-dump" ] + }, + { + "name": "Tests of tools/gn", + "language": "python3", + "script": "flutter/tools/gn_test.py" } ] } diff --git a/engine/src/flutter/impeller/tools/BUILD.gn b/engine/src/flutter/impeller/tools/BUILD.gn index a8bf1e00685..ae1bf978241 100644 --- a/engine/src/flutter/impeller/tools/BUILD.gn +++ b/engine/src/flutter/impeller/tools/BUILD.gn @@ -11,19 +11,7 @@ declare_args() { # Maximum number of malioc processes to run in parallel. # # To avoid out-of-memory errors we explicitly reduce the number of jobs. - impeller_concurrent_malioc_jobs = -1 -} - -if (impeller_concurrent_malioc_jobs == -1) { - _script = "//flutter/build/get_concurrent_jobs.py" - _args = [ - "--reserve-memory=1GB", - "--memory-per-job", - "malioc=100MB", - ] - _concurrent_jobs = exec_script(_script, _args, "json", [ _script ]) - impeller_concurrent_malioc_jobs = _concurrent_jobs.malioc - assert(impeller_concurrent_malioc_jobs > 0) + impeller_concurrent_malioc_jobs = 1 } pool("malioc_pool") { diff --git a/engine/src/flutter/tools/gn b/engine/src/flutter/tools/gn index bddc58333cd..375af5d7079 100755 --- a/engine/src/flutter/tools/gn +++ b/engine/src/flutter/tools/gn @@ -8,10 +8,13 @@ from __future__ import division from __future__ import print_function import argparse -import subprocess -import sys +import ctypes +import multiprocessing import os import platform +import re +import subprocess +import sys SRC_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) @@ -378,6 +381,73 @@ def setup_git_versions(): return revision_args +# pylint: disable=line-too-long +# See https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/nf-sysinfoapi-globalmemorystatusex +# and https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/ns-sysinfoapi-memorystatusex +# pylint: enable=line-too-long +class MEMORYSTATUSEX(ctypes.Structure): + _fields_ = [ + ('dwLength', ctypes.c_ulong), + ('dwMemoryLoad', ctypes.c_ulong), + ('ullTotalPhys', ctypes.c_ulonglong), + ('ullAvailPhys', ctypes.c_ulonglong), + ('ullTotalPageFile', ctypes.c_ulonglong), + ('ullAvailPageFile', ctypes.c_ulonglong), + ('ullTotalVirtual', ctypes.c_ulonglong), + ('ullAvailVirtual', ctypes.c_ulonglong), + ('sullAvailExtendedVirtual', ctypes.c_ulonglong), + ] + + +def get_total_memory(): + if sys.platform in ('win32', 'cygwin'): + stat = MEMORYSTATUSEX(dwLength=ctypes.sizeof(MEMORYSTATUSEX)) + success = ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat)) + return stat.ullTotalPhys if success else 0 + if sys.platform.startswith('linux'): + if os.path.exists('/proc/meminfo'): + with open('/proc/meminfo') as meminfo: + memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB') + for line in meminfo: + match = memtotal_re.match(line) + if match: + return float(match.group(1)) * 2**10 + if sys.platform == 'darwin': + try: + return int(subprocess.check_output(['sysctl', '-n', 'hw.memsize'])) + except: # pylint: disable=bare-except + return 0 + return 0 + + +def parse_size(string): + units = {'B': 1, 'KB': 2**10, 'MB': 2**20, 'GB': 2**30, 'TB': 2**40} + i = next(i for (i, c) in enumerate(string) if not c.isdigit()) + number = string[:i].strip() + unit = string[i:].strip() + return int(float(number) * units[unit]) + + +def get_concurrent_jobs(reserve_memory, memory_per_job): + # reserve_memory = 2**30 + total_memory = get_total_memory() + # Ensure the total memory used in the calculation below is at least 0 + mem_total_bytes = max(0, total_memory - parse_size(reserve_memory)) + # Ensure the number of cpus used in the calculation below is at least 1 + try: + cpu_cap = multiprocessing.cpu_count() + except: # pylint: disable=bare-except + cpu_cap = 1 + + # Calculate the number of jobs that will fit in memory. Ensure the + # value is at least 1. + num_concurrent_jobs = int(max(1, mem_total_bytes / parse_size(memory_per_job))) + # Cap the number of jobs by the number of cpus available. + concurrent_jobs = min(num_concurrent_jobs, cpu_cap) + + return concurrent_jobs + + def to_gn_args(args): if args.simulator: if args.target_os != 'ios': @@ -529,6 +599,8 @@ def to_gn_args(args): else: gn_args['dart_runtime_mode'] = runtime_mode + gn_args['concurrent_dart_jobs'] = get_concurrent_jobs('1GB', '1GB') + # Desktop embeddings can have more dependencies than the engine library, # which can be problematic in some build environments (e.g., building on # Linux will bring in pkg-config dependencies at generation time). These @@ -712,12 +784,12 @@ def to_gn_args(args): if args.prebuilt_impellerc is not None: gn_args['impeller_use_prebuilt_impellerc'] = args.prebuilt_impellerc - if args.malioc_path is not None: - gn_args['impeller_malioc_path'] = args.malioc_path - else: + malioc_path = args.malioc_path + if not malioc_path: malioc_path = os.environ.get('MALIOC_PATH') - if malioc_path: - gn_args['impeller_malioc_path'] = malioc_path + if malioc_path: + gn_args['impeller_malioc_path'] = malioc_path + gn_args['impeller_concurrent_malioc_jobs'] = get_concurrent_jobs('1GB', '100MB') if args.use_glfw_swiftshader: if get_host_os() == 'mac': diff --git a/engine/src/flutter/tools/gn_test.py b/engine/src/flutter/tools/gn_test.py index 36bb6ec982b..33ab13e6850 100644 --- a/engine/src/flutter/tools/gn_test.py +++ b/engine/src/flutter/tools/gn_test.py @@ -1,10 +1,10 @@ # Copyright 2013 The Flutter Authors. All rights reserved. # Use of this source code is governed by a BSD-style license that can be # found in the LICENSE file. -import unittest -import os import imp +import os +import unittest SKY_TOOLS = os.path.dirname(os.path.abspath(__file__)) gn = imp.load_source('gn', os.path.join(SKY_TOOLS, 'gn')) @@ -14,17 +14,21 @@ class GNTestCase(unittest.TestCase): def _expect_build_dir(self, arg_list, expected_build_dir): args = gn.parse_args(['gn'] + arg_list) - self.assertEquals(gn.get_out_dir(args), expected_build_dir) + self.assertEqual(gn.get_out_dir(args), expected_build_dir) def test_get_out_dir(self): - self._expect_build_dir(['--debug'], 'out/Debug') - self._expect_build_dir(['--release'], 'out/Release') - self._expect_build_dir(['--ios'], 'out/ios_Debug') - self._expect_build_dir(['--ios'], 'out/ios_Debug_extension_safe') - self._expect_build_dir(['--ios', '--release'], 'out/ios_Release') - self._expect_build_dir(['--ios'], 'out/ios_Release_extension_safe') - self._expect_build_dir(['--android'], 'out/android_Debug') - self._expect_build_dir(['--android', '--release'], 'out/android_Release') + self._expect_build_dir(['--runtime-mode', 'debug'], os.path.join('out', 'host_debug')) + self._expect_build_dir(['--runtime-mode', 'release'], os.path.join('out', 'host_release')) + self._expect_build_dir(['--ios'], os.path.join('out', 'ios_debug')) + self._expect_build_dir(['--ios', '--darwin-extension-safe'], + os.path.join('out', 'ios_debug_extension_safe')) + self._expect_build_dir(['--ios', '--runtime-mode', 'release'], + os.path.join('out', 'ios_release')) + self._expect_build_dir(['--ios', '--darwin-extension-safe', '--runtime-mode', 'release'], + os.path.join('out', 'ios_release_extension_safe')) + self._expect_build_dir(['--android'], os.path.join('out', 'android_debug')) + self._expect_build_dir(['--android', '--runtime-mode', 'release'], + os.path.join('out', 'android_release')) def _gn_args(self, arg_list): args = gn.parse_args(['gn'] + arg_list) @@ -32,17 +36,25 @@ class GNTestCase(unittest.TestCase): def test_to_gn_args(self): # This would not necessarily be true on a 32-bit machine? - self.assertEquals(self._gn_args(['--ios', '--simulator'])['target_cpu'], 'x64') - self.assertEquals(self._gn_args(['--ios'])['target_cpu'], 'arm') + self.assertEqual( + self._gn_args(['--ios', '--simulator', '--simulator-cpu', 'x64'])['target_cpu'], 'x64' + ) + self.assertEqual(self._gn_args(['--ios'])['target_cpu'], 'arm64') def test_cannot_use_android_and_enable_unittests(self): - with self.assertRaises(SystemExit): + with self.assertRaises(Exception): self._gn_args(['--android', '--enable-unittests']) def test_cannot_use_ios_and_enable_unittests(self): - with self.assertRaises(SystemExit): + with self.assertRaises(Exception): self._gn_args(['--ios', '--enable-unittests']) + def test_parse_size(self): + self.assertEqual(gn.parse_size('5B'), 5) + self.assertEqual(gn.parse_size('5KB'), 5 * 2**10) + self.assertEqual(gn.parse_size('5MB'), 5 * 2**20) + self.assertEqual(gn.parse_size('5GB'), 5 * 2**30) + if __name__ == '__main__': unittest.main()