DEV/flutter_flutter
1
0
Fork 0
mirror of https://github.com/flutter/flutter.git synced 2026-02-20 02:29:02 +08:00
Code Issues Packages Projects Releases Wiki Activity

Folds concurrent job calculation into tools/gn (flutter/engine#51193)

Browse Source 
Folds the `get_concurrent_jobs.py` script into `tools/gn` and stops calling it with `exec_script()` during GN.

Part of https://github.com/flutter/flutter/issues/144430
This commit is contained in:
Zachary Anderson 2024-03-05 17:57:14 +00:00 committed by GitHub
parent 46a92d295e
commit 8928d5284c
9 changed files with 123 additions and 179 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
engine/src/flutter/build/concurrent_jobs.gni
View File

@ -1,12 +0,0 @@
# Copyright 2013 The Flutter Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
_script = "//flutter/build/get_concurrent_jobs.py"
_args = [
"--reserve-memory=1GB",
"--memory-per-job",
"dart=1GB",
]
concurrent_jobs = exec_script(_script, _args, "json", [ _script ])

3
engine/src/flutter/build/dart/BUILD.gn
View File

@ -2,7 +2,6 @@
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import("//flutter/build/concurrent_jobs.gni")
import("//flutter/build/dart/dart.gni")
import("//flutter/common/config.gni")
@ -12,7 +11,7 @@ declare_args() {
# Maximum number of Dart processes to run in parallel.
#
# To avoid out-of-memory errors we explicitly reduce the number of jobs.
concurrent_dart_jobs = concurrent_jobs.dart
concurrent_dart_jobs = 1
}
pool("dart_pool") {

130
engine/src/flutter/build/get_concurrent_jobs.py
View File

@ -1,130 +0,0 @@
#!/usr/bin/env python3
# Copyright 2019 The Fuchsia Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
# This script computes the number of concurrent jobs that can run in the
# build as a function of the machine. It accepts a set of key value pairs
# given by repeated --memory-per-job arguments. For example:
#
# $ get_concurrent_jobs.py --memory-per-job dart=1GB
#
# The result is a json map printed to stdout that gives the number of
# concurrent jobs allowed of each kind. For example:
#
# {"dart": 8}
#
# Some memory can be held out of the calculation with the --reserve-memory flag.
import argparse
import ctypes
import json
import multiprocessing
import os
import re
import subprocess
import sys
UNITS = {'B': 1, 'KB': 2**10, 'MB': 2**20, 'GB': 2**30, 'TB': 2**40}
# pylint: disable=line-too-long
# See https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/nf-sysinfoapi-globalmemorystatusex
# and https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/ns-sysinfoapi-memorystatusex
# pylint: enable=line-too-long
class MEMORYSTATUSEX(ctypes.Structure):
_fields_ = [
('dwLength', ctypes.c_ulong),
('dwMemoryLoad', ctypes.c_ulong),
('ullTotalPhys', ctypes.c_ulonglong),
('ullAvailPhys', ctypes.c_ulonglong),
('ullTotalPageFile', ctypes.c_ulonglong),
('ullAvailPageFile', ctypes.c_ulonglong),
('ullTotalVirtual', ctypes.c_ulonglong),
('ullAvailVirtual', ctypes.c_ulonglong),
('sullAvailExtendedVirtual', ctypes.c_ulonglong),
]
def get_total_memory():
if sys.platform in ('win32', 'cygwin'):
stat = MEMORYSTATUSEX(dwLength=ctypes.sizeof(MEMORYSTATUSEX))
success = ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat))
return stat.ullTotalPhys if success else 0
if sys.platform.startswith('linux'):
if os.path.exists('/proc/meminfo'):
with open('/proc/meminfo') as meminfo:
memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB')
for line in meminfo:
match = memtotal_re.match(line)
if match:
return float(match.group(1)) * 2**10
if sys.platform == 'darwin':
try:
return int(subprocess.check_output(['sysctl', '-n', 'hw.memsize']))
except: # pylint: disable=bare-except
return 0
return 0
def parse_size(string):
i = next(i for (i, c) in enumerate(string) if not c.isdigit())
number = string[:i].strip()
unit = string[i:].strip()
return int(float(number) * UNITS[unit])
class ParseSizeAction(argparse.Action):
def __call__(self, parser, args, values, option_string=None):
sizes = getattr(args, self.dest, [])
for value in values:
(k, val) = value.split('=', 1)
sizes.append((k, parse_size(val)))
setattr(args, self.dest, sizes)
def main():
parser = argparse.ArgumentParser()
parser.add_argument(
'--memory-per-job',
action=ParseSizeAction,
default=[],
nargs='*',
help='Key value pairings (dart=1GB) giving an estimate of the amount of '
'memory needed for the class of job.'
)
parser.add_argument(
'--reserve-memory',
type=parse_size,
default=0,
help='The amount of memory to be held out of the amount for jobs to use.'
)
args = parser.parse_args()
total_memory = get_total_memory()
# Ensure the total memory used in the calculation below is at least 0
mem_total_bytes = max(0, total_memory - args.reserve_memory)
# Ensure the number of cpus used in the calculation below is at least 1
try:
cpu_cap = multiprocessing.cpu_count()
except: # pylint: disable=bare-except
cpu_cap = 1
concurrent_jobs = {}
for job, memory_per_job in args.memory_per_job:
# Calculate the number of jobs that will fit in memory. Ensure the
# value is at least 1.
num_concurrent_jobs = int(max(1, mem_total_bytes / memory_per_job))
# Cap the number of jobs by the number of cpus available.
concurrent_jobs[job] = min(num_concurrent_jobs, cpu_cap)
print(json.dumps(concurrent_jobs))
return 0
if __name__ == '__main__':
sys.exit(main())

5
engine/src/flutter/ci/builders/linux_unopt.json
View File

@ -94,6 +94,11 @@
{
"name": "Check build configs",
"script": "flutter/ci/check_build_configs.sh"
},
{
"name": "Tests of tools/gn",
"language": "python3",
"script": "flutter/tools/gn_test.py"
}
]
},

5
engine/src/flutter/ci/builders/mac_unopt.json
View File

@ -44,6 +44,11 @@
"dart,dart-host,engine",
"--engine-capture-core-dump"
]
},
{
"name": "Tests of tools/gn",
"language": "python3",
"script": "flutter/tools/gn_test.py"
}
]
},

5
engine/src/flutter/ci/builders/standalone/windows_unopt.json
View File

@ -37,6 +37,11 @@
"engine",
"--engine-capture-core-dump"
]
},
{
"name": "Tests of tools/gn",
"language": "python3",
"script": "flutter/tools/gn_test.py"
}
]
}

14
engine/src/flutter/impeller/tools/BUILD.gn
View File

@ -11,19 +11,7 @@ declare_args() {
# Maximum number of malioc processes to run in parallel.
#
# To avoid out-of-memory errors we explicitly reduce the number of jobs.
impeller_concurrent_malioc_jobs = -1
}
if (impeller_concurrent_malioc_jobs == -1) {
_script = "//flutter/build/get_concurrent_jobs.py"
_args = [
"--reserve-memory=1GB",
"--memory-per-job",
"malioc=100MB",
]
_concurrent_jobs = exec_script(_script, _args, "json", [ _script ])
impeller_concurrent_malioc_jobs = _concurrent_jobs.malioc
assert(impeller_concurrent_malioc_jobs > 0)
impeller_concurrent_malioc_jobs = 1
}
pool("malioc_pool") {

86
engine/src/flutter/tools/gn
View File

@ -8,10 +8,13 @@ from __future__ import division
from __future__ import print_function
import argparse
import subprocess
import sys
import ctypes
import multiprocessing
import os
import platform
import re
import subprocess
import sys
SRC_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
@ -378,6 +381,73 @@ def setup_git_versions():
return revision_args
# pylint: disable=line-too-long
# See https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/nf-sysinfoapi-globalmemorystatusex
# and https://docs.microsoft.com/en-us/windows/win32/api/sysinfoapi/ns-sysinfoapi-memorystatusex
# pylint: enable=line-too-long
class MEMORYSTATUSEX(ctypes.Structure):
_fields_ = [
('dwLength', ctypes.c_ulong),
('dwMemoryLoad', ctypes.c_ulong),
('ullTotalPhys', ctypes.c_ulonglong),
('ullAvailPhys', ctypes.c_ulonglong),
('ullTotalPageFile', ctypes.c_ulonglong),
('ullAvailPageFile', ctypes.c_ulonglong),
('ullTotalVirtual', ctypes.c_ulonglong),
('ullAvailVirtual', ctypes.c_ulonglong),
('sullAvailExtendedVirtual', ctypes.c_ulonglong),
]
def get_total_memory():
if sys.platform in ('win32', 'cygwin'):
stat = MEMORYSTATUSEX(dwLength=ctypes.sizeof(MEMORYSTATUSEX))
success = ctypes.windll.kernel32.GlobalMemoryStatusEx(ctypes.byref(stat))
return stat.ullTotalPhys if success else 0
if sys.platform.startswith('linux'):
if os.path.exists('/proc/meminfo'):
with open('/proc/meminfo') as meminfo:
memtotal_re = re.compile(r'^MemTotal:\s*(\d*)\s*kB')
for line in meminfo:
match = memtotal_re.match(line)
if match:
return float(match.group(1)) * 2**10
if sys.platform == 'darwin':
try:
return int(subprocess.check_output(['sysctl', '-n', 'hw.memsize']))
except: # pylint: disable=bare-except
return 0
return 0
def parse_size(string):
units = {'B': 1, 'KB': 2**10, 'MB': 2**20, 'GB': 2**30, 'TB': 2**40}
i = next(i for (i, c) in enumerate(string) if not c.isdigit())
number = string[:i].strip()
unit = string[i:].strip()
return int(float(number) * units[unit])
def get_concurrent_jobs(reserve_memory, memory_per_job):
# reserve_memory = 2**30
total_memory = get_total_memory()
# Ensure the total memory used in the calculation below is at least 0
mem_total_bytes = max(0, total_memory - parse_size(reserve_memory))
# Ensure the number of cpus used in the calculation below is at least 1
try:
cpu_cap = multiprocessing.cpu_count()
except: # pylint: disable=bare-except
cpu_cap = 1
# Calculate the number of jobs that will fit in memory. Ensure the
# value is at least 1.
num_concurrent_jobs = int(max(1, mem_total_bytes / parse_size(memory_per_job)))
# Cap the number of jobs by the number of cpus available.
concurrent_jobs = min(num_concurrent_jobs, cpu_cap)
return concurrent_jobs
def to_gn_args(args):
if args.simulator:
if args.target_os != 'ios':
@ -529,6 +599,8 @@ def to_gn_args(args):
else:
gn_args['dart_runtime_mode'] = runtime_mode
gn_args['concurrent_dart_jobs'] = get_concurrent_jobs('1GB', '1GB')
# Desktop embeddings can have more dependencies than the engine library,
# which can be problematic in some build environments (e.g., building on
# Linux will bring in pkg-config dependencies at generation time). These
@ -712,12 +784,12 @@ def to_gn_args(args):
if args.prebuilt_impellerc is not None:
gn_args['impeller_use_prebuilt_impellerc'] = args.prebuilt_impellerc
if args.malioc_path is not None:
gn_args['impeller_malioc_path'] = args.malioc_path
else:
malioc_path = args.malioc_path
if not malioc_path:
malioc_path = os.environ.get('MALIOC_PATH')
if malioc_path:
gn_args['impeller_malioc_path'] = malioc_path
if malioc_path:
gn_args['impeller_malioc_path'] = malioc_path
gn_args['impeller_concurrent_malioc_jobs'] = get_concurrent_jobs('1GB', '100MB')
if args.use_glfw_swiftshader:
if get_host_os() == 'mac':

42
engine/src/flutter/tools/gn_test.py
View File

@ -1,10 +1,10 @@
# Copyright 2013 The Flutter Authors. All rights reserved.
# Use of this source code is governed by a BSD-style license that can be
# found in the LICENSE file.
import unittest
import os
import imp
import os
import unittest
SKY_TOOLS = os.path.dirname(os.path.abspath(__file__))
gn = imp.load_source('gn', os.path.join(SKY_TOOLS, 'gn'))
@ -14,17 +14,21 @@ class GNTestCase(unittest.TestCase):
def _expect_build_dir(self, arg_list, expected_build_dir):
args = gn.parse_args(['gn'] + arg_list)
self.assertEquals(gn.get_out_dir(args), expected_build_dir)
self.assertEqual(gn.get_out_dir(args), expected_build_dir)
def test_get_out_dir(self):
self._expect_build_dir(['--debug'], 'out/Debug')
self._expect_build_dir(['--release'], 'out/Release')
self._expect_build_dir(['--ios'], 'out/ios_Debug')
self._expect_build_dir(['--ios'], 'out/ios_Debug_extension_safe')
self._expect_build_dir(['--ios', '--release'], 'out/ios_Release')
self._expect_build_dir(['--ios'], 'out/ios_Release_extension_safe')
self._expect_build_dir(['--android'], 'out/android_Debug')
self._expect_build_dir(['--android', '--release'], 'out/android_Release')
self._expect_build_dir(['--runtime-mode', 'debug'], os.path.join('out', 'host_debug'))
self._expect_build_dir(['--runtime-mode', 'release'], os.path.join('out', 'host_release'))
self._expect_build_dir(['--ios'], os.path.join('out', 'ios_debug'))
self._expect_build_dir(['--ios', '--darwin-extension-safe'],
os.path.join('out', 'ios_debug_extension_safe'))
self._expect_build_dir(['--ios', '--runtime-mode', 'release'],
os.path.join('out', 'ios_release'))
self._expect_build_dir(['--ios', '--darwin-extension-safe', '--runtime-mode', 'release'],
os.path.join('out', 'ios_release_extension_safe'))
self._expect_build_dir(['--android'], os.path.join('out', 'android_debug'))
self._expect_build_dir(['--android', '--runtime-mode', 'release'],
os.path.join('out', 'android_release'))
def _gn_args(self, arg_list):
args = gn.parse_args(['gn'] + arg_list)
@ -32,17 +36,25 @@ class GNTestCase(unittest.TestCase):
def test_to_gn_args(self):
# This would not necessarily be true on a 32-bit machine?
self.assertEquals(self._gn_args(['--ios', '--simulator'])['target_cpu'], 'x64')
self.assertEquals(self._gn_args(['--ios'])['target_cpu'], 'arm')
self.assertEqual(
self._gn_args(['--ios', '--simulator', '--simulator-cpu', 'x64'])['target_cpu'], 'x64'
)
self.assertEqual(self._gn_args(['--ios'])['target_cpu'], 'arm64')
def test_cannot_use_android_and_enable_unittests(self):
with self.assertRaises(SystemExit):
with self.assertRaises(Exception):
self._gn_args(['--android', '--enable-unittests'])
def test_cannot_use_ios_and_enable_unittests(self):
with self.assertRaises(SystemExit):
with self.assertRaises(Exception):
self._gn_args(['--ios', '--enable-unittests'])
def test_parse_size(self):
self.assertEqual(gn.parse_size('5B'), 5)
self.assertEqual(gn.parse_size('5KB'), 5 * 2**10)
self.assertEqual(gn.parse_size('5MB'), 5 * 2**20)
self.assertEqual(gn.parse_size('5GB'), 5 * 2**30)
if __name__ == '__main__':
unittest.main()