From faf29ed4589f4d025dd3d2349f34536de28844b7 Mon Sep 17 00:00:00 2001 From: Zachary Anderson Date: Tue, 17 May 2022 12:56:44 -0700 Subject: [PATCH] Use CIPD to download prebuilt Dart SDKs (flutter/engine#33345) --- DEPS | 62 +++- engine/src/flutter/build/archives/BUILD.gn | 13 +- engine/src/flutter/common/config.gni | 5 - engine/src/flutter/tools/download_dart_sdk.py | 293 ------------------ 4 files changed, 61 insertions(+), 312 deletions(-) delete mode 100755 engine/src/flutter/tools/download_dart_sdk.py diff --git a/DEPS b/DEPS index 7e17de56630..c3654f4b2f6 100644 --- a/DEPS +++ b/DEPS @@ -351,6 +351,58 @@ deps = { # WARNING: end of dart dependencies list that is cleaned up automatically - see create_updated_flutter_deps.py. + # Prebuilt Dart SDK of the same revision as the Dart SDK source checkout + 'src/flutter/prebuilts/linux-x64/dart-sdk': { + 'packages': [ + { + 'package': 'flutter/dart-sdk/linux-amd64', + 'version': 'git_revision:'+Var('dart_revision') + } + ], + 'dep_type': 'cipd', + 'condition': 'host_os == "linux"' + }, + 'src/flutter/prebuilts/linux-arm64/dart-sdk': { + 'packages': [ + { + 'package': 'flutter/dart-sdk/linux-arm64', + 'version': 'git_revision:'+Var('dart_revision') + } + ], + 'dep_type': 'cipd', + 'condition': 'host_os == "linux"' + }, + 'src/flutter/prebuilts/macos-x64/dart-sdk': { + 'packages': [ + { + 'package': 'flutter/dart-sdk/mac-amd64', + 'version': 'git_revision:'+Var('dart_revision') + } + ], + 'dep_type': 'cipd', + 'condition': 'host_os == "mac"' + }, + 'src/flutter/prebuilts/macos-arm64/dart-sdk': { + 'packages': [ + { + 'package': 'flutter/dart-sdk/mac-arm64', + 'version': 'git_revision:'+Var('dart_revision') + } + ], + 'dep_type': 'cipd', + 'condition': 'host_os == "mac"' + }, + 'src/flutter/prebuilts/windows-x64/dart-sdk': { + 'packages': [ + { + 'package': 'flutter/dart-sdk/windows-amd64', + 'version': 'git_revision:'+Var('dart_revision') + } + ], + 'dep_type': 'cipd', + 'condition': 'host_os == "win"' + }, + 'src/third_party/colorama/src': Var('chromium_git') + '/external/colorama.git' + '@' + '799604a1041e9b3bc5d2789ecbd7e8db2e18e6b8', @@ -680,16 +732,6 @@ hooks = [ 'src/flutter/tools/pub_get_offline.py', ] }, - { - 'name': 'Download prebuilt Dart SDK', - 'pattern': '.', - 'condition': 'download_dart_sdk', - 'action': [ - 'python3', - 'src/flutter/tools/download_dart_sdk.py', - '--fail-loudly', - ] - }, { 'name': 'Download Fuchsia SDK', 'pattern': '.', diff --git a/engine/src/flutter/build/archives/BUILD.gn b/engine/src/flutter/build/archives/BUILD.gn index 777ee5118cf..d751a6fe4f6 100644 --- a/engine/src/flutter/build/archives/BUILD.gn +++ b/engine/src/flutter/build/archives/BUILD.gn @@ -104,10 +104,15 @@ zip_bundle("flutter_patched_sdk") { # Flutter consumes the dart sdk as a prebuilt. Rather than regenerating # the zip file we are just copying the original file to the artifacts location. if (build_engine_artifacts && flutter_prebuilt_dart_sdk) { - copy("dart_sdk_archive") { - sources = [ prebuilt_dart_sdk_archive ] - outputs = - [ "$root_out_dir/zip_archives/dart-sdk-$full_target_platform_name.zip" ] + zip_bundle("dart_sdk_archive") { + deps = [] + output = "dart-sdk-$full_target_platform_name.zip" + files = [ + { + source = prebuilt_dart_sdk + destination = "dart-sdk" + }, + ] } } diff --git a/engine/src/flutter/common/config.gni b/engine/src/flutter/common/config.gni index c17e95dc12a..26df26700f6 100644 --- a/engine/src/flutter/common/config.gni +++ b/engine/src/flutter/common/config.gni @@ -112,9 +112,6 @@ if (flutter_prebuilt_dart_sdk) { _target_prebuilt_dart_sdk_config = "$_target_os_name-$target_cpu" _host_prebuilt_dart_sdk_config = "$_host_os_name-$host_cpu" - _target_prebuilt_dart_sdk_archive = "//flutter/prebuilts/dartsdk-$_target_prebuilt_dart_sdk_config-release.zip" - _host_prebuilt_dart_sdk_archive = - "//flutter/prebuilts/dartsdk-$_host_prebuilt_dart_sdk_config-release.zip" target_prebuilt_dart_sdk = "//flutter/prebuilts/$_target_prebuilt_dart_sdk_config/dart-sdk" @@ -127,11 +124,9 @@ if (flutter_prebuilt_dart_sdk) { if (current_toolchain == host_toolchain || is_fuchsia) { prebuilt_dart_sdk = host_prebuilt_dart_sdk prebuilt_dart_sdk_config = _host_prebuilt_dart_sdk_config - prebuilt_dart_sdk_archive = _host_prebuilt_dart_sdk_archive } else { prebuilt_dart_sdk = target_prebuilt_dart_sdk prebuilt_dart_sdk_config = _target_prebuilt_dart_sdk_config - prebuilt_dart_sdk_archive = _target_prebuilt_dart_sdk_archive } } diff --git a/engine/src/flutter/tools/download_dart_sdk.py b/engine/src/flutter/tools/download_dart_sdk.py deleted file mode 100755 index 80804ebdc34..00000000000 --- a/engine/src/flutter/tools/download_dart_sdk.py +++ /dev/null @@ -1,293 +0,0 @@ -#!/usr/bin/env python3 -# Copyright 2013 The Flutter Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -# When the environment variable in FLUTTER_PREBUILTS_ENV_VAR below is defined -# and is not '0' or 'false', this script downloads the Dart SDK that matches the -# version in the source tree and puts it in prebuilts/. -# -# The return code of this script will always be 0, even if there is an error, -# unless the --fail-loudly flag is passed. - -# TODO(zra): Eliminate this script and download through the DEPS file if/when -# the Dart SDKs pulled by this script are uploaded to cipd. - -import argparse -import os -import multiprocessing -import platform -import re -import shutil -import subprocess -import sys -import zipfile - - -SRC_ROOT = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) -FLUTTER_DIR = os.path.join(SRC_ROOT, 'flutter') -FLUTTER_PREBUILTS_DIR = os.path.join(FLUTTER_DIR, 'prebuilts') -DART_DIR = os.path.join(SRC_ROOT, 'third_party', 'dart') -DART_VERSION = os.path.join(DART_DIR, 'tools', 'VERSION') -FLUTTER_PREBUILTS_ENV_VAR = 'FLUTTER_PREBUILT_DART_SDK' - - -# The Dart SDK script is the source of truth about the sematic version. -sys.path.append(os.path.join(DART_DIR, 'tools')) -import utils - - -# Prints to stderr. -def eprint(*args, **kwargs): - print(*args, file=sys.stderr, **kwargs) - - -# Try to guess the host operating system. -def GuessOS(): - os_name = utils.GuessOS() - if os_name == 'win32': - os_name = 'windows' - if os_name not in ['linux', 'macos', 'windows']: - eprint('Could not determine the OS: "%s"' % os_name) - return None - return os_name - - -# For os `os_name` return a list of architectures for which prebuilts are -# supported. Kepp in sync with `can_use_prebuilt_dart` in //flutter/tools/gn. -def ArchitecturesForOS(os_name): - if os_name == 'linux': - return ['x64', 'arm64'] - elif os_name == 'macos': - return ['x64', 'arm64'] - elif os_name =='windows': - return ['x64'] - - eprint('Could not determine architectures for os "%s"' % os_name) - return None - - -# Downloads a Dart SDK to //flutter/prebuilts. -def DownloadDartSDK(channel, version, os_name, arch, verbose): - file = 'dartsdk-{}-{}-release.zip'.format(os_name, arch) - url = 'https://storage.googleapis.com/dart-archive/channels/{}/raw/{}/sdk/{}'.format( - channel, version, file, - ) - dest = os.path.join(FLUTTER_PREBUILTS_DIR, file) - - if verbose: - print('Dart SDK url: "%s"' % url) - print('Dart SDK destination path: "%s"' % dest) - - stamp_file = '{}.stamp'.format(dest) - version_stamp = None - try: - with open(stamp_file) as fd: - version_stamp = fd.read() - except: - version_stamp = 'none' - - if verbose: - print('Dart SDK version stamp = "%s"' % version_stamp) - - if version == version_stamp: - # The prebuilt Dart SDK is already up-to-date. Indicate that the download - # should be skipped by returning the empty string. - if verbose: - print('Dart SDK stamp files match. Skipping download.') - return '' - - if os.path.isfile(dest): - os.unlink(dest) - - curl_command = [ - 'curl', - '--retry', '3', - '--continue-at', '-', '--location', - '--output', dest, - url, - ] - if verbose: - print('Running: "%s"' % (' '.join(curl_command))) - curl_result = subprocess.run( - curl_command, - stdout=subprocess.PIPE, - stderr=subprocess.PIPE, - universal_newlines=True, - ) - if curl_result.returncode == 0 and verbose: - print('curl output:stdout:\n{}\nstderr:\n{}'.format( - curl_result.stdout, curl_result.stderr, - )) - elif curl_result.returncode != 0: - eprint('Failed to download: stdout:\n{}\nstderr:\n{}'.format( - curl_result.stdout, curl_result.stderr, - )) - return None - - return dest - - -# A custom ZipFile class that preserves file permissions. -class ZipFileWithPermissions(zipfile.ZipFile): - def _extract_member(self, member, targetpath, pwd): - if not isinstance(member, zipfile.ZipInfo): - member = self.getinfo(member) - - targetpath = super()._extract_member(member, targetpath, pwd) - - attr = member.external_attr >> 16 - if attr != 0: - os.chmod(targetpath, attr) - return targetpath - - -def OnErrorRmTree(func, path, exc_info): - """ - Error handler for ``shutil.rmtree``. - - If the error is due to an access error (read only file) - it attempts to add write permission and then retries. - If the error is for another reason it re-raises the error. - - Usage : ``shutil.rmtree(path, onerror=onerror)`` - """ - import stat - # Is the error an access error? - if not os.access(path, os.W_OK): - os.chmod(path, stat.S_IWUSR) - func(path) - else: - raise - -# Extracts a Dart SDK in //fluter/prebuilts -def ExtractDartSDK(archive, os_name, arch, verbose): - os_arch = '{}-{}'.format(os_name, arch) - dart_sdk = os.path.join(FLUTTER_PREBUILTS_DIR, os_arch, 'dart-sdk') - if os.path.isdir(dart_sdk): - shutil.rmtree(dart_sdk, onerror=OnErrorRmTree) - - extract_dest = os.path.join(FLUTTER_PREBUILTS_DIR, os_arch, 'temp') - if os.path.isdir(extract_dest): - shutil.rmtree(extract_dest, onerror=OnErrorRmTree) - os.makedirs(extract_dest, exist_ok=True) - - if verbose: - print('Extracting "%s" to "%s"' % (archive, extract_dest)) - - with ZipFileWithPermissions(archive, "r") as z: - z.extractall(extract_dest) - - shutil.move(os.path.join(extract_dest, 'dart-sdk'), dart_sdk) - - -def PrintFileIfSmall(file): - if not os.path.isfile(file): - return - size = os.path.getsize(file) - if size < (1 << 14): # < 16KB - with open(file) as f: - contents = f.read() - eprint(contents) - - -def DownloadAndExtract(channel, version, os_name, arch, verbose): - archive = DownloadDartSDK(channel, version, os_name, arch, verbose) - if archive == None: - return 1 - if archive == '': - return 0 - try: - ExtractDartSDK(archive, os_name, arch, verbose) - except Exception as e: - eprint('Failed to extract Dart SDK archive:\n%s' % e) - PrintFileIfSmall(archive) - return 1 - try: - stamp_file = '{}.stamp'.format(archive) - with open(stamp_file, "w") as fd: - fd.write(version) - except Exception as e: - eprint('Failed to write Dart SDK version stamp file:\n{}'.format(e)) - return 1 - return 0 - - -def Main(): - parser = argparse.ArgumentParser() - parser.add_argument( - '--fail-loudly', - action='store_true', - default=False, - help="Return an error code if a prebuilt couldn't be fetched and extracted") - parser.add_argument( - '--verbose', - action='store_true', - default='LUCI_CONTEXT' in os.environ, - help='Emit verbose output') - args = parser.parse_args() - fail_loudly = 1 if args.fail_loudly else 0 - verbose = args.verbose - - prebuilt_enabled = os.environ.get(FLUTTER_PREBUILTS_ENV_VAR, 'true') - if prebuilt_enabled == '0' or prebuilt_enabled.lower() == 'false': - if verbose: - print('Skipping prebuild Dart SDK download.') - return 0 - - os.makedirs(FLUTTER_PREBUILTS_DIR, exist_ok=True) - - # Read //third_party/dart/tools/VERSION to extract information about the - # Dart SDK version. - version = utils.ReadVersionFile() - if version == None: - eprint('Failed to read the Dart VERSION file.') - return fail_loudly - channel = version.channel - if verbose: - print('Dart SDK channel = "%s".' % channel) - - # A short Dart SDK version string used in the download url. - if channel == 'be': - dart_git_rev = utils.GetGitRevision() - semantic_version = 'hash/{}'.format(dart_git_rev) - else: - semantic_version = utils.GetSemanticSDKVersion() - if verbose: - print('Semantic Dart SDK version = "%s".' % semantic_version) - - os_name = GuessOS() - if os_name == None: - return fail_loudly - - architectures = ArchitecturesForOS(os_name) - if architectures == None: - return fail_loudly - - # Work around a bug in Python. - # - # The multiprocessing package relies on the win32 WaitForMultipleObjects() - # call, which supports waiting on a maximum of MAXIMUM_WAIT_OBJECTS (defined - # by Windows to be 64) handles, processes in this case. To avoid hitting - # this, we limit ourselves to 60 handles (since there are a couple extra - # processes launched for the queue reader and thread wakeup reader). - # - # See: https://bugs.python.org/issue26903 - max_processes = os.cpu_count() - if sys.platform.startswith(('cygwin', 'win')) and max_processes > 60: - max_processes = 60 - - # Download and extract variants in parallel - pool = multiprocessing.Pool(processes=max_processes) - tasks = [(channel, semantic_version, os_name, arch, verbose) for arch in architectures] - async_results = [pool.apply_async(DownloadAndExtract, t) for t in tasks] - success = True - for async_result in async_results: - result = async_result.get() - success = success and (result == 0) - - return 0 if success else fail_loudly - - -if __name__ == '__main__': - sys.exit(Main())