From 946e248675efad3af2ebcef50ccfd044ec3ed674 Mon Sep 17 00:00:00 2001 From: Adam Barth Date: Mon, 16 May 2016 10:54:49 -0700 Subject: [PATCH] Revert "Remove vendored dart-sdk (#2688)" We still need this for the observatory build. This reverts commit 716e82edd8fa7b354d6cb5e16a218e8dcbd095fd. --- DEPS | 6 + engine/src/flutter/CONTRIBUTING.md | 1 + .../src/flutter/sky/tools/release_engine.py | 187 ++++++++++++++++++ engine/src/flutter/tools/dart/update.py | 101 ++++++++++ 4 files changed, 295 insertions(+) create mode 100755 engine/src/flutter/sky/tools/release_engine.py create mode 100755 engine/src/flutter/tools/dart/update.py diff --git a/DEPS b/DEPS index 52fb08a91fb..1027ac209f6 100644 --- a/DEPS +++ b/DEPS @@ -133,6 +133,12 @@ hooks = [ 'pattern': '.', 'action': ['python', 'src/tools/clang/scripts/update.py', '--if-needed'], }, + { + # Pull dart sdk if needed + 'name': 'dart', + 'pattern': '.', + 'action': ['python', 'src/tools/dart/update.py'], + }, { # Update LASTCHANGE. This is also run by export_tarball.py in # src/tools/export_tarball - please keep them in sync. diff --git a/engine/src/flutter/CONTRIBUTING.md b/engine/src/flutter/CONTRIBUTING.md index be17b198093..0664f85e5a3 100644 --- a/engine/src/flutter/CONTRIBUTING.md +++ b/engine/src/flutter/CONTRIBUTING.md @@ -46,6 +46,7 @@ target_os = ["android"] * `cd src` (Change to the directory that `gclient sync` created in your `engine` directory.) * `git remote add upstream git@github.com:flutter/engine.git` (So that you fetch from the master repository, not your clone, when running `git fetch` et al.) * Run `./tools/android/download_android_tools.py` to add Android dependencies to your tree. You will need to run this command again if you ever run `git clean -xdf`, since that deletes these dependencies. (`git clean -df` is fine since it skips these `.gitignore`d files.) + * Add `.../engine/src/third_party/dart-sdk/dart-sdk/bin/` to your path so that you can run the `pub` tool more easily. * Add `.../engine/src/third_party/android_tools/sdk/platform-tools` to your path so that you can run the `adb` tool more easily. This is also required by the `flutter` tool, which is used to run Flutter apps. * Make sure you are still in the `src` directory that the `gclient sync` step created earlier. * If you're on Linux, run `sudo ./build/install-build-deps-android.sh` diff --git a/engine/src/flutter/sky/tools/release_engine.py b/engine/src/flutter/sky/tools/release_engine.py new file mode 100755 index 00000000000..d9dc824f461 --- /dev/null +++ b/engine/src/flutter/sky/tools/release_engine.py @@ -0,0 +1,187 @@ +#!/usr/bin/env python +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +# See https://github.com/domokit/sky_engine/wiki/Release-process + +import argparse +import os +import subprocess +import sys +import distutils.util +import tempfile +import zipfile + + +DRY_RUN = False + + +def run(cwd, args): + print 'RUNNING:', ' '.join(args), 'IN:', cwd + if DRY_RUN: + return + subprocess.check_call(args, cwd=cwd) + + +def confirm(prompt): + user_input = raw_input("%s (y/N) " % prompt) + try: + return distutils.util.strtobool(user_input) == 1 + except ValueError: + return False + + +def git_revision(cwd): + return subprocess.check_output([ + 'git', 'rev-parse', 'HEAD', + ], cwd=cwd).strip() + + +GS_URL = 'gs://mojo/flutter/%(commit_hash)s/%(config)s/%(name)s' + + +# Paths of the artifacts that will be packaged into a zip file. +ZIP_ARTIFACTS = { + 'android-arm': [ + 'chromium-debug.keystore', + 'icudtl.dat', + 'dist/shell/SkyShell.apk', + 'dist/shell/flutter.mojo', + 'gen/sky/shell/shell/classes.dex.jar', + 'gen/sky/shell/shell/shell/libs/armeabi-v7a/libsky_shell.so', + # TODO(mpcomplete): obsolete. Remove after updating the flutter tool. + 'gen/sky/shell/shell/classes.dex', + ], + 'linux-x64': [ + 'dist/shell/icudtl.dat', + 'dist/shell/sky_shell', + 'dist/shell/sky_snapshot', + 'dist/shell/flutter.mojo', + ], +} + + +# Paths of the artifacts that will be uploaded to GCS as individual files. +FILE_ARTIFACTS = { + 'android-arm': [ + 'dist/shell/flutter.mojo', + 'dist/shell/libflutter_library.so', + ], + 'linux-x64': [ + 'dist/shell/flutter.mojo', + 'dist/shell/libflutter_library.so', + ], +} + + +def find_missing_artifacts(config, config_root): + result = [] + for artifact_map in [ZIP_ARTIFACTS, FILE_ARTIFACTS]: + for artifact_path in artifact_map[config]: + full_path = os.path.join(config_root, artifact_path) + if not os.path.exists(full_path): + result.append(full_path) + return result + + +# Do not try to compress file types that are already compressed. +FILE_TYPE_COMPRESSION = { + '.apk': zipfile.ZIP_STORED, +} + + +def upload_artifacts(dist_root, config, commit_hash): + # Build and upload a zip file of artifacts + zip_fd, zip_filename = tempfile.mkstemp('.zip', 'artifacts_') + try: + os.close(zip_fd) + artifact_zip = zipfile.ZipFile(zip_filename, 'w') + for artifact_path in ZIP_ARTIFACTS[config]: + _, extension = os.path.splitext(artifact_path) + artifact_zip.write(os.path.join(dist_root, artifact_path), + os.path.basename(artifact_path), + FILE_TYPE_COMPRESSION.get(extension, zipfile.ZIP_DEFLATED)) + artifact_zip.close() + dst = GS_URL % { + 'config': config, + 'commit_hash': commit_hash, + 'name': 'artifacts.zip', + } + run(dist_root, ['gsutil', 'cp', zip_filename, dst]) + finally: + os.remove(zip_filename) + + # Upload individual file artifacts + for artifact_path in FILE_ARTIFACTS[config]: + dst = GS_URL % { + 'config': config, + 'commit_hash': commit_hash, + 'name': os.path.basename(artifact_path), + } + z = ','.join([ 'mojo', 'so' ]) + run(dist_root, ['gsutil', 'cp', '-z', z, artifact_path, dst]) + + +def main(): + parser = argparse.ArgumentParser(description='Deploy!') + parser.add_argument('--dry-run', action='store_true', default=False, + help='Just print commands w/o executing.') + parser.add_argument('--revision', help='The git revision to publish.') + args = parser.parse_args() + + global DRY_RUN + DRY_RUN = args.dry_run + + engine_root = os.path.abspath('.') + if not os.path.exists(os.path.join(engine_root, 'sky')): + print "Cannot find //sky. Is %s the Flutter engine repository?" % engine_root + return 1 + + commit_hash = git_revision(engine_root) + + if commit_hash != args.revision: + print "Current revision %s does not match requested revision %s." % (commit_hash, args.revision) + print "Please update the current revision to %s." % args.revision + return 1 + + # Derived paths: + dart_sdk_root = os.path.join(engine_root, 'third_party/dart-sdk/dart-sdk') + pub_path = os.path.join(dart_sdk_root, 'bin/pub') + android_out_root = os.path.join(engine_root, 'out/android_Release') + linux_out_root = os.path.join(engine_root, 'out/Release') + sky_engine_package_root = os.path.join(android_out_root, 'dist/packages/sky_engine/sky_engine') + sky_services_package_root = os.path.join(android_out_root, 'dist/packages/sky_services/sky_services') + sky_engine_revision_file = os.path.join(sky_engine_package_root, 'lib', 'REVISION') + + run(engine_root, ['sky/tools/gn', '--android', '--release']) + run(engine_root, ['ninja', '-C', 'out/android_Release', ':dist']) + + run(engine_root, ['sky/tools/gn', '--release']) + run(engine_root, ['ninja', '-C', 'out/Release', ':dist']) + + with open(sky_engine_revision_file, 'w') as stream: + stream.write(commit_hash) + + configs = [('android-arm', android_out_root), + ('linux-x64', linux_out_root)] + + # Check for missing artifact files + missing_artifacts = [] + for config, config_root in configs: + missing_artifacts.extend(find_missing_artifacts(config, config_root)) + if missing_artifacts: + print ('Build is missing files:\n%s' % + '\n'.join('\t%s' % path for path in missing_artifacts)) + return 1 + + # Upload artifacts + for config, config_root in configs: + upload_artifacts(config_root, config, commit_hash) + + run(sky_engine_package_root, [pub_path, 'publish', '--force']) + run(sky_services_package_root, [pub_path, 'publish', '--force']) + + +if __name__ == '__main__': + sys.exit(main()) diff --git a/engine/src/flutter/tools/dart/update.py b/engine/src/flutter/tools/dart/update.py new file mode 100755 index 00000000000..41532ab72d7 --- /dev/null +++ b/engine/src/flutter/tools/dart/update.py @@ -0,0 +1,101 @@ +#!/usr/bin/python +# Copyright 2015 The Chromium Authors. All rights reserved. +# Use of this source code is governed by a BSD-style license that can be +# found in the LICENSE file. + +"""Pulls down the current dart sdk to third_party/dart-sdk/. + +You can manually force this to run again by removing +third_party/dart-sdk/STAMP_FILE, which contains the URL of the SDK that +was downloaded. Rolling works by updating LINUX_64_SDK to a new URL. +""" + +import os +import shutil +import subprocess +import sys + +# How to roll the dart sdk: Just change this url! We write this to the stamp +# file after we download, and then check the stamp file for differences. +SDK_URL_BASE = ('http://gsdview.appspot.com/dart-archive/channels/stable/raw/' + '1.14.1/sdk/') + +LINUX_64_SDK = 'dartsdk-linux-x64-release.zip' +MACOS_64_SDK = 'dartsdk-macos-x64-release.zip' + +# Path constants. (All of these should be absolute paths.) +THIS_DIR = os.path.abspath(os.path.dirname(__file__)) +MOJO_DIR = os.path.abspath(os.path.join(THIS_DIR, '..', '..')) +DART_SDK_DIR = os.path.join(MOJO_DIR, 'third_party', 'dart-sdk') +STAMP_FILE = os.path.join(DART_SDK_DIR, 'STAMP_FILE') +LIBRARIES_FILE = os.path.join(DART_SDK_DIR,'dart-sdk', + 'lib', '_internal', 'libraries.dart') +PATCH_FILE = os.path.join(MOJO_DIR, 'tools', 'dart', 'patch_sdk.diff') + +def RunCommand(command, fail_hard=True): + """Run command and return success (True) or failure; or if fail_hard is + True, exit on failure.""" + + print 'Running %s' % (str(command)) + if subprocess.call(command, shell=False) == 0: + return True + print 'Failed.' + if fail_hard: + sys.exit(1) + return False + +def main(): + # Only get the SDK if we don't have a stamp for or have an out of date stamp + # file. + get_sdk = False + if sys.platform.startswith('linux'): + sdk_url = SDK_URL_BASE + LINUX_64_SDK + output_file = os.path.join(DART_SDK_DIR, LINUX_64_SDK) + elif sys.platform.startswith('darwin'): + sdk_url = SDK_URL_BASE + MACOS_64_SDK + output_file = os.path.join(DART_SDK_DIR, MACOS_64_SDK) + else: + print "Platform not supported" + return 1 + + if not os.path.exists(STAMP_FILE): + get_sdk = True + else: + # Get the contents of the stamp file. + with open(STAMP_FILE, "r") as stamp_file: + stamp_url = stamp_file.read().replace('\n', '') + if stamp_url != sdk_url: + get_sdk = True + + if get_sdk: + # Completely remove all traces of the previous SDK. + if os.path.exists(DART_SDK_DIR): + shutil.rmtree(DART_SDK_DIR) + os.mkdir(DART_SDK_DIR) + + # Download the Linux x64 based Dart SDK. + # '-C -': Resume transfer if possible. + # '--location': Follow Location: redirects. + # '-o': Output file. + curl_command = ['curl', + '-C', '-', + '--location', + '-o', output_file, + sdk_url] + if not RunCommand(curl_command, fail_hard=False): + print "Failed to get dart sdk from server." + return 1 + + # Write our stamp file so we don't redownload the sdk. + with open(STAMP_FILE, "w") as stamp_file: + stamp_file.write(sdk_url) + + unzip_command = ['unzip', '-o', '-q', output_file, '-d', DART_SDK_DIR] + if not RunCommand(unzip_command, fail_hard=False): + print "Failed to unzip the dart sdk." + return 1 + + return 0 + +if __name__ == '__main__': + sys.exit(main())