From 3c07f99d6790b5d3dd3ac1c0fa0923cd266fcf9d Mon Sep 17 00:00:00 2001 From: Adam Barth Date: Wed, 22 Jul 2015 10:41:17 -0700 Subject: [PATCH] Update big_red_button to push to Google storage This patch updates big_red_button to release the artifacts produced from the |dist| target in ninja. --- .../src/flutter/sky/tools/big_red_button.py | 81 +++++++---- engine/src/flutter/sky/tools/deploy_sdk.py | 129 ------------------ 2 files changed, 56 insertions(+), 154 deletions(-) delete mode 100755 engine/src/flutter/sky/tools/deploy_sdk.py diff --git a/engine/src/flutter/sky/tools/big_red_button.py b/engine/src/flutter/sky/tools/big_red_button.py index bf49cf070ab..ac376b399ea 100755 --- a/engine/src/flutter/sky/tools/big_red_button.py +++ b/engine/src/flutter/sky/tools/big_red_button.py @@ -13,7 +13,7 @@ import distutils.util CONFIRM_MESSAGE = """This tool is destructive and will revert your current branch to -origin/master among other things. Are you sure you wish to continue?""" +upstream/master among other things. Are you sure you wish to continue?""" DRY_RUN = False @@ -32,11 +32,50 @@ def confirm(prompt): return False +def git_revision(cwd): + return subprocess.check_output([ + 'git', 'rev-parse', 'HEAD', + ], cwd=cwd).strip() + + +class Artifact(object): + def __init__(self, category, name): + self.category = category + self.name = name + + +GS_URL = 'gs://mojo/sky/%(category)s/%(config)s/%(commit_hash)s/%(name)s' + + +ARTIFACTS = { + 'android-arm': [ + Artifact('shell', 'SkyDemo.apk'), + Artifact('viewer', 'sky_viewer.mojo'), + ], + 'linux-x64': [ + Artifact('shell', 'icudtl.dat'), + Artifact('shell', 'sky_shell'), + Artifact('viewer', 'sky_viewer.mojo'), + ] +} + + +def upload_artifacts(dist_root, config, commit_hash): + for artifact in ARTIFACTS[config]: + src = os.path.join(artifact.category, artifact.name) + dst = GS_URL % { + 'category': artifact.category, + 'config': config, + 'commit_hash': commit_hash, + 'name': artifact.name, + } + z = ','.join([ 'mojo', 'dat' ]) + run(dist_root, ['gsutil', 'cp', '-z', z, src, dst]) + + def main(): parser = argparse.ArgumentParser(description='Deploy!') parser.add_argument('sky_engine_root', help='Path to sky_engine/src') - parser.add_argument('sky_sdk_root', help='Path to sky_sdk') - parser.add_argument('demo_site_root', help='Path to domokit.github.io') parser.add_argument('--dry-run', action='store_true', default=False, help='Just print commands w/o executing.') parser.add_argument('--no-pub-publish', dest='publish', @@ -51,39 +90,31 @@ def main(): return 1 sky_engine_root = os.path.abspath(os.path.expanduser(args.sky_engine_root)) - sky_sdk_root = os.path.abspath(os.path.expanduser(args.sky_sdk_root)) - demo_site_root = os.path.abspath(os.path.expanduser(args.demo_site_root)) # Derived paths: dart_sdk_root = os.path.join(sky_engine_root, 'third_party/dart-sdk/dart-sdk') pub_path = os.path.join(dart_sdk_root, 'bin/pub') - packages_root = os.path.join(sky_sdk_root, 'packages') + android_dist_root = os.path.join(sky_engine_root, 'out/android_Release/dist') + linux_dist_root = os.path.join(sky_engine_root, 'out/Release/dist') + sky_package_root = os.path.join(linux_dist_root, 'sdk/sky') - run(sky_engine_root, ['git', 'pull', '--rebase']) + run(sky_engine_root, ['git', 'fetch', 'upstream']) + run(sky_engine_root, ['git', 'reset', 'upstream/master', '--hard']) run(sky_engine_root, ['gclient', 'sync']) + + commit_hash = git_revision(sky_engine_root) + run(sky_engine_root, ['sky/tools/gn', '--android', '--release']) - # TODO(eseidel): We shouldn't use mojob anymore, it likely will break. - run(sky_engine_root, ['mojo/tools/mojob.py', 'build', '--android', '--release']) - # Run tests? + run(sky_engine_root, ['ninja', '-C', 'out/android_Release', ':dist']) - run(sky_engine_root, [ - 'sky/tools/deploy_sdk.py', - '--non-interactive', - sky_sdk_root - ]) - # tag for version? + run(sky_engine_root, ['sky/tools/gn', '--release']) + run(sky_engine_root, ['ninja', '-C', 'out/Release', ':dist']) - run(demo_site_root, ['git', 'fetch']) - run(demo_site_root, ['git', 'reset', '--hard', 'origin/master']) - # TODO(eseidel): We should move this script back into sky/tools. - run(sky_engine_root, ['mojo/tools/deploy_domokit_site.py', demo_site_root]) - # tag for version? + upload_artifacts(android_dist_root, 'android-arm', commit_hash) + upload_artifacts(linux_dist_root, 'linux-x64', commit_hash) if args.publish: - package_path = os.path.join(packages_root, 'sky') - run(package_path, [pub_path, 'publish', '--force']) - - run(demo_site_root, ['git', 'push']) + run(sky_package_root, [pub_path, 'publish', '--force']) if __name__ == '__main__': diff --git a/engine/src/flutter/sky/tools/deploy_sdk.py b/engine/src/flutter/sky/tools/deploy_sdk.py deleted file mode 100755 index 5770f818d77..00000000000 --- a/engine/src/flutter/sky/tools/deploy_sdk.py +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python -# Copyright 2015 The Chromium Authors. All rights reserved. -# Use of this source code is governed by a BSD-style license that can be -# found in the LICENSE file. - -import argparse -from datetime import datetime -import logging -import os -import shutil -import subprocess -import sys - -# Generates the sky_sdk from the template at sky/sdk. - -SKY_TOOLS_DIR = os.path.dirname(os.path.abspath(__file__)) -SKY_DIR = os.path.dirname(SKY_TOOLS_DIR) -SRC_ROOT = os.path.dirname(SKY_DIR) - -DEFAULT_REL_BUILD_DIR = os.path.join('out', 'android_Release') - -def git_revision(): - return subprocess.check_output(['git', 'rev-parse', 'HEAD']).strip() - - -def ensure_dir_exists(path): - if not os.path.exists(path): - os.makedirs(path) - - -def copy(from_root, to_root, filter_func=None): - assert os.path.exists(from_root), "%s does not exist!" % from_root - if os.path.isfile(from_root): - ensure_dir_exists(os.path.dirname(to_root)) - shutil.copy(from_root, to_root) - return - - ensure_dir_exists(to_root) - - for root, dirs, files in os.walk(from_root): - # filter_func expects paths not names, so wrap it to make them absolute. - wrapped_filter = None - if filter_func: - wrapped_filter = lambda name: filter_func(os.path.join(root, name)) - - for name in filter(wrapped_filter, files): - from_path = os.path.join(root, name) - root_rel_path = os.path.relpath(from_path, from_root) - to_path = os.path.join(to_root, root_rel_path) - to_dir = os.path.dirname(to_path) - if not os.path.exists(to_dir): - os.makedirs(to_dir) - shutil.copy(from_path, to_path) - - dirs[:] = filter(wrapped_filter, dirs) - - -def confirm(prompt): - response = raw_input('%s [N]|y: ' % prompt) - return response and response.lower() == 'y' - - -def delete_all_non_hidden_files_in_directory(root, non_interactive=False): - to_delete = [os.path.join(root, p) - for p in os.listdir(root) if not p.startswith('.')] - if not to_delete: - return - if not non_interactive: - prompt = 'This will delete everything in %s:\n%s\nAre you sure?' % ( - root, '\n'.join(to_delete)) - if not confirm(prompt): - print 'User aborted.' - sys.exit(2) - - for path in to_delete: - if os.path.isdir(path) and not os.path.islink(path): - shutil.rmtree(path) - else: - os.remove(path) - - -def main(): - logging.basicConfig(level=logging.WARN) - parser = argparse.ArgumentParser(description='Deploy a new sky_sdk.') - parser.add_argument('sdk_root', type=str) - parser.add_argument('--build-dir', action='store', type=str, - default=os.path.join(SRC_ROOT, DEFAULT_REL_BUILD_DIR)) - parser.add_argument('--non-interactive', action='store_true') - args = parser.parse_args() - - build_dir = os.path.abspath(args.build_dir) - sdk_root = os.path.abspath(args.sdk_root) - - print 'Building SDK from %s into %s' % (build_dir, sdk_root) - start_time = datetime.now() - - def sdk_path(rel_path): - return os.path.join(sdk_root, rel_path) - - def src_path(rel_path): - return os.path.join(SRC_ROOT, rel_path) - - ensure_dir_exists(sdk_root) - # Manually clear sdk_root above to avoid deleting dot-files. - delete_all_non_hidden_files_in_directory(sdk_root, args.non_interactive) - - copy(src_path('sky/sdk/README.md'), sdk_root) - - ensure_dir_exists(sdk_path('packages')) - subprocess.check_call([ - 'mojo/tools/prepare_pub_packages.py', - '--out-dir', sdk_path('packages'), - 'out' - ]) - - with open(sdk_path('LICENSES.sky'), 'w') as license_file: - subprocess.check_call([src_path('tools/licenses.py'), 'credits'], - stdout=license_file) - - material_design_icons = sdk_path('packages/sky/lib/assets/material-design-icons') - if os.path.exists(material_design_icons): - shutil.rmtree(material_design_icons) - - time_delta = datetime.now() - start_time - print 'SDK built at %s in %ss' % (sdk_root, time_delta.total_seconds()) - - -if __name__ == '__main__': - main()