diff --git a/b9/b9 b/b9/b9 deleted file mode 100755 index b052ec9f..00000000 --- a/b9/b9 +++ /dev/null @@ -1,133 +0,0 @@ -#!/bin/bash - -set -e -set -o pipefail - -export CLOUDSDK_PYTHON_SITEPACKAGES=1 - -SCRIPT_NAME=$BASH_SOURCE -if [ -h $SCRIPT_NAME ]; then SCRIPT_NAME=$(readlink $SCRIPT_NAME); fi - -cd $(dirname $SCRIPT_NAME) - -B9_DIR=$(pwd) -C9_DIR=$B9_DIR/.. -B9=$B9_DIR/b9 - -source ./lib/_init.sh - -MODE=$(_b9_init_mode) -NODEJS="$(_b9_init_nodejs)" -NPM="$(_b9_init_npm)" - -TMPDIR=$(_b9_init_temp) -TMP=$TMPDIR -TEMP=$TMPDIR - -DEBUG="" - -for MODULE in ./lib/*.sh; do - [ $MODULE == "./lib/_init.sh" ] && continue - source $MODULE -done - -source $C9_DIR/plugins/c9.docker/d9/_testing.sh -source $C9_DIR/plugins/c9.docker/d9/_redis.sh -source $C9_DIR/plugins/c9.docker/d9/_settings.sh - -usage() { - echo "Usage: $B9 [global options ...] COMMAND [commands options...]" - echo - echo "Cloud9 build tool" - echo - echo "Global options:" - echo " --help show this help message" - echo " --debug trace bash commands" - echo " --settings=MODE, -s=MODE which settings file to use" - echo " --deploy shortcut for '-settings=deploy'; use prod database" - echo " --devel shortcut for '-settings=devel'; use devel database" - echo - echo "Commands:" - echo - echo "[Deployment]" - echo " package Package and upload version of Cloud9" - echo " deploy Deploy a Cloud9 version" - echo " static Package static assets" - echo "" - echo "[Internal]" - echo " check Run b9 tests" - echo " exec COMMAND [ARGS] Run arbitrary b9 function" - echo - echo "Environment variables:" - echo "B9_GITHUB_SSH_FILE SSH file to use for downloading from Github" - exit 1 -} - -for ARG in "$@"; do - case $ARG in - --help|-h) - usage - ;; - --debug) - DEBUG="--debug" - B9="$B9 --debug" - shift - ;; - -s=*|--settings=*) - MODE="${ARG#*=}" - shift - ;; - --deploy) - MODE=deploy - shift - ;; - --devel) - MODE=devel - shift - ;; - --odev|--onlinedev) - MODE=onlinedev - shift - ;; - *) - break - ;; - esac -done - -ORIGINAL_COMMAND=$1 -case $ORIGINAL_COMMAND in - package) - COMMAND=b9_package - ;; - deploy) - COMMAND=b9_deploy - ;; - static) - COMMAND=b9_static - ;; - prepare) - COMMAND=b9_prepare - ;; - check) - COMMAND=b9_check - ;; - exec) # for debugging only! - shift - COMMAND=$1 - ;; - "") - usage - ;; - *) - echo "Invalid command. See $B9 --help for usage." - exit 1 - ;; -esac -shift - -if [ "$DEBUG" ]; then - set -x -fi - -$COMMAND "$@" \ No newline at end of file diff --git a/b9/containers/c9/Dockerfile b/b9/containers/c9/Dockerfile deleted file mode 100644 index 77576a76..00000000 --- a/b9/containers/c9/Dockerfile +++ /dev/null @@ -1,21 +0,0 @@ -FROM debian:8.2 - -MAINTAINER Cloud9 IDE, Inc. -ENV DEBIAN_FRONTEND noninteractive - -# ubuntu user -RUN useradd --uid 1000 --shell /bin/bash -m --home-dir /home/ubuntu ubuntu && \ - chmod 777 /tmp - -RUN apt-get update && \ - apt-get install -y curl openssh-client rsync && \ - apt-get clean && \ - rm -rf /var/lib/apt/lists/* /tmp/* /var/tmp/* - -# install nodejs -RUN curl https://nodejs.org/dist/v0.10.41/node-v0.10.41-linux-x64.tar.gz | tar xvzf - -C /usr/local --strip-components=1 - -USER ubuntu - -EXPOSE 8080 -WORKDIR /home/ubuntu/newclient \ No newline at end of file diff --git a/b9/containers/newclient/Dockerfile b/b9/containers/newclient/Dockerfile deleted file mode 100644 index a779c78b..00000000 --- a/b9/containers/newclient/Dockerfile +++ /dev/null @@ -1,52 +0,0 @@ -FROM debian:8.2 - -MAINTAINER Cloud9 IDE, Inc. -ENV DEBIAN_FRONTEND noninteractive - -RUN apt-get update -RUN apt-get install -y curl - -# ubuntu user -RUN useradd --uid 1000 --shell /bin/bash -m --home-dir /home/ubuntu ubuntu && \ - chmod 777 /tmp - -# make ubuntu user sudo -RUN apt-get install -y sudo && \ - sed -i 's/%sudo\s.*/%sudo ALL=NOPASSWD:ALL/' /etc/sudoers && \ - usermod -a -G sudo ubuntu - -RUN chown root:root /usr/bin/sudo && chmod 4755 /usr/bin/sudo - -# install nodejs -RUN mkdir /nodejs && curl https://nodejs.org/dist/v0.10.40/node-v0.10.40-linux-x64.tar.gz | tar xvzf - -C /usr/local --strip-components=1 -RUN npm install -g npm@2.14.11 - -# oldclient -RUN apt-get install -y openssh-client - -# test runner -RUN npm install -g mocha - -# install jsonalzyer dependencies -RUN apt-get install -y golang tmux python python-pip pylint php5 ruby build-essential - -# test runner -RUN apt-get install -y redis-server - -# for odev -RUN apt-get install -y haproxy - -# for CI -RUN apt-get install -y git - -ADD files/ssh_config /home/ubuntu/.ssh/config -RUN chown -R ubuntu:ubuntu /home/ubuntu/.ssh - -USER ubuntu - -# Cloud9 installer -RUN curl -L https://raw.githubusercontent.com/c9/install/master/install.sh | bash - -EXPOSE 8080 -VOLUME /home/ubuntu/newclient -WORKDIR /home/ubuntu/newclient \ No newline at end of file diff --git a/b9/containers/newclient/files/ssh_config b/b9/containers/newclient/files/ssh_config deleted file mode 100644 index 209ddac5..00000000 --- a/b9/containers/newclient/files/ssh_config +++ /dev/null @@ -1,14 +0,0 @@ -Host github.com - User git - Port 22 - Hostname github.com - IdentityFile /home/ubuntu/.ssh/id_rsa_deploy - TCPKeepAlive yes - IdentitiesOnly yes - StrictHostKeyChecking no - -Host static.c9.io - IdentityFile /home/ubuntu/.ssh/id_rsa_deploy - StrictHostKeyChecking no - TCPKeepAlive yes - IdentitiesOnly yes \ No newline at end of file diff --git a/b9/lib/_check_save_deploy.sh b/b9/lib/_check_save_deploy.sh deleted file mode 100644 index 940f7aa4..00000000 --- a/b9/lib/_check_save_deploy.sh +++ /dev/null @@ -1,70 +0,0 @@ -_b9_check_save_deploy_usage() { - echo "Use: check-deploy.sh --service=SERVICE [--server=SERVER] [--wait=WAIT_SECONDS] [--port=PORT]" - exit 1 -} - -_b9_check_save_deploy() { - local SERVICE - local PORT - local WAIT=default - local SERVER=${SERVER:-localhost} - - local P - for P in "$@"; do - case "$P" in - --wait=*) - WAIT=${P#*=} - ;; - --server=*) - SERVER=${P#*=} - ;; - --service=*) - SERVICE=${P#*=} - PORT=$(_do_get_setting $SERVICE.port) - ;; - --port=*) - PORT=${P#*=} - ;; - *) - echo "Illegal argument: $P" - _b9_check_save_deploy_usage - esac - done - - if [ "$WAIT" == "default" ]; then - WAIT=45 - elif [ "$WAIT" == "long" ]; then - WAIT=60 - fi - if [ ! "$PORT" ]; then - _b9_check_save_deploy_usage - fi - - for I in {1..160}; do - if curl http://$SERVER:$PORT/_health &>/dev/null; then - break - fi - sleep 1 - done - - echo "Service $SERVICE running, testing if it stays up for $WAIT seconds..." - - if ! curl http://$SERVER:$PORT/_health?delay=$((WAIT * 1000)) &>/dev/null; then - echo "Failed: service $SERVICE at $SERVER:$PORT failed to stay up for $WAIT seconds" >&2 - echo "Likely it exited with an error, shown in raygun, or the process was killed." >&2 - echo "" >&2 - - echo "Recent errors in service log that might help debug this:" >&2 - if [ "$SERVICE" ] && [ "$(type -t ssh)" != "function" ]; then - echo "Warning: ssh function not defined, trying gssh instead" - gssh $SERVER ${SERVICE//_/-}/scripts/tail-log.sh $(_do_get_setting $SERVICE.logFile) >&2 - elif [ "$SERVICE" ]; then - ssh $SERVER ${SERVICE//_/-}/scripts/tail-log.sh $(_do_get_setting $SERVICE.logFile) >&2 - fi - echo >&2 - - return 1 - fi - - echo "Confirmed successful deploy to $SERVER!" -} \ No newline at end of file diff --git a/b9/lib/_docker.sh b/b9/lib/_docker.sh deleted file mode 100644 index 4403d37c..00000000 --- a/b9/lib/_docker.sh +++ /dev/null @@ -1,17 +0,0 @@ -_DO_NEWCLIENT_IMAGE= -_b9_get_newclient_image() { - if [ ! -z "$_DO_NEWCLIENT_IMAGE" ]; then - echo $_DO_NEWCLIENT_IMAGE - return - fi - - local RESULT=$(docker build -t newclient --rm $B9_DIR/containers/newclient) - if [[ $(echo "$RESULT" | tail -n1) =~ Successfully\ built ]]; then - _DO_NEWCLIENT_IMAGE=$(echo "$RESULT" | tail -n1 | awk '{print $3}') - echo $_DO_NEWCLIENT_IMAGE - return - fi - - echo $RESULT - return 1 -} \ No newline at end of file diff --git a/b9/lib/_git.sh b/b9/lib/_git.sh deleted file mode 100644 index e92c5ac3..00000000 --- a/b9/lib/_git.sh +++ /dev/null @@ -1,15 +0,0 @@ -_b9_git_get_hash() { - pushd $C9_DIR &> /dev/null - - git rev-parse HEAD - - popd &> /dev/null -} - -_b9_git_get_hash_short() { - pushd $C9_DIR &> /dev/null - - git rev-parse --short=10 HEAD - - popd &> /dev/null -} \ No newline at end of file diff --git a/b9/lib/_init.sh b/b9/lib/_init.sh deleted file mode 100644 index 6edad51d..00000000 --- a/b9/lib/_init.sh +++ /dev/null @@ -1,95 +0,0 @@ -_b9_init_mode() { - local SUFFIX - SUFFIX=$(hostname | sed 's/.*-//') - local MODE - - case $SUFFIX in - prod) - MODE=deploy - ;; - dev) - MODE=devel - shift - ;; - test) - MODE=test - ;; - onlinedev) - MODE=$SUFFIX - ;; - *) - MODE=devel - ;; - esac - - if [ ! "$MODE" ]; then - MODE=devel - fi - echo $MODE -} - -_b9_init_temp() { - local TMPDIR - local UNAME=$(id -n -u) - - for TMPDIR in /var/lib/docker/tmp /tmp; do - TMPDIR=$TMPDIR/$UNAME - mkdir -p $TMPDIR &> /dev/null && break - TMPDIR="" - done - - if [ -z "$TMPDIR" ]; then - echo "Can't find temp dir" 1>&2 - exit 1 - fi - - rm $(mktemp --tmpdir=$TMPDIR) - - echo $TMPDIR -} - -_b9_init_nodejs() { - local NODEJS - - . ~/.nvm/nvm.sh &> /dev/null || : - for NODEJS in $(which node 2>/dev/null) $(which nodejs 2>/dev/null) /usr/local/bin/node /usr/bin/nodejs; do - [ -x $NODEJS ] && break - NODEJS="" - done - - if [ -z "$NODEJS" ]; then - echo "Can't find node executable" 1>&2 - exit 1 - fi - - echo $NODEJS -} - -_b9_init_npm() { - local NPM - - . ~/.nvm/nvm.sh &> /dev/null || : - for NPM in $(which npm) /usr/local/bin/npm /usr/bin/npm; do - [ -x $NPM ] && break - NPM="" - done - - if [ -z "$NPM" ]; then - echo "Can't find npm executable" 1>&2 - exit 1 - fi - - echo $NODEJS $NPM -} - -_B9_NODE_HELPER_INITIALIZED=0 - -_b9_init_node_helper() { - [ "$_B9_NODE_HELPER_INITIALIZED" == "1" ] && return - _B9_NODE_HELPER_INITIALIZED=1 - - pushd $B9_DIR/lib/js &> /dev/null - rm -rf node_modules - $NPM install - popd &> /dev/null -} diff --git a/b9/lib/_npm.sh b/b9/lib/_npm.sh deleted file mode 100644 index 7cdef76d..00000000 --- a/b9/lib/_npm.sh +++ /dev/null @@ -1,37 +0,0 @@ - - -_b9_npm() { - local WORKDIR=$1 - shift - - # TODO run all build steps in a container - if [ $(id -u) == "1000" ]; then - docker run \ - --rm \ - -w /home/ubuntu/newclient \ - -v $WORKDIR:/home/ubuntu/newclient \ - --sig-proxy -a STDIN -a STDOUT -a STDERR \ - $(_b9_get_newclient_image) bash -c " - echo \"$(_b9_npm_get_github_ssh)\" >> /home/ubuntu/.ssh/id_rsa_deploy - chmod 600 /home/ubuntu/.ssh/id_rsa_deploy - npm "$@" - " - else - pushd $WORKDIR &> /dev/null - $NPM "$@" - popd &> /dev/null - fi -} - -_b9_npm_get_github_ssh() { - local FILE - for FILE in "$B9_GITHUB_SSH_FILE" ~/.ssh/id_rsa_c9robot ~/.ssh/id_rsa_deploy ~/.ssh/id_rsa; do - if [ -e "$FILE" ]; then - cat $FILE - return - fi - done - - echo "Can't find SSH key for Github" 1>&2 - exit 1 -} \ No newline at end of file diff --git a/b9/lib/check.sh b/b9/lib/check.sh deleted file mode 100644 index 7a56f037..00000000 --- a/b9/lib/check.sh +++ /dev/null @@ -1,4 +0,0 @@ -b9_check() { - echo "Running B9 tests" - _do_check_package -} \ No newline at end of file diff --git a/b9/lib/deploy.sh b/b9/lib/deploy.sh deleted file mode 100644 index 97f18695..00000000 --- a/b9/lib/deploy.sh +++ /dev/null @@ -1,241 +0,0 @@ -b9_deploy_usage() { - echo "Usage: $B9 deploy SERVICES TREEISH SERVER_PATTERN [ARG...]" - echo - echo "Deploy a Cloud9 version" - echo - echo "Options:" - echo " --strategy=[slow_start|parallel|serial] Deploy strategy to use (default: slow_start)" - echo " --regex Interpret server patter as regular expression" - echo " --no-check skip the health check" - exit 1 -} - -b9_deploy() { - [ "$1" == "--help" ] && b9_deploy_usage - - local SERVICES=$1 && shift - local TREEISH=$1 && shift - local SERVER_PATTERN=$1 && shift - - local DRY_RUN="" - local ASSET="gcs" - local USE_REGEX="" - local NO_CHECK="" - local TYPE=newclient - local STRATEGY=slow_start - - [ -z "$SERVICES" ] && b9_deploy_usage - [ -z "$TREEISH" ] && b9_deploy_usage - [ -z "$SERVER_PATTERN" ] && b9_deploy_usage - - local ARG - for ARG in "$@"; do - case $ARG in - --strategy=*) - STRATEGY="${ARG#*=}" - shift - ;; - --docker) - ASSET="docker" - shift - ;; - --no-check) - NO_CHECK="--no-check" - shift - ;; - --regex) - USE_REGEX="--regex" - shift - ;; - --dry-run) - DRY_RUN="1" - shift - ;; - --help) - b9_deploy_usage - shift - ;; - *) - b9_deploy_usage - ;; - esac - done - - [ "$SERVICES" == "docker" ] && TYPE=docker - - local SERVER_LIST - local VERSION - - local TMPFILE=$(mktemp --tmpdir=$TMPDIR) - b9_package $TREEISH --type=$TYPE | tee $TMPFILE - VERSION=$(cat $TMPFILE | tail -n1) - rm $TMPFILE - - SERVER_LIST="$(_b9_deploy_server_list $SERVER_PATTERN $USE_REGEX)" - local CMD="$B9 --settings=$MODE exec _b9_deploy_one_from_${ASSET} $NO_CHECK $VERSION $SERVICES" - if [ "$DRY_RUN" == "1" ]; then - CMD="echo $CMD" - fi - - _b9_deploy_release_event "$SERVICES" $VERSION $SERVER_PATTERN - _b9_deploy_strategy_${STRATEGY} "$SERVER_LIST" "$CMD" -} - -_b9_deploy_strategy_slow_start() { - local SERVER_LIST=$1 - local CMD=$2 - - # first one - $CMD $(echo "$SERVER_LIST" | head -n1) - - # then two - echo "$SERVER_LIST" | tail -n +2 | head -n2 | parallel --halt 1 $CMD - - # then the rest - echo "$SERVER_LIST" | tail -n +4 | parallel --halt 1 -j 15 $CMD -} - -_b9_deploy_strategy_parallel() { - local SERVER_LIST=$1 - local CMD=$2 - - # first one - $CMD $(echo "$SERVER_LIST" | head -n1) - - # then the rest - echo "$SERVER_LIST" | tail -n +2 | parallel --halt 1 -j 30 $CMD -} - -_b9_deploy_strategy_serial() { - local SERVER_LIST=$1 - local CMD=$2 - - echo "$SERVER_LIST" | xargs -n1 $CMD -} - -_b9_deploy_server_list () { - local SERVER_PATTERN=$1 - local USE_REGEX=$2 - $C9_DIR/scripts/gssh --no-cache $USE_REGEX --print-names "$SERVER_PATTERN" | shuf -} - -_b9_deploy_one_from_gcs() { - local NO_CHECK=$1 - if [ "$NO_CHECK" == "--no-check" ]; then - shift - else - NO_CHECK="" - fi - - local VERSION=$1 - local SERVICES=$2 - local SERVER=$3 - - echo Deploying $VERSION \($SERVICES\) to $SERVER ... >&2 - - _b9_deploy_upload_from_gcs $VERSION $SERVER - _b9_deploy_update_services $VERSION $SERVICES $SERVER - [ -z "$NO_CHECK" ] && _b9_deploy_check $SERVER $SERVICES - - echo Deployed $VERSION to $SERVER >&2 -} - -_b9_deploy_upload_from_gcs() { - local VERSION=$1 - local SERVER=$2 - - local TGZ - TGZ=$(_b9_package_download_gcs $VERSION) - - local VERSIONS_DIR="/home/ubuntu/versions" - local TARGET_FILE=${VERSIONS_DIR}/$(basename $TGZ) - local TARGET_DIR=${VERSIONS_DIR}/$(basename $TGZ ".tar.xz") - - - _b9_deploy_ssh $SERVER "rm -rf $TARGET_DIR $TARGET_FILE; mkdir -p /home/ubuntu/versions/history" - _b9_deploy_scp $TGZ $SERVER:$TARGET_FILE - _b9_deploy_ssh $SERVER "cd /home/ubuntu/versions && tar xf $TARGET_FILE && rm $TARGET_FILE" -} - -_b9_deploy_update_services() { - local VERSION=$1 - local SERVICES=$2 - local SERVER=$3 - - local TOTAL_VERSIONS_TO_KEEP=7 - - local VERSIONS_DIR="/home/ubuntu/versions" - local TARGET_DIR=${VERSIONS_DIR}/$VERSION - local BUILD_NAME=$(echo $VERSION | awk -F- '{printf "%s-%s-%s", $1, $2, $3}') - - _b9_deploy_ssh $SERVER " - for SERVICE in $(echo $SERVICES | sed 's/,/ /g'); do - mv /home/ubuntu/\$SERVICE /home/ubuntu/versions/history/\$SERVICE-$(date +%FT%T) &>/dev/null; - ln -s $TARGET_DIR /home/ubuntu/\$SERVICE; - done - ~/supervisord_start_script.sh || ~/supervisord_start_script.sh -f || ~/supervisord_start_script.sh -f; - cd /home/ubuntu/versions; - USED_VERSIONS=\$(ls -l /home/ubuntu | grep -F -- '-> /home/ubuntu/versions' | awk -F'-> ' '{print \$2}' | awk -F/ '{print \$(NF)}'); - ls -t 2>/dev/null | grep $BUILD_NAME | grep -v -F \"\$USED_VERSIONS\" | tail -n +$TOTAL_VERSIONS_TO_KEEP | xargs sudo rm -rf;" -} - -_b9_deploy_check() { - local SERVER=$1 - local SERVICES=$2 - - echo $SERVICES | sed 's/,/\n/g' | parallel --halt 1 -j 0 $B9 exec _b9_deploy_check_one $SERVER -} - -_b9_deploy_check_one() { - local SERVER=$1 - local SERVICE=$2 - - local HOST - local PORT - local WAIT=default - HOST=$(echo $SERVER | awk -F@ '{ print $2}') - - if [ "$SERVICE" == "oldclient" ]; then - SERVICE="c9" - elif [ "$SERVICE" == "docker" ]; then - WAIT=long - SERVICE="docker-daemon" - elif [[ $SERVICE =~ ^vfs-[0-9]$ ]]; then - PORT="--port=804$(echo $SERVICE | awk -F- '{print $2}')" - SERVICE="vfs" - else - SERVICE=${SERVICE//-/_} - fi - - if ! _b9_check_save_deploy --wait=$WAIT $PORT --server=$HOST --service=$SERVICE; then - echo "One or more safe deploy checks failed :(" >&2 - exit 1 - fi -} - -_b9_deploy_release_event() { - local SERVICES=$1 - local VERSION=$2 - local SERVER_PATTERN=$3 - - _b9_init_node_helper - echo $SERVICES | sed 's/,/\n/g' | xargs -I '{}' -n1 $NODEJS $B9_DIR/lib/js/release_event.js '{}' $MODE $VERSION $SERVER_PATTERN -} - -_b9_deploy_ssh() { - /usr/bin/ssh \ - -o LogLevel=ERROR \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -i $(find ~/.ssh/ -name "*" | grep -Pe "./(google_compute_engine|id_rsa_ansible|id_rsa)$" | head -1)\ - "$@" -} - -_b9_deploy_scp() { - /usr/bin/scp \ - -o LogLevel=ERROR \ - -o StrictHostKeyChecking=no \ - -o UserKnownHostsFile=/dev/null \ - -i $(find ~/.ssh/ -name "*" | grep -Pe "./(google_compute_engine|id_rsa_ansible|id_rsa)$" | head -1) \ - "$@" -} diff --git a/b9/lib/js/filter_node_modules.js b/b9/lib/js/filter_node_modules.js deleted file mode 100644 index b1f35718..00000000 --- a/b9/lib/js/filter_node_modules.js +++ /dev/null @@ -1,118 +0,0 @@ -#!/usr/bin/env node -"use strict"; - -var DEFAULT_MODULES = ["c9", "simple-template", "architect", "amd-loader", "heapdump", "optimist"]; -var DEFAULT_SETTINGS = "deploy"; - -var npmBuild = require("architect-build/npm_build"); -var async = require("async"); -var fs = require("fs"); -var optimist = require("optimist"); -var _ = require("lodash"); - -module.exports = nodeModules; - -if (!module.parent) { - main(process.argv.slice(2), function(err) { - if (err) { - console.error(err); - console.error("Stacktrace: ", err.stack); - process.exit(1); - } - }); -} - -function main(argv, callback) { - var options = optimist(argv) - .usage("Usage: $0 [CONFIG_NAME] [--help]") - .alias("s", "settings") - .default("settings", DEFAULT_SETTINGS) - .describe("settings", "Settings file to use") - .default("source", __dirname + "/../../..") - .describe("source", "Source directory") - .describe("targetFile", "Target package.json") - .boolean("help") - .describe("help", "Show command line options."); - - argv = options.argv; - if (argv.help) { - options.showHelp(); - return callback(); - } - - if (argv._.length != 1) { - options.showHelp(); - return callback(); - } - var config = argv._[0]; - var settings = argv.settings; - var source = argv.source; - - nodeModules(source, config, settings, function(err, json) { - if (err) return callback(err); - - if (argv.targetFile) - fs.writeFileSync(argv.targetFile, JSON.stringify(json, null, 2)); - else - console.log(JSON.stringify(json, null, 2)); - }); -} - -function calculateRequiredNodeModules(sourceDir, buildConfig, configNames, settingsName, callback) { - if (buildConfig.nodeModulesInclude === "*") { // If the user specifically asks for all don't bother calculating - return callback(); - } - - async.map(configNames.split(","), calculateModules, function (err, modules) { - if (err) return callback(err); - return callback(null, _.uniq(_.flatten(modules))); /* Flatten array and remove duplicates */ - }); - - function calculateModules (configName, done) { - npmBuild({ - root: sourceDir, - args: [configName, '-s', settingsName] - }, function (err, result) { - if (err) return done(err); - - var deps = result.roots; - - var nodeModules = []; - deps.forEach(function (dep) { - if (dep.match(/node_modules/)) { - nodeModules.push(dep.replace(/^(.*\/)?node_modules\//, "")); - } - }); - nodeModules.sort(); - return done(null, nodeModules); - }); - } -} - -function nodeModules(sourceDir, configNames, settingsName, callback) { - var buildConfig = require(sourceDir + "/configs/" + configNames).buildConfig({mode: settingsName}); - var packageJson = require(sourceDir + "/package.json"); - - var nodeModules = packageJson.dependencies; - - delete packageJson.devDependencies; - delete packageJson.scripts; - - /* Calculates what modules are needed for this config, so they can be contact'd to nodeModulesInclude */ - calculateRequiredNodeModules(sourceDir, buildConfig, configNames, settingsName, function(err, modules) { - if (err) return callback(err); - - var allModules = buildConfig.nodeModulesInclude.concat(modules).concat(DEFAULT_MODULES); - - packageJson.dependencies = allModules.reduce(function(deps, name) { - if (nodeModules[name]) - deps[name] = nodeModules[name]; - else - deps[name] = "*"; - - return deps; - }, {}); - - callback(null, packageJson); - }); -} \ No newline at end of file diff --git a/b9/lib/js/filter_node_modules_test.js b/b9/lib/js/filter_node_modules_test.js deleted file mode 100644 index 33e99589..00000000 --- a/b9/lib/js/filter_node_modules_test.js +++ /dev/null @@ -1,23 +0,0 @@ -#!/usr/bin/env node - -/*global describe it before after beforeEach afterEach */ -"use strict"; -"use server"; - -require("c9/inline-mocha")(module, null, { globals: ["define"]}); - -var assert = require("assert-diff"); -var nodeModules = require("./filter_node_modules"); - -describe(__filename, function() { - - it("should filter node modules for docker", function(done) { - nodeModules(__dirname + "/../../..", "docker", "deploy", function(err, config) { - assert(!err, err); - - assert(config.dependencies["optimist"]); - assert(!config.dependencies["nodemailer-smtp-transport"]); - done(); - }); - }); -}); \ No newline at end of file diff --git a/b9/lib/js/generate_settings.js b/b9/lib/js/generate_settings.js deleted file mode 100644 index 688a8a23..00000000 --- a/b9/lib/js/generate_settings.js +++ /dev/null @@ -1,100 +0,0 @@ -#!/usr/bin/env node -"use strict"; - -var DEFAULT_SETTINGS = "deploy"; -var ALWAYS_INCLUDE_SETTINGS = ["node", "mode", "manifest", "domains", "primaryDomain", "primaryBaseUrl", "baseUrlPattern"]; - -var fs = require("fs"); -var optimist = require("optimist"); -var loadManifest = require("c9/manifest").load; -var reJSON = require("c9/json-with-re"); - -module.exports = generateSettings; - -if (!module.parent) { - main(process.argv.slice(2), function(err) { - if (err) { - console.error(err); - console.error("Stacktrace: ", err.stack); - process.exit(1); - } - }); -} - -function main(argv, callback) { - var options = optimist(argv) - .usage("Usage: $0 [CONFIG_NAME] [--help]") - .alias("s", "settings") - .default("settings", DEFAULT_SETTINGS) - .describe("settings", "Settings file to use") - .default("source", __dirname + "/../../..") - .describe("source", "Source directory") - .describe("targetFile", "Target package.json") - .boolean("help") - .describe("help", "Show command line options."); - - argv = options.argv; - if (argv.help) { - options.showHelp(); - return callback(); - } - - if (argv._.length != 1) { - options.showHelp(); - return callback(); - } - var config = argv._[0]; - var settingsName = argv.settings; - var source = argv.source; - - generateSettings(source, config, settingsName, function(err, contents) { - if (err) return callback(err); - - if (argv.targetFile) - fs.writeFileSync(argv.targetFile, contents); - else - console.log(contents); - }); -} - -function generateSettings(source, config, settingsName, callback) { - // Check if build already exists. - var manifest = loadManifest(source); - manifest.hostname = "[%type%]-[%provider%]-[%region%]-[%index%]-[%env%]"; - - var oldSettings; - try { - oldSettings = require(source + "/settings/" + settingsName)(manifest); - } catch (e) { - return callback(e); - } - - var buildConfig = require(source + "/configs/" + config).buildConfig({mode: settingsName}); - - if (buildConfig.settingsInclude == "*") { - newSettings = oldSettings; - } - else { - buildConfig.settingsInclude = buildConfig.settingsInclude.concat(ALWAYS_INCLUDE_SETTINGS); - var newSettings = - buildConfig.settingsInclude.reduce(function(settings, name) { - settings[name] = oldSettings[name]; - return settings; - }, {}); - } - - newSettings.node = oldSettings.node; - - var contents = - "var hostname = require('c9/hostname');\n" + - "var reJSON = require('c9/json-with-re');\n" + - "var fill = require('simple-template').fill;\n" + - "module.exports = function() {\n" + - " var options = hostname.parse(hostname.get());\n" + - " options.root = __dirname + '/..';\n" + - " var template = " + reJSON.stringify(newSettings, 2).replace(new RegExp(source, "g"), "[%root%]") + ";\n" + - " return reJSON.parse(fill(JSON.stringify(template), options));\n" + - "};"; - - callback(null, contents); -} \ No newline at end of file diff --git a/b9/lib/js/generate_settings_test.js b/b9/lib/js/generate_settings_test.js deleted file mode 100644 index ef50f77e..00000000 --- a/b9/lib/js/generate_settings_test.js +++ /dev/null @@ -1,52 +0,0 @@ -#!/usr/bin/env node - -/*global describe it before after beforeEach afterEach */ -"use strict"; -"use server"; - -require("c9/inline-mocha")(module, null, { globals: ["define"]}); - -var assert = require("assert-diff"); -var vm = require("vm"); -var generateSettings = require("./generate_settings"); - -describe(__filename, function() { - - it("should filter settings file", function(done) { - generateSettings(__dirname + "/../../..", "docker", "deploy", function(err, settings) { - assert(!err, err); - - settings = eval(settings)(); - - assert(settings.docker); - assert(settings["docker-daemon"]); - assert(settings.aws); - assert(settings.sapi); - assert(settings.rabbitmq); - - assert(!settings.c9); - assert(!settings.auth); - assert(!settings.worker); - assert(!settings.captcha); - assert(!settings.sendgrid); - assert(!settings.redis); - assert(!settings["redis-slave"]); - assert(!settings.sessionredis); - assert(!settings["sessionredis-slave"]); - assert(!settings.github); - assert(!settings.bitbucket); - assert(!settings.salesforce); - assert(!settings.google); - assert(!settings.c9_auth); - assert(!settings.services); - assert(!settings.mailer); - assert(!settings.zuora); - assert(!settings.pricing); - assert(!settings.catalog); - assert(!settings.minfraud); - assert(!settings.support); - - done(); - }); - }); -}); \ No newline at end of file diff --git a/b9/lib/js/list_plugins.js b/b9/lib/js/list_plugins.js deleted file mode 100644 index a533010e..00000000 --- a/b9/lib/js/list_plugins.js +++ /dev/null @@ -1,69 +0,0 @@ -#!/usr/bin/env node -"use strict"; - -var DEFAULT_SETTINGS = "deploy"; - -var optimist = require("optimist"); -var loadManifest = require("c9/manifest").load; - -module.exports = listPlugins; - -if (!module.parent) { - main(process.argv.slice(2), function(err) { - if (err) { - console.error(err); - console.error("Stacktrace: ", err.stack); - process.exit(1); - } - }); -} - -function main(argv, callback) { - var options = optimist(argv) - .usage("Usage: $0 [CONFIG_NAME] [--help]") - .alias("s", "settings") - .default("settings", DEFAULT_SETTINGS) - .describe("settings", "Settings file to use") - .default("source", __dirname + "/../../..") - .describe("source", "Source directory") - .boolean("help") - .describe("help", "Show command line options."); - - argv = options.argv; - if (argv.help) { - options.showHelp(); - return callback(); - } - - if (argv._.length != 1) { - options.showHelp(); - return callback(); - } - var config = argv._[0]; - var settingsName = argv.settings; - var source = argv.source; - - listPlugins(source, config, settingsName).forEach(function(line) { - console.log(line); - }); -} - -function listPlugins(source, configName, settingsName) { - var manifest = loadManifest(source); - manifest.hostname = "[%type%]-[%provider%]-[%region%]-[%index%]-[%env%]"; - - var settings = require(source + "/settings/" + settingsName)(manifest); - var config = require(source + "/configs/" + configName)(settings, optimist([])); - - var plugins = Object.keys(config.reduce(function(processedPlugins, plugin) { - var packagePath = plugin.packagePath || plugin; - if (packagePath.indexOf("./") === 0) { - var pluginDir = packagePath.slice(2, packagePath.indexOf("/", 2)); - processedPlugins[pluginDir] = true; - } - - return processedPlugins; - }, {})); - - return plugins; -} \ No newline at end of file diff --git a/b9/lib/js/list_plugins_test.js b/b9/lib/js/list_plugins_test.js deleted file mode 100644 index 13aa4ad4..00000000 --- a/b9/lib/js/list_plugins_test.js +++ /dev/null @@ -1,22 +0,0 @@ -#!/usr/bin/env node - -/*global describe it before after beforeEach afterEach */ -"use strict"; -"use server"; - -require("c9/inline-mocha")(module, null, { globals: ["define"]}); - -var assert = require("assert-diff"); -var listPlugins = require("./list_plugins"); - -describe(__filename, function() { - - it("should filter node modules for docker", function() { - var list = listPlugins(__dirname + "/../../..", "docker", "deploy"); - - assert(list.indexOf("c9.docker") >= 0); - assert(list.indexOf("c9.mq") >= 0); - - assert(list.indexOf("c9.db.redis") == -1); - }); -}); \ No newline at end of file diff --git a/b9/lib/js/package.json b/b9/lib/js/package.json deleted file mode 100644 index d3822a45..00000000 --- a/b9/lib/js/package.json +++ /dev/null @@ -1,10 +0,0 @@ -{ - "name": "b9-utils", - "private": true, - "dependencies": { - "async": "^1.5.2", - "lodash": "^4.2.1", - "mkdirp": "^0.5.1", - "optimist": "^0.6.1" - } -} diff --git a/b9/lib/js/release_event.js b/b9/lib/js/release_event.js deleted file mode 100644 index 4b0b4faf..00000000 --- a/b9/lib/js/release_event.js +++ /dev/null @@ -1,61 +0,0 @@ -"use strict"; - -var https = require("https"); - -var DATADOG_API_KEY = '64e56d39dfdd7f2bbf06f09100d51a18'; -var DATADOG_API_URL = 'https://app.datadoghq.com'; - -module.exports = releaseEvent; - -if (!module.parent) { - var argv = process.argv; - releaseEvent(argv[2], argv[3], argv[4], argv[5], function(err) { - if (err) { - console.error("Error posting release event to datadog" + err.message); - process.exit(1); - } - process.exit(0); - }); -} - -function datadogEvent(msg, callback) { - - var payload = JSON.stringify(msg); - - var req = https.request({ - hostname: "app.datadoghq.com", - port: 443, - path: "/api/v1/events?api_key=" + encodeURIComponent(DATADOG_API_KEY), - method: "POST", - headers: { - "Accept": "application/json", - "Content-Type": "application/json", - "Content-Length": payload.length - } - }, function(res) { - if (res.statusCode >= 400) - return callback(new Error("request failed with status code " + res.statusCode)); - - callback(); - }); - - req.on("error", function(e) { - callback(e); - }); - - req.write(payload); - req.end(); -} - -function releaseEvent(application, mode, version, pattern, callback) { - datadogEvent({ - title: 'Release: ' + application + ' version ' + version + ' to "' + pattern + '"', - tags: [ - 'release', - 'application:' + application, - 'mode:' + mode, - 'version:' + version, - 'pattern:' + pattern - ] - }, callback); -} \ No newline at end of file diff --git a/b9/lib/package.sh b/b9/lib/package.sh deleted file mode 100644 index 869afa33..00000000 --- a/b9/lib/package.sh +++ /dev/null @@ -1,199 +0,0 @@ -readonly B9_PACKAGE_GIT_CACHE=$C9_DIR - -b9_package_usage() { - echo "Usage: $B9 package TREEISH [ARG...]" - echo - echo "Package and upload a version of Cloud 9" - echo - echo "Options:" - echo " --type=[newclient|docker] (default: newclient)" - echo " --no-cache" - exit 1 -} - -b9_package() { - [ "$1" == "--help" ] && b9_package_usage - - local TREEISH=$1 - local TYPE=newclient - local SETTINGS=$MODE - local STORAGE=gcs - local USE_CACHE=1 - - [ -z "$TREEISH" ] && b9_package_usage - [[ $TREEISH =~ -- ]] && b9_package_usage - shift - - local ARG - for ARG in "$@"; do - case $ARG in - --type=*) - TYPE="${ARG#*=}" - shift - ;; - --docker) - STORAGE=docker - shift - ;; - --no-cache) - USE_CACHE=0 - shift - ;; - *) - b9_package_usage - ;; - esac - done - - local VERSION - local WORKDIR - - [ "$TYPE" == "newclient" ] && SETTINGS=all - - _b9_package_init_git_cache - - VERSION=$(_b9_get_version $TREEISH $TYPE $SETTINGS) - - if [ "$USE_CACHE" == "1" ] && _b9_package_is_cached $STORAGE $VERSION; then - echo $VERSION - return - fi - - WORKDIR=$(_b9_package_init_work_dir $VERSION) - _b9_package_sync_workdir $TYPE $WORKDIR $VERSION $SETTINGS - _b9_package_npm_install $WORKDIR - _b9_package_cleanup_workdir $WORKDIR - _b9_package_upload $STORAGE $WORKDIR $VERSION - - echo $VERSION -} - -_b9_package_init_git_cache() { - pushd $B9_PACKAGE_GIT_CACHE &> /dev/null - - if [ ! -d .git ]; then - git clone git@github.com:c9/newclient.git . - fi - - git fetch origin - - popd &> /dev/null -} - -_b9_package_init_work_dir() { - local VERSION=$1 - mktemp -d b9-package-${VERSION}-XXXXXXXXXXXXX --tmpdir=$TMP -} - -_b9_get_version() { - local TREEISH=$1 - local TYPE=${2:-newclient} - local SETTINGS=${3:-all} - - pushd $B9_PACKAGE_GIT_CACHE &> /dev/null - echo c9-${TYPE}-${SETTINGS}-$(git show $TREEISH:package.json | jq -r .version)-$(git rev-parse --short=8 $TREEISH) - popd &> /dev/null -} - -_b9_package_is_cached() { - local STORAGE=$1 - local VERSION=$2 - - if [ -d $TMP/$VERSION ]; then - return - fi - - case $STORAGE in - gcs) - _b9_package_is_cached_gcs $VERSION - ;; - docker) - _b9_package_is_cached_docker $VERSION - ;; - *) - echo "Invalid storage type: $STORAGE" 1>&2 - exit 1 - ;; - esac -} - -_b9_package_upload() { - local STORAGE=$1 - local WORKDIR=$2 - local VERSION=$3 - - case $STORAGE in - gcs) - _b9_package_upload_gcs $WORKDIR $VERSION - ;; - docker) - _b9_package_upload_docker $WORKDIR $VERSION - ;; - *) - exit 1 - ;; - esac - - mv $WORKDIR $TMP/$VERSION -} - -_b9_package_sync_workdir() { - local TYPE=$1 - local WORKDIR=$2 - local VERSION=$3 - local SETTINGS=$4 - - case $TYPE in - newclient) - _b9_package_sync_workdir_newclient $WORKDIR $VERSION $SETTINGS - ;; - docker) - _b9_package_sync_workdir_docker $WORKDIR $VERSION $SETTINGS - ;; - *) - exit 1 - ;; - esac -} - -_b9_package_npm_install() { - local WORKDIR=$1 - - pushd $WORKDIR &> /dev/null - _b9_install_deps - popd &> /dev/null -} - -_b9_package_cleanup_workdir() { - local WORKDIR=$1 - local REVISION - [ -z "$WORKDIR" ] && return 1 - - pushd $WORKDIR &> /dev/null - - _b9_package_patch_package_json - rm -rf .git build bin local - - popd &> /dev/null -} - -_b9_package_patch_package_json() { - [ ! -d .git ] && return 0 - - REVISION=$(git rev-parse HEAD) - mv package.json _package.json - cat _package.json | jq ".revision=\"$REVISION\"" > package.json - rm _package.json -} - -_do_check_package() { - MODE=devel - b9_package origin/master --type=newclient --no-cache - b9_package origin/master --type=newclient - - MODE=deploy - b9_package origin/master --type=docker --no-cache - - MODE=devel - b9_package origin/master --docker --no-cache -} diff --git a/b9/lib/package_service_docker.sh b/b9/lib/package_service_docker.sh deleted file mode 100644 index ea4d6efe..00000000 --- a/b9/lib/package_service_docker.sh +++ /dev/null @@ -1,116 +0,0 @@ -_b9_package_sync_workdir_docker() { - local WORKDIR=$1 - local VERSION=$2 - local SETTINGS=$3 - local SOURCE=$WORKDIR/source - - pushd $WORKDIR &> /dev/null - - _b9_package_docker_init_source $WORKDIR $SOURCE $VERSION $SETTINGS - _b9_package_docker_init_workdir - - _b9_init_node_helper - _b9_package_docker_node_modules $WORKDIR $SOURCE $SETTINGS - _b9_package_docker_generate_settings $WORKDIR $SOURCE $SETTINGS - _b9_package_docker_include_files $WORKDIR $SOURCE $SETTINGS - _b9_package_docker_copy_plugins $WORKDIR $SOURCE $SETTINGS - - rm -rf $SOURCE - - popd &> /dev/null -} - -_b9_package_docker_init_source() { - local WORKDIR=$1 - local SOURCE=$2 - local VERSION=$3 - local SETTINGS=$4 - - rm -rf $WORKDIR - mkdir -p $SOURCE - - _b9_package_sync_workdir_newclient $SOURCE $VERSION $SETTINGS - _b9_package_npm_install $SOURCE -} - -_b9_package_docker_init_workdir() { - mkdir -p plugins - mkdir -p node_modules - mkdir -p settings - mkdir -p configs -} - -_b9_package_docker_node_modules() { - local WORKDIR=$1 - local SOURCE=$2 - local SETTINGS=$3 - local NODE_MODULES - local MODULE - - NODE_PATH=$B9_DIR/lib/js/node_modules:$C9_DIR/node_modules:$NODE_PATH $NODEJS $B9_DIR/lib/js/filter_node_modules.js docker --targetFile=$WORKDIR/package.json --source=$SOURCE --settings=$SETTINGS - - NODE_MODULES=$(cat $WORKDIR/package.json | jq -r '.dependencies | keys | @sh') - - mkdir -p $WORKDIR/node_modules - for MODULE in $NODE_MODULES; do - MODULE=${MODULE:1:-1} - if [ -d $SOURCE/node_modules/$MODULE ]; then - cp -a $SOURCE/node_modules/$MODULE $WORKDIR/node_modules - fi - done - - pushd $WORKDIR &> /dev/null - _b9_npm "$WORKDIR" install - - popd &> /dev/null -} - -_b9_package_docker_generate_settings() { - local WORKDIR=$1 - local SOURCE=$2 - local SETTINGS=$3 - - $NODEJS $B9_DIR/lib/js/generate_settings.js docker --targetFile=$WORKDIR/settings/$SETTINGS.js --source=$SOURCE --settings=$SETTINGS -} - -_b9_package_docker_include_files() { - local WORKDIR=$1 - local SOURCE=$2 - local SETTINGS=$3 - - local BUILD_CONFIG - local FILE_MODULES_INCLUDE - local PATTERN - - pushd $WORKDIR &> /dev/null - - BUILD_CONFIG=$($NODEJS -e "console.log(JSON.stringify(require('$SOURCE/configs/docker').buildConfig({mode: '$SETTINGS'})))") - FILE_INCLUDE=$(echo $BUILD_CONFIG | jq -r '.fileInclude | @sh') - - for PATTERN in $FILE_INCLUDE; do - PATTERN=${PATTERN:1:-1} - mkdir -p $(dirname $PATTERN) - cp -a -R $SOURCE/$PATTERN $(dirname $PATTERN) - done - - for PATTERN in "server.js" "scripts/tail-log.sh" "configs/docker.js"; do - mkdir -p $(dirname $PATTERN) - cp -a -R $SOURCE/$PATTERN $(dirname $PATTERN) || : - done - - popd &> /dev/null -} - -_b9_package_docker_copy_plugins() { - local WORKDIR=$1 - local SOURCE=$2 - local SETTINGS=$3 - - local PLUGINS - local PLUGIN - - PLUGINS=$($NODEJS $B9_DIR/lib/js/list_plugins.js docker --source=$SOURCE --settings=$SETTINGS) - for PLUGIN in $PLUGINS; do - cp -a $SOURCE/plugins/$PLUGIN $WORKDIR/plugins - done -} diff --git a/b9/lib/package_service_newclient.sh b/b9/lib/package_service_newclient.sh deleted file mode 100644 index 82c56dbb..00000000 --- a/b9/lib/package_service_newclient.sh +++ /dev/null @@ -1,17 +0,0 @@ -_b9_package_sync_workdir_newclient() { - local WORKDIR=$1 - local VERSION=$2 - local HASH - - HASH=$(echo $VERSION | awk -F- '{print $5}') - - rm -rf $WORKDIR - mkdir -p $WORKDIR - - pushd $WORKDIR &> /dev/null - rsync -qrtv --delete $B9_PACKAGE_GIT_CACHE/.git $WORKDIR/ - git reset --hard - git checkout $HASH - - popd &> /dev/null -} \ No newline at end of file diff --git a/b9/lib/package_storage_docker.sh b/b9/lib/package_storage_docker.sh deleted file mode 100644 index 3e12a3b6..00000000 --- a/b9/lib/package_storage_docker.sh +++ /dev/null @@ -1,74 +0,0 @@ -readonly B9_DOCKER_REGISTRY=gcr.io/c9.io/cloud9gce -readonly B9_DOCKER_BUCKET=gs://artifacts.cloud9gce.c9.io.a.appspot.com - -_b9_dockerize_update_base() { - local TREEISH=origin/master - - local CID - local VERSION - - # build package - local TMPFILE=$(mktemp --tmpdir=$TMPDIR) - b9_package $TREEISH --type=newclient | tee $TMPFILE - VERSION=$(cat $TMPFILE | tail -n1) - rm $TMPFILE - - # build base image - docker build -t $B9_DOCKER_REGISTRY/c9:base $B9_DIR/containers/c9 - - CID=$(docker run -d $B9_DOCKER_REGISTRY/c9:base sleep 1h) - - # copy package to base - docker exec $CID bash -c " - cd /home/ubuntu && - tar xf $TMP/$VERSION.tar.xz - rm -rf $VERSION.tgz newclient - mv $VERSION newclient" - - # commit image - docker stop $CID - docker commit $CID $B9_DOCKER_REGISTRY/c9:base - - # push - gcloud docker push $B9_DOCKER_REGISTRY/c9:base -} - -_b9_package_is_cached_docker() { - local VERSION=$1 - local TAG - - TAG=$(echo $VERSION | awk -F- '{printf "%s-%s", $4, $5}') - _b9_dockerize_has_tag c9 $TAG -} - -_b9_package_upload_docker() { - local WORKDIR=$1 - local VERSION=$2 - - local CID - local TAG - - gcloud docker pull $B9_DOCKER_REGISTRY/c9:base - - CID=$(docker run -d -v $WORKDIR:/home/ubuntu/$(basename $WORKDIR):ro $B9_DOCKER_REGISTRY/c9:base sleep 1h) - - # copy package - docker exec $CID bash -c " - cd /home/ubuntu && - rsync -qrt --delete --checksum /home/ubuntu/$(basename $WORKDIR)/* newclient" - - # commit image - TAG=$(echo $VERSION | awk -F- '{printf "%s-%s", $4, $5}') - docker stop $CID - docker commit $CID $B9_DOCKER_REGISTRY/c9:$TAG - - # push - gcloud docker push $B9_DOCKER_REGISTRY/c9:$TAG -} - -_b9_dockerize_has_tag() { - local REPO=$1 - local TAG=$2 - - gsutil ls $B9_DOCKER_BUCKET/containers/repositories/library/${REPO}/tag_${TAG} -} diff --git a/b9/lib/package_storage_gcs.sh b/b9/lib/package_storage_gcs.sh deleted file mode 100644 index 3969d74c..00000000 --- a/b9/lib/package_storage_gcs.sh +++ /dev/null @@ -1,34 +0,0 @@ -_b9_package_is_cached_gcs() { - local VERSION=$1 - gsutil ls gs://cloud9_ci_cache/$(basename $VERSION).tar.xz &> /dev/null -} - -_b9_package_upload_gcs() { - local WORKDIR=$1 - local VERSION=$2 - - local TMP_TAR - local CACHE_FILE - - pushd $WORKDIR/.. &> /dev/null - - TMP_TAR=$(mktemp b9-package-XXXXXXXXXXXXX --tmpdir=$TMP --suffix=.tar.xz) - tar --transform="s/^$(basename $WORKDIR)/$VERSION/" -cJf $TMP_TAR $(basename $WORKDIR) - gsutil cp $TMP_TAR gs://cloud9_ci_cache/$VERSION.tar.xz - mv $TMP_TAR $TMP/$VERSION.tar.xz - - popd &> /dev/null -} - -_b9_package_download_gcs() { - local VERSION=$1 - local CACHE_FILE=$TMP/${VERSION}.tar.xz - - if [ -f "$CACHE_FILE" ]; then - echo $CACHE_FILE - return - fi - - gsutil cp gs://cloud9_ci_cache/$(basename $CACHE_FILE) $TMP - echo $CACHE_FILE -} \ No newline at end of file diff --git a/b9/lib/prepare.sh b/b9/lib/prepare.sh deleted file mode 100644 index 25444fba..00000000 --- a/b9/lib/prepare.sh +++ /dev/null @@ -1,82 +0,0 @@ -readonly NPMCACHE=$TMP - -b9_prepare_usage() { - echo "Usage: $B9 prepare [OPTIONS]" - echo - echo "Prepare checkout for testing" - echo - echo "Options:" - echo " --help show this help message" - exit 1 -} - -b9_prepare() { - for ARG in "$@"; do - case $ARG in - --help|-h) - usage - ;; - *) - usage - ;; - esac - done - - pushd $C9_DIR &> /dev/null - - # npm - rm -rf node_modules - git checkout -- node_modules - _b9_install_deps - git checkout -- node_modules - - popd &> /dev/null -} - -_b9_install_deps() { - if [ -f plugins/c9.profile/npm-shrinkwrap.json ]; then - _b9_setup_node_modules npm-shrinkwrap.json - fi - - if [ -f plugins/c9.profile/npm-shrinkwrap.json ]; then - _b9_setup_node_modules plugins/c9.profile/npm-shrinkwrap.json - fi -} - -_b9_setup_node_modules() { - local PACKAGE_FILE=$1 - local PACKAGE_PATH=$(dirname $PACKAGE_FILE) - local PACKAGE=$(cat $PACKAGE_FILE | jq 'del(.version)') - local GIT_HASH=$(git log --pretty=oneline -1 -- $PACKAGE_PATH/node_modules | awk '{ print $1 }') - local PACKAGE_MD5=$(echo "$PACKAGE -- $GIT_HASH" | md5sum | awk '{print $1}') - local CACHE_FILE="npm-${PACKAGE_MD5}.tar.xz" - - if [ -e "$TMP/$CACHE_FILE" ] || gsutil cp gs://cloud9_ci_cache/$CACHE_FILE $TMP &> /dev/null; then - rm -rf $PACKAGE_PATH/node_modules - tar -xkf $TMP/$CACHE_FILE || ( - rm $CACHE_FILE &>/dev/null - _b9_compile_node_modules "$CACHE_FILE" "$PACKAGE_PATH" - ) - else - _b9_compile_node_modules "$CACHE_FILE" "$PACKAGE_PATH" - fi -} - -_b9_compile_node_modules() { - local CACHE_FILE=$1 - local PACKAGE_PATH=$2 - local NPM_CMD - local TMP_TAR - - if ! _b9_npm "$(pwd)/$PACKAGE_PATH" install; then - rm -rf node_modules - git checkout node_modules - _b9_npm "$(pwd)/$PACKAGE_PATH" install - fi - - TMP_TAR=$(mktemp -d b9-npm-XXXXXXXXXXXXX --tmpdir=$TMP)/$CACHE_FILE - tar -cJf $TMP_TAR $PACKAGE_PATH/node_modules - - gsutil cp $TMP_TAR gs://cloud9_ci_cache - mv $TMP_TAR $TMP/$CACHE_FILE -}