From d27a2d1b6f4c82a475200518134b6ddea7ce4ab2 Mon Sep 17 00:00:00 2001 From: thelamer Date: Thu, 1 Nov 2018 23:52:56 -0700 Subject: [PATCH 1/2] adding updated build logic for package tagging to repo --- Jenkinsfile | 289 +++++++++++++++++++++++++++++++++++----------------- 1 file changed, 194 insertions(+), 95 deletions(-) diff --git a/Jenkinsfile b/Jenkinsfile index 521fb16..93e74da 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -2,6 +2,10 @@ pipeline { agent { label 'X86-64-MULTI' } + // Input to determine if this is a package check + parameters { + string(defaultValue: 'false', description: 'package check run', name: 'PACKAGE_CHECK') + } // Configuraiton for the variables used for this specific repo environment { EXT_GIT_BRANCH = 'master' @@ -17,8 +21,6 @@ pipeline { BUILDS_DISCORD = credentials('build_webhook_url') GITHUB_TOKEN = credentials('498b4638-2d02-4ce5-832d-8a57d01d97ab') DIST_IMAGE = 'alpine' - DIST_TAG = '3.8' - DIST_PACKAGES = 'none' MULTIARCH='true' CI='true' CI_WEB='true' @@ -33,6 +35,7 @@ pipeline { stage("Set ENV Variables base"){ steps{ script{ + env.EXIT_STATUS = '' env.LS_RELEASE = sh( script: '''curl -s https://api.github.com/repos/${LS_USER}/${LS_REPO}/releases/latest | jq -r '. | .tag_name' ''', returnStdout: true).trim() @@ -72,11 +75,22 @@ pipeline { /* ####################### Package Version Tagging ####################### */ - // If this does not track package tags - stage("Set Package tag None"){ + // Grab the current package versions in Git to determine package tag + stage("Set Package tag"){ steps{ script{ - env.PACKAGE_TAG = 'none' + env.PACKAGE_TAG = sh( + script: '''#!/bin/bash + http_code=$(curl --write-out %{http_code} -s -o /dev/null \ + https://raw.githubusercontent.com/${LS_USER}/${LS_REPO}/master/package_versions.txt) + if [[ "${http_code}" -ne 200 ]] ; then + echo none + else + curl -s \ + https://raw.githubusercontent.com/${LS_USER}/${LS_REPO}/master/package_versions.txt \ + | md5sum | cut -c1-8 + fi''', + returnStdout: true).trim() } } } @@ -202,92 +216,165 @@ pipeline { } steps { script{ - env.CI_URL = 'README_UPDATE' - env.RELEASE_LINK = 'README_UPDATE' - currentBuild.rawBuild.result = Result.ABORTED - throw new hudson.AbortException('ABORTED_README') + env.EXIT_STATUS = 'ABORTED' } } } /* ############### Build Container ############### */ - // Build Docker container for push to LS Repo - stage('Build-Single') { - when { - environment name: 'MULTIARCH', value: 'false' - } - steps { - sh "docker build --no-cache -t ${IMAGE}:${META_TAG} \ - --build-arg ${BUILD_VERSION_ARG}=${EXT_RELEASE} --build-arg VERSION=\"${META_TAG}\" --build-arg BUILD_DATE=${GITHUB_DATE} ." - } - } - // Build MultiArch Docker containers for push to LS Repo - stage('Build-Multi') { - when { - environment name: 'MULTIARCH', value: 'true' - } - parallel { - stage('Build X86') { - steps { - sh "docker build --no-cache -t ${IMAGE}:amd64-${META_TAG} \ - --build-arg ${BUILD_VERSION_ARG}=${EXT_RELEASE} --build-arg VERSION=\"${META_TAG}\" --build-arg BUILD_DATE=${GITHUB_DATE} ." - } - } - stage('Build ARMHF') { - agent { - label 'ARMHF' - } - steps { - withCredentials([ - [ - $class: 'UsernamePasswordMultiBinding', - credentialsId: '3f9ba4d5-100d-45b0-a3c4-633fd6061207', - usernameVariable: 'DOCKERUSER', - passwordVariable: 'DOCKERPASS' - ] - ]) { - echo 'Logging into DockerHub' - sh '''#! /bin/bash - echo $DOCKERPASS | docker login -u $DOCKERUSER --password-stdin - ''' - sh "curl https://lsio-ci.ams3.digitaloceanspaces.com/qemu-arm-static -o qemu-arm-static" - sh "chmod +x qemu-*" - sh "docker build --no-cache -f Dockerfile.armhf -t ${IMAGE}:arm32v6-${META_TAG} \ - --build-arg ${BUILD_VERSION_ARG}=${EXT_RELEASE} --build-arg VERSION=\"${META_TAG}\" --build-arg BUILD_DATE=${GITHUB_DATE} ." - sh "docker tag ${IMAGE}:arm32v6-${META_TAG} lsiodev/buildcache:arm32v6-${COMMIT_SHA}-${BUILD_NUMBER}" - sh "docker push lsiodev/buildcache:arm32v6-${COMMIT_SHA}-${BUILD_NUMBER}" - } - } - } - stage('Build ARM64') { - agent { - label 'ARM64' - } - steps { - withCredentials([ - [ - $class: 'UsernamePasswordMultiBinding', - credentialsId: '3f9ba4d5-100d-45b0-a3c4-633fd6061207', - usernameVariable: 'DOCKERUSER', - passwordVariable: 'DOCKERPASS' - ] - ]) { - echo 'Logging into DockerHub' - sh '''#! /bin/bash - echo $DOCKERPASS | docker login -u $DOCKERUSER --password-stdin - ''' - sh "curl https://lsio-ci.ams3.digitaloceanspaces.com/qemu-aarch64-static -o qemu-aarch64-static" - sh "chmod +x qemu-*" - sh "docker build --no-cache -f Dockerfile.aarch64 -t ${IMAGE}:arm64v8-${META_TAG} \ - --build-arg ${BUILD_VERSION_ARG}=${EXT_RELEASE} --build-arg VERSION=\"${META_TAG}\" --build-arg BUILD_DATE=${GITHUB_DATE} ." - sh "docker tag ${IMAGE}:arm64v8-${META_TAG} lsiodev/buildcache:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER}" - sh "docker push lsiodev/buildcache:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER}" - } - } - } - } - } + // Build Docker container for push to LS Repo + stage('Build-Single') { + when { + environment name: 'MULTIARCH', value: 'false' + environment name: 'EXIT_STATUS', value: '' + } + steps { + sh "docker build --no-cache -t ${IMAGE}:${META_TAG} \ + --build-arg ${BUILD_VERSION_ARG}=${EXT_RELEASE} --build-arg VERSION=\"${META_TAG}\" --build-arg BUILD_DATE=${GITHUB_DATE} ." + } + } + // Build MultiArch Docker containers for push to LS Repo + stage('Build-Multi') { + when { + environment name: 'MULTIARCH', value: 'true' + environment name: 'EXIT_STATUS', value: '' + } + parallel { + stage('Build X86') { + steps { + sh "docker build --no-cache -t ${IMAGE}:amd64-${META_TAG} \ + --build-arg ${BUILD_VERSION_ARG}=${EXT_RELEASE} --build-arg VERSION=\"${META_TAG}\" --build-arg BUILD_DATE=${GITHUB_DATE} ." + } + } + stage('Build ARMHF') { + agent { + label 'ARMHF' + } + steps { + withCredentials([ + [ + $class: 'UsernamePasswordMultiBinding', + credentialsId: '3f9ba4d5-100d-45b0-a3c4-633fd6061207', + usernameVariable: 'DOCKERUSER', + passwordVariable: 'DOCKERPASS' + ] + ]) { + echo 'Logging into DockerHub' + sh '''#! /bin/bash + echo $DOCKERPASS | docker login -u $DOCKERUSER --password-stdin + ''' + sh "curl https://lsio-ci.ams3.digitaloceanspaces.com/qemu-arm-static -o qemu-arm-static" + sh "chmod +x qemu-*" + sh "docker build --no-cache -f Dockerfile.armhf -t ${IMAGE}:arm32v6-${META_TAG} \ + --build-arg ${BUILD_VERSION_ARG}=${EXT_RELEASE} --build-arg VERSION=\"${META_TAG}\" --build-arg BUILD_DATE=${GITHUB_DATE} ." + sh "docker tag ${IMAGE}:arm32v6-${META_TAG} lsiodev/buildcache:arm32v6-${COMMIT_SHA}-${BUILD_NUMBER}" + sh "docker push lsiodev/buildcache:arm32v6-${COMMIT_SHA}-${BUILD_NUMBER}" + } + } + } + stage('Build ARM64') { + agent { + label 'ARM64' + } + steps { + withCredentials([ + [ + $class: 'UsernamePasswordMultiBinding', + credentialsId: '3f9ba4d5-100d-45b0-a3c4-633fd6061207', + usernameVariable: 'DOCKERUSER', + passwordVariable: 'DOCKERPASS' + ] + ]) { + echo 'Logging into DockerHub' + sh '''#! /bin/bash + echo $DOCKERPASS | docker login -u $DOCKERUSER --password-stdin + ''' + sh "curl https://lsio-ci.ams3.digitaloceanspaces.com/qemu-aarch64-static -o qemu-aarch64-static" + sh "chmod +x qemu-*" + sh "docker build --no-cache -f Dockerfile.aarch64 -t ${IMAGE}:arm64v8-${META_TAG} \ + --build-arg ${BUILD_VERSION_ARG}=${EXT_RELEASE} --build-arg VERSION=\"${META_TAG}\" --build-arg BUILD_DATE=${GITHUB_DATE} ." + sh "docker tag ${IMAGE}:arm64v8-${META_TAG} lsiodev/buildcache:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER}" + sh "docker push lsiodev/buildcache:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER}" + } + } + } + } + } + // Take the image we just built and dump package versions for comparison + stage('Update-packages') { + when { + branch "master" + environment name: 'CHANGE_ID', value: '' + environment name: 'EXIT_STATUS', value: '' + } + steps { + sh '''#! /bin/bash + TEMPDIR=$(mktemp -d) + if [ "${MULTIARCH}" == "true" ]; then + LOCAL_CONTAINER=${IMAGE}:amd64-${META_TAG} + else + LOCAL_CONTAINER=${IMAGE}:${META_TAG} + fi + if [ "${DIST_IMAGE}" == "alpine" ]; then + docker run --rm -v ${TEMPDIR}:/tmp ${LOCAL_CONTAINER} sh -c '\ + apk info > packages && \ + apk info -v > versions && \ + paste -d " " packages versions > /tmp/package_versions.txt' + elif [ "${DIST_IMAGE}" == "ubuntu" ]; then + docker run --rm -v ${TEMPDIR}:/tmp ${LOCAL_CONTAINER} sh -c '\ + apt -qq list --installed | awk "{print \$1,\$2}" > /tmp/package_versions.txt' + fi + if [ "$(md5sum ${TEMPDIR}/package_versions.txt | cut -c1-8 )" != "${PACKAGE_TAG}" ]; then + git clone https://github.com/${LS_USER}/${LS_REPO}.git ${TEMPDIR}/${LS_REPO} + cp ${TEMPDIR}/package_versions.txt ${TEMPDIR}/${LS_REPO}/ + cd ${TEMPDIR}/${LS_REPO}/ + git --git-dir ${TEMPDIR}/${LS_REPO}/.git add package_versions.txt + git --git-dir ${TEMPDIR}/${LS_REPO}/.git commit -m 'Bot Updating Package Versions' + git --git-dir ${TEMPDIR}/${LS_REPO}/.git push https://LinuxServer-CI:${GITHUB_TOKEN}@github.com/${LS_USER}/${LS_REPO}.git --all + echo "true" > /tmp/packages-${COMMIT_SHA}-${BUILD_NUMBER} + else + echo "false" > /tmp/packages-${COMMIT_SHA}-${BUILD_NUMBER} + fi + rm -Rf ${TEMPDIR}''' + script{ + env.PACKAGE_UPDATED = sh( + script: '''cat /tmp/packages-${COMMIT_SHA}-${BUILD_NUMBER}''', + returnStdout: true).trim() + } + } + } + // Exit the build if the package file was just updated + stage('PACKAGE-exit') { + when { + branch "master" + environment name: 'CHANGE_ID', value: '' + environment name: 'PACKAGE_UPDATED', value: 'true' + environment name: 'EXIT_STATUS', value: '' + } + steps { + script{ + env.EXIT_STATUS = 'ABORTED' + } + } + } + // Exit the build if this is just a package check and there are no changes to push + stage('PACKAGECHECK-exit') { + when { + branch "master" + environment name: 'CHANGE_ID', value: '' + environment name: 'PACKAGE_UPDATED', value: 'false' + environment name: 'EXIT_STATUS', value: '' + expression { + params.PACKAGE_CHECK == 'true' + } + } + steps { + script{ + env.EXIT_STATUS = 'ABORTED' + } + } + } /* ####### Testing ####### */ @@ -295,6 +382,7 @@ pipeline { stage('Test') { when { environment name: 'CI', value: 'true' + environment name: 'EXIT_STATUS', value: '' } steps { withCredentials([ @@ -320,7 +408,7 @@ pipeline { -e BASE=\"${DIST_IMAGE}\" \ -e SECRET_KEY=\"${DO_SECRET}\" \ -e ACCESS_KEY=\"${DO_KEY}\" \ - -e DOCKER_ENV=\"DB_HOST=${TEST_MYSQL_HOST}|DB_DATABASE=bookstack|DB_USERNAME=root|DB_PASSWORD=${TEST_MYSQL_PASSWORD}\" \ + -e DOCKER_ENV=\"${CI_DOCKERENV}\" \ -e WEB_SCREENSHOT=\"${CI_WEB}\" \ -e WEB_AUTH=\"${CI_AUTH}\" \ -e WEB_PATH=\"${CI_WEBPATH}\" \ @@ -341,6 +429,7 @@ pipeline { stage('Docker-Push-Single') { when { environment name: 'MULTIARCH', value: 'false' + environment name: 'EXIT_STATUS', value: '' } steps { withCredentials([ @@ -365,6 +454,7 @@ pipeline { stage('Docker-Push-Multi') { when { environment name: 'MULTIARCH', value: 'true' + environment name: 'EXIT_STATUS', value: '' } steps { withCredentials([ @@ -415,6 +505,7 @@ pipeline { env.LS_RELEASE != env.EXT_RELEASE + '-pkg-' + env.PACKAGE_TAG + '-ls' + env.LS_TAG_NUMBER } environment name: 'CHANGE_ID', value: '' + environment name: 'EXIT_STATUS', value: '' } steps { echo "Pushing New tag for current commit ${EXT_RELEASE}-pkg-${PACKAGE_TAG}-ls${LS_TAG_NUMBER}" @@ -440,6 +531,7 @@ pipeline { stage('Sync-README') { when { environment name: 'CHANGE_ID', value: '' + environment name: 'EXIT_STATUS', value: '' } steps { withCredentials([ @@ -467,15 +559,22 @@ pipeline { Send status to Discord ###################### */ post { - success { - sh ''' curl -X POST --data '{"avatar_url": "https://wiki.jenkins-ci.org/download/attachments/2916393/headshot.png","embeds": [{"color": 1681177,\ - "description": "**Build:** '${BUILD_NUMBER}'\\n**CI Results:** '${CI_URL}'\\n**Status:** Success\\n**Job:** '${RUN_DISPLAY_URL}'\\n**Change:** '${CODE_URL}'\\n**External Release:**: '${RELEASE_LINK}'\\n**DockerHub:** '${DOCKERHUB_LINK}'\\n"}],\ - "username": "Jenkins"}' ${BUILDS_DISCORD} ''' - } - failure { - sh ''' curl -X POST --data '{"avatar_url": "https://wiki.jenkins-ci.org/download/attachments/2916393/headshot.png","embeds": [{"color": 16711680,\ - "description": "**Build:** '${BUILD_NUMBER}'\\n**CI Results:** '${CI_URL}'\\n**Status:** failure\\n**Job:** '${RUN_DISPLAY_URL}'\\n**Change:** '${CODE_URL}'\\n**External Release:**: '${RELEASE_LINK}'\\n**DockerHub:** '${DOCKERHUB_LINK}'\\n"}],\ - "username": "Jenkins"}' ${BUILDS_DISCORD} ''' + always { + script{ + if (env.EXIT_STATUS == "ABORTED"){ + sh 'echo "build aborted"' + } + else if (currentBuild.currentResult == "SUCCESS"){ + sh ''' curl -X POST --data '{"avatar_url": "https://wiki.jenkins-ci.org/download/attachments/2916393/headshot.png","embeds": [{"color": 1681177,\ + "description": "**Build:** '${BUILD_NUMBER}'\\n**CI Results:** '${CI_URL}'\\n**Status:** Success\\n**Job:** '${RUN_DISPLAY_URL}'\\n**Change:** '${CODE_URL}'\\n**External Release:**: '${RELEASE_LINK}'\\n**DockerHub:** '${DOCKERHUB_LINK}'\\n"}],\ + "username": "Jenkins"}' ${BUILDS_DISCORD} ''' + } + else { + sh ''' curl -X POST --data '{"avatar_url": "https://wiki.jenkins-ci.org/download/attachments/2916393/headshot.png","embeds": [{"color": 16711680,\ + "description": "**Build:** '${BUILD_NUMBER}'\\n**CI Results:** '${CI_URL}'\\n**Status:** failure\\n**Job:** '${RUN_DISPLAY_URL}'\\n**Change:** '${CODE_URL}'\\n**External Release:**: '${RELEASE_LINK}'\\n**DockerHub:** '${DOCKERHUB_LINK}'\\n"}],\ + "username": "Jenkins"}' ${BUILDS_DISCORD} ''' + } + } } } } From b7cad8f1029e4d4ba21715e27b6268ac5a587af9 Mon Sep 17 00:00:00 2001 From: thelamer Date: Fri, 2 Nov 2018 00:20:36 -0700 Subject: [PATCH 2/2] forgot to add env variables to the ci run --- Jenkinsfile | 1 + 1 file changed, 1 insertion(+) diff --git a/Jenkinsfile b/Jenkinsfile index 93e74da..e4d699f 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -27,6 +27,7 @@ pipeline { CI_PORT='6767' CI_SSL='false' CI_DELAY='120' + CI_DOCKERENV='TZ=US/Pacific' CI_AUTH='user:password' CI_WEBPATH='' }