From 4c04b3eb32c36b541d82346ebdd39460ddaf39fc Mon Sep 17 00:00:00 2001 From: aptalca <541623+aptalca@users.noreply.github.com> Date: Tue, 25 Apr 2023 15:21:02 -0400 Subject: [PATCH] update version command --- .github/workflows/call_issue_pr_tracker.yml | 6 +- .github/workflows/external_trigger.yml | 10 +- .../workflows/external_trigger_scheduler.yml | 18 ++-- .github/workflows/package_trigger.yml | 4 + .../workflows/package_trigger_scheduler.yml | 8 +- Dockerfile | 6 +- Dockerfile.aarch64 | 6 +- Dockerfile.armhf | 6 +- Jenkinsfile | 93 +++++++++++++++++-- jenkins-vars.yml | 2 +- 10 files changed, 127 insertions(+), 32 deletions(-) diff --git a/.github/workflows/call_issue_pr_tracker.yml b/.github/workflows/call_issue_pr_tracker.yml index 87243e2..2c30784 100755 --- a/.github/workflows/call_issue_pr_tracker.yml +++ b/.github/workflows/call_issue_pr_tracker.yml @@ -2,9 +2,11 @@ name: Issue & PR Tracker on: issues: - types: [opened,reopened,labeled,unlabeled] + types: [opened,reopened,labeled,unlabeled,closed] pull_request_target: - types: [opened,reopened,review_requested,review_request_removed,labeled,unlabeled] + types: [opened,reopened,review_requested,review_request_removed,labeled,unlabeled,closed] + pull_request_review: + types: [submitted,edited,dismissed] jobs: manage-project: diff --git a/.github/workflows/external_trigger.yml b/.github/workflows/external_trigger.yml index 8a77861..735e3ec 100755 --- a/.github/workflows/external_trigger.yml +++ b/.github/workflows/external_trigger.yml @@ -14,11 +14,13 @@ jobs: run: | if [ -n "${{ secrets.PAUSE_EXTERNAL_TRIGGER_NEXTCLOUD_MASTER }}" ]; then echo "**** Github secret PAUSE_EXTERNAL_TRIGGER_NEXTCLOUD_MASTER is set; skipping trigger. ****" + echo "Github secret \`PAUSE_EXTERNAL_TRIGGER_NEXTCLOUD_MASTER\` is set; skipping trigger." >> $GITHUB_STEP_SUMMARY exit 0 fi echo "**** External trigger running off of master branch. To disable this trigger, set a Github secret named \"PAUSE_EXTERNAL_TRIGGER_NEXTCLOUD_MASTER\". ****" + echo "External trigger running off of master branch. To disable this trigger, set a Github secret named \`PAUSE_EXTERNAL_TRIGGER_NEXTCLOUD_MASTER\`" >> $GITHUB_STEP_SUMMARY echo "**** Retrieving external version ****" - EXT_RELEASE=$(curl -u ${{ secrets.CR_USER }}:${{ secrets.CR_PAT }} -sX GET https://api.github.com/repos/nextcloud/server/releases/latest | jq -r '. | .tag_name' | sed 's|^v||') + EXT_RELEASE=$(curl -u ${{ secrets.CR_USER }}:${{ secrets.CR_PAT }} -sX GET https://api.github.com/repos/nextcloud/server/releases | jq -r '.[] | select(.prerelease != true) | .tag_name' | sed 's|^v||g' | sort -rV | head -1) if [ -z "${EXT_RELEASE}" ] || [ "${EXT_RELEASE}" == "null" ]; then echo "**** Can't retrieve external version, exiting ****" FAILURE_REASON="Can't retrieve external version for nextcloud branch master" @@ -30,6 +32,7 @@ jobs: fi EXT_RELEASE=$(echo ${EXT_RELEASE} | sed 's/[~,%@+;:/]//g') echo "**** External version: ${EXT_RELEASE} ****" + echo "External version: ${EXT_RELEASE}" >> $GITHUB_STEP_SUMMARY echo "**** Retrieving last pushed version ****" image="linuxserver/nextcloud" tag="latest" @@ -65,14 +68,18 @@ jobs: exit 1 fi echo "**** Last pushed version: ${IMAGE_VERSION} ****" + echo "Last pushed version: ${IMAGE_VERSION}" >> $GITHUB_STEP_SUMMARY if [ "${EXT_RELEASE}" == "${IMAGE_VERSION}" ]; then echo "**** Version ${EXT_RELEASE} already pushed, exiting ****" + echo "Version ${EXT_RELEASE} already pushed, exiting" >> $GITHUB_STEP_SUMMARY exit 0 elif [ $(curl -s https://ci.linuxserver.io/job/Docker-Pipeline-Builders/job/docker-nextcloud/job/master/lastBuild/api/json | jq -r '.building') == "true" ]; then echo "**** New version ${EXT_RELEASE} found; but there already seems to be an active build on Jenkins; exiting ****" + echo "New version ${EXT_RELEASE} found; but there already seems to be an active build on Jenkins; exiting" >> $GITHUB_STEP_SUMMARY exit 0 else echo "**** New version ${EXT_RELEASE} found; old version was ${IMAGE_VERSION}. Triggering new build ****" + echo "New version ${EXT_RELEASE} found; old version was ${IMAGE_VERSION}. Triggering new build" >> $GITHUB_STEP_SUMMARY response=$(curl -iX POST \ https://ci.linuxserver.io/job/Docker-Pipeline-Builders/job/docker-nextcloud/job/master/buildWithParameters?PACKAGE_CHECK=false \ --user ${{ secrets.JENKINS_USER }}:${{ secrets.JENKINS_TOKEN }} | grep -i location | sed "s|^[L|l]ocation: \(.*\)|\1|") @@ -82,6 +89,7 @@ jobs: buildurl=$(curl -s "${response%$'\r'}api/json" | jq -r '.executable.url') buildurl="${buildurl%$'\r'}" echo "**** Jenkins job build url: ${buildurl} ****" + echo "Jenkins job build url: ${buildurl}" >> $GITHUB_STEP_SUMMARY echo "**** Attempting to change the Jenkins job description ****" curl -iX POST \ "${buildurl}submitDescription" \ diff --git a/.github/workflows/external_trigger_scheduler.yml b/.github/workflows/external_trigger_scheduler.yml index a5c41ac..885f341 100755 --- a/.github/workflows/external_trigger_scheduler.yml +++ b/.github/workflows/external_trigger_scheduler.yml @@ -2,7 +2,7 @@ name: External Trigger Scheduler on: schedule: - - cron: '00 * * * *' + - cron: '27 * * * *' workflow_dispatch: jobs: @@ -17,18 +17,18 @@ jobs: run: | echo "**** Branches found: ****" git for-each-ref --format='%(refname:short)' refs/remotes - echo "**** Pulling the yq docker image ****" - docker pull ghcr.io/linuxserver/yq for br in $(git for-each-ref --format='%(refname:short)' refs/remotes) do br=$(echo "$br" | sed 's|origin/||g') echo "**** Evaluating branch ${br} ****" - ls_branch=$(curl -sX GET https://raw.githubusercontent.com/linuxserver/docker-nextcloud/${br}/jenkins-vars.yml \ - | docker run --rm -i --entrypoint yq ghcr.io/linuxserver/yq -r .ls_branch) - if [ "$br" == "$ls_branch" ]; then - echo "**** Branch ${br} appears to be live; checking workflow. ****" + ls_jenkins_vars=$(curl -sX GET https://raw.githubusercontent.com/linuxserver/docker-nextcloud/${br}/jenkins-vars.yml) + ls_branch=$(echo "${ls_jenkins_vars}" | yq -r '.ls_branch') + ls_trigger=$(echo "${ls_jenkins_vars}" | yq -r '.external_type') + if [[ "${br}" == "${ls_branch}" ]] && [[ "${ls_trigger}" != "os" ]]; then + echo "**** Branch ${br} appears to be live and trigger is not os; checking workflow. ****" if curl -sfX GET https://raw.githubusercontent.com/linuxserver/docker-nextcloud/${br}/.github/workflows/external_trigger.yml > /dev/null 2>&1; then echo "**** Workflow exists. Triggering external trigger workflow for branch ${br} ****." + echo "Triggering external trigger workflow for branch ${br}" >> $GITHUB_STEP_SUMMARY curl -iX POST \ -H "Authorization: token ${{ secrets.CR_PAT }}" \ -H "Accept: application/vnd.github.v3+json" \ @@ -36,8 +36,10 @@ jobs: https://api.github.com/repos/linuxserver/docker-nextcloud/actions/workflows/external_trigger.yml/dispatches else echo "**** Workflow doesn't exist; skipping trigger. ****" + echo "Skipping branch ${br} due to no external trigger workflow present." >> $GITHUB_STEP_SUMMARY fi else - echo "**** ${br} appears to be a dev branch; skipping trigger. ****" + echo "**** ${br} is either a dev branch, or has no external version; skipping trigger. ****" + echo "Skipping branch ${br} due to being detected as dev branch or having no external version." >> $GITHUB_STEP_SUMMARY fi done diff --git a/.github/workflows/package_trigger.yml b/.github/workflows/package_trigger.yml index 117e483..f2b95ec 100755 --- a/.github/workflows/package_trigger.yml +++ b/.github/workflows/package_trigger.yml @@ -14,13 +14,16 @@ jobs: run: | if [ -n "${{ secrets.PAUSE_PACKAGE_TRIGGER_NEXTCLOUD_MASTER }}" ]; then echo "**** Github secret PAUSE_PACKAGE_TRIGGER_NEXTCLOUD_MASTER is set; skipping trigger. ****" + echo "Github secret \`PAUSE_PACKAGE_TRIGGER_NEXTCLOUD_MASTER\` is set; skipping trigger." >> $GITHUB_STEP_SUMMARY exit 0 fi if [ $(curl -s https://ci.linuxserver.io/job/Docker-Pipeline-Builders/job/docker-nextcloud/job/master/lastBuild/api/json | jq -r '.building') == "true" ]; then echo "**** There already seems to be an active build on Jenkins; skipping package trigger ****" + echo "There already seems to be an active build on Jenkins; skipping package trigger" >> $GITHUB_STEP_SUMMARY exit 0 fi echo "**** Package trigger running off of master branch. To disable, set a Github secret named \"PAUSE_PACKAGE_TRIGGER_NEXTCLOUD_MASTER\". ****" + echo "Package trigger running off of master branch. To disable, set a Github secret named \`PAUSE_PACKAGE_TRIGGER_NEXTCLOUD_MASTER\`" >> $GITHUB_STEP_SUMMARY response=$(curl -iX POST \ https://ci.linuxserver.io/job/Docker-Pipeline-Builders/job/docker-nextcloud/job/master/buildWithParameters?PACKAGE_CHECK=true \ --user ${{ secrets.JENKINS_USER }}:${{ secrets.JENKINS_TOKEN }} | grep -i location | sed "s|^[L|l]ocation: \(.*\)|\1|") @@ -30,6 +33,7 @@ jobs: buildurl=$(curl -s "${response%$'\r'}api/json" | jq -r '.executable.url') buildurl="${buildurl%$'\r'}" echo "**** Jenkins job build url: ${buildurl} ****" + echo "Jenkins job build url: ${buildurl}" >> $GITHUB_STEP_SUMMARY echo "**** Attempting to change the Jenkins job description ****" curl -iX POST \ "${buildurl}submitDescription" \ diff --git a/.github/workflows/package_trigger_scheduler.yml b/.github/workflows/package_trigger_scheduler.yml index 9fcd7d1..6b97339 100755 --- a/.github/workflows/package_trigger_scheduler.yml +++ b/.github/workflows/package_trigger_scheduler.yml @@ -17,18 +17,16 @@ jobs: run: | echo "**** Branches found: ****" git for-each-ref --format='%(refname:short)' refs/remotes - echo "**** Pulling the yq docker image ****" - docker pull ghcr.io/linuxserver/yq for br in $(git for-each-ref --format='%(refname:short)' refs/remotes) do br=$(echo "$br" | sed 's|origin/||g') echo "**** Evaluating branch ${br} ****" - ls_branch=$(curl -sX GET https://raw.githubusercontent.com/linuxserver/docker-nextcloud/${br}/jenkins-vars.yml \ - | docker run --rm -i --entrypoint yq ghcr.io/linuxserver/yq -r .ls_branch) + ls_branch=$(curl -sX GET https://raw.githubusercontent.com/linuxserver/docker-nextcloud/${br}/jenkins-vars.yml | yq -r '.ls_branch') if [ "${br}" == "${ls_branch}" ]; then echo "**** Branch ${br} appears to be live; checking workflow. ****" if curl -sfX GET https://raw.githubusercontent.com/linuxserver/docker-nextcloud/${br}/.github/workflows/package_trigger.yml > /dev/null 2>&1; then echo "**** Workflow exists. Triggering package trigger workflow for branch ${br}. ****" + echo "Triggering package trigger workflow for branch ${br}" >> $GITHUB_STEP_SUMMARY triggered_branches="${triggered_branches}${br} " curl -iX POST \ -H "Authorization: token ${{ secrets.CR_PAT }}" \ @@ -38,9 +36,11 @@ jobs: sleep 30 else echo "**** Workflow doesn't exist; skipping trigger. ****" + echo "Skipping branch ${br} due to no package trigger workflow present." >> $GITHUB_STEP_SUMMARY fi else echo "**** ${br} appears to be a dev branch; skipping trigger. ****" + echo "Skipping branch ${br} due to being detected as dev branch." >> $GITHUB_STEP_SUMMARY fi done echo "**** Package check build(s) triggered for branch(es): ${triggered_branches} ****" diff --git a/Dockerfile b/Dockerfile index 4e868d4..567b1d9 100644 --- a/Dockerfile +++ b/Dockerfile @@ -77,9 +77,9 @@ RUN \ echo "env[PATH] = /usr/local/bin:/usr/bin:/bin" >> /etc/php81/php-fpm.conf && \ echo "**** set version tag ****" && \ if [ -z ${NEXTCLOUD_RELEASE+x} ]; then \ - NEXTCLOUD_RELEASE=$(curl -sX GET https://api.github.com/repos/nextcloud/server/releases/latest \ - | awk '/tag_name/{print $4;exit}' FS='[""]' \ - | sed 's|^v||'); \ + NEXTCLOUD_RELEASE=$(curl -sX GET https://api.github.com/repos/nextcloud/server/releases \ + | jq -r '.[] | select(.prerelease != true) | .tag_name' \ + | sed 's|^v||g' | sort -rV | head -1); \ fi && \ echo "**** download nextcloud ****" && \ curl -o /app/nextcloud.tar.bz2 -L \ diff --git a/Dockerfile.aarch64 b/Dockerfile.aarch64 index 2ed819b..74965e8 100644 --- a/Dockerfile.aarch64 +++ b/Dockerfile.aarch64 @@ -77,9 +77,9 @@ RUN \ echo "env[PATH] = /usr/local/bin:/usr/bin:/bin" >> /etc/php81/php-fpm.conf && \ echo "**** set version tag ****" && \ if [ -z ${NEXTCLOUD_RELEASE+x} ]; then \ - NEXTCLOUD_RELEASE=$(curl -sX GET https://api.github.com/repos/nextcloud/server/releases/latest \ - | awk '/tag_name/{print $4;exit}' FS='[""]' \ - | sed 's|^v||'); \ + NEXTCLOUD_RELEASE=$(curl -sX GET https://api.github.com/repos/nextcloud/server/releases \ + | jq -r '.[] | select(.prerelease != true) | .tag_name' \ + | sed 's|^v||g' | sort -rV | head -1); \ fi && \ echo "**** download nextcloud ****" && \ curl -o /app/nextcloud.tar.bz2 -L \ diff --git a/Dockerfile.armhf b/Dockerfile.armhf index cd94560..6ee1dd0 100644 --- a/Dockerfile.armhf +++ b/Dockerfile.armhf @@ -77,9 +77,9 @@ RUN \ echo "env[PATH] = /usr/local/bin:/usr/bin:/bin" >> /etc/php81/php-fpm.conf && \ echo "**** set version tag ****" && \ if [ -z ${NEXTCLOUD_RELEASE+x} ]; then \ - NEXTCLOUD_RELEASE=$(curl -sX GET https://api.github.com/repos/nextcloud/server/releases/latest \ - | awk '/tag_name/{print $4;exit}' FS='[""]' \ - | sed 's|^v||'); \ + NEXTCLOUD_RELEASE=$(curl -sX GET https://api.github.com/repos/nextcloud/server/releases \ + | jq -r '.[] | select(.prerelease != true) | .tag_name' \ + | sed 's|^v||g' | sort -rV | head -1); \ fi && \ echo "**** download nextcloud ****" && \ curl -o /app/nextcloud.tar.bz2 -L \ diff --git a/Jenkinsfile b/Jenkinsfile index 4dee3c2..15265b1 100644 --- a/Jenkinsfile +++ b/Jenkinsfile @@ -39,10 +39,11 @@ pipeline { // Setup all the basic environment variables needed for the build stage("Set ENV Variables base"){ steps{ + sh '''docker pull quay.io/skopeo/stable:v1 || : ''' script{ env.EXIT_STATUS = '' env.LS_RELEASE = sh( - script: '''docker run --rm ghcr.io/linuxserver/alexeiled-skopeo sh -c 'skopeo inspect docker://docker.io/'${DOCKERHUB_IMAGE}':latest 2>/dev/null' | jq -r '.Labels.build_version' | awk '{print $3}' | grep '\\-ls' || : ''', + script: '''docker run --rm quay.io/skopeo/stable:v1 inspect docker://ghcr.io/${LS_USER}/${CONTAINER_NAME}:latest 2>/dev/null | jq -r '.Labels.build_version' | awk '{print $3}' | grep '\\-ls' || : ''', returnStdout: true).trim() env.LS_RELEASE_NOTES = sh( script: '''cat readme-vars.yml | awk -F \\" '/date: "[0-9][0-9].[0-9][0-9].[0-9][0-9]:/ {print $4;exit;}' | sed -E ':a;N;$!ba;s/\\r{0,1}\\n/\\\\n/g' ''', @@ -104,7 +105,7 @@ pipeline { steps{ script{ env.EXT_RELEASE = sh( - script: ''' curl -sX GET https://api.github.com/repos/nextcloud/server/releases/latest | jq -r '. | .tag_name' | sed 's|^v||' ''', + script: ''' curl -sX GET https://api.github.com/repos/nextcloud/server/releases | jq -r '.[] | select(.prerelease != true) | .tag_name' | sed 's|^v||g' | sort -rV | head -1 ''', returnStdout: true).trim() env.RELEASE_LINK = 'custom_command' } @@ -228,7 +229,7 @@ pipeline { script{ env.SHELLCHECK_URL = 'https://ci-tests.linuxserver.io/' + env.IMAGE + '/' + env.META_TAG + '/shellcheck-result.xml' } - sh '''curl -sL https://raw.githubusercontent.com/linuxserver/docker-shellcheck/master/checkrun.sh | /bin/bash''' + sh '''curl -sL https://raw.githubusercontent.com/linuxserver/docker-jenkins-builder/master/checkrun.sh | /bin/bash''' sh '''#! /bin/bash docker run --rm \ -v ${WORKSPACE}:/mnt \ @@ -376,6 +377,26 @@ pipeline { } } } + // If this is a master build check the S6 service file perms + stage("Check S6 Service file Permissions"){ + when { + branch "master" + environment name: 'CHANGE_ID', value: '' + environment name: 'EXIT_STATUS', value: '' + } + steps { + script{ + sh '''#! /bin/bash + WRONG_PERM=$(find ./ -path "./.git" -prune -o \\( -name "run" -o -name "finish" -o -name "check" \\) -not -perm -u=x,g=x,o=x -print) + if [[ -n "${WRONG_PERM}" ]]; then + echo "The following S6 service files are missing the executable bit; canceling the faulty build: ${WRONG_PERM}" + exit 1 + else + echo "S6 service file perms look good." + fi ''' + } + } + } /* ####################### GitLab Mirroring ####################### */ @@ -668,6 +689,7 @@ pipeline { ]) { script{ env.CI_URL = 'https://ci-tests.linuxserver.io/' + env.IMAGE + '/' + env.META_TAG + '/index.html' + env.CI_JSON_URL = 'https://ci-tests.linuxserver.io/' + env.IMAGE + '/' + env.META_TAG + '/report.json' } sh '''#! /bin/bash set -e @@ -694,8 +716,6 @@ pipeline { -e WEB_SCREENSHOT=\"${CI_WEB}\" \ -e WEB_AUTH=\"${CI_AUTH}\" \ -e WEB_PATH=\"${CI_WEBPATH}\" \ - -e DO_REGION="ams3" \ - -e DO_BUCKET="lsio-ci" \ -t ghcr.io/linuxserver/ci:latest \ python3 test_build.py''' } @@ -949,8 +969,67 @@ pipeline { environment name: 'EXIT_STATUS', value: '' } steps { - sh '''curl -H "Authorization: token ${GITHUB_TOKEN}" -X POST https://api.github.com/repos/${LS_USER}/${LS_REPO}/issues/${PULL_REQUEST}/comments \ - -d '{"body": "I am a bot, here are the test results for this PR: \\n'${CI_URL}' \\n'${SHELLCHECK_URL}'"}' ''' + sh '''#! /bin/bash + # Function to retrieve JSON data from URL + get_json() { + local url="$1" + local response=$(curl -s "$url") + if [ $? -ne 0 ]; then + echo "Failed to retrieve JSON data from $url" + return 1 + fi + local json=$(echo "$response" | jq .) + if [ $? -ne 0 ]; then + echo "Failed to parse JSON data from $url" + return 1 + fi + echo "$json" + } + + build_table() { + local data="$1" + + # Get the keys in the JSON data + local keys=$(echo "$data" | jq -r 'to_entries | map(.key) | .[]') + + # Check if keys are empty + if [ -z "$keys" ]; then + echo "JSON report data does not contain any keys or the report does not exist." + return 1 + fi + + # Build table header + local header="| Tag | Passed |\\n| --- | --- |\\n" + + # Loop through the JSON data to build the table rows + local rows="" + for build in $keys; do + local status=$(echo "$data" | jq -r ".[\\"$build\\"].test_success") + if [ "$status" = "true" ]; then + status="✅" + else + status="❌" + fi + local row="| "$build" | "$status" |\\n" + rows="${rows}${row}" + done + + local table="${header}${rows}" + local escaped_table=$(echo "$table" | sed 's/\"/\\\\"/g') + echo "$escaped_table" + } + + # Retrieve JSON data from URL + data=$(get_json "$CI_JSON_URL") + # Create table from JSON data + table=$(build_table "$data") + echo -e "$table" + + curl -X POST -H "Authorization: token $GITHUB_TOKEN" \ + -H "Accept: application/vnd.github.v3+json" \ + "https://api.github.com/repos/$LS_USER/$LS_REPO/issues/$PULL_REQUEST/comments" \ + -d "{\\"body\\": \\"I am a bot, here are the test results for this PR: \\n${CI_URL}\\n${SHELLCHECK_URL}\\n${table}\\"}"''' + } } } diff --git a/jenkins-vars.yml b/jenkins-vars.yml index dd32663..82aeb3f 100644 --- a/jenkins-vars.yml +++ b/jenkins-vars.yml @@ -3,7 +3,7 @@ # jenkins variables project_name: docker-nextcloud external_type: na -custom_version_command: "curl -sX GET https://api.github.com/repos/nextcloud/server/releases/latest | jq -r '. | .tag_name' | sed 's|^v||'" +custom_version_command: "curl -sX GET https://api.github.com/repos/nextcloud/server/releases | jq -r '.[] | select(.prerelease != true) | .tag_name' | sed 's|^v||g' | sort -rV | head -1" release_type: stable release_tag: latest ls_branch: master