Bot Updating Templated Files

Switch alpine321 to prerelease

.github/CONTRIBUTING.md

@@ -6,7 +6,7 @@
 * Read, and fill the Pull Request template
   * If this is a fix for a typo (in code, documentation, or the README) please file an issue and let us sort it out. We do not need a PR
   * If the PR is addressing an existing issue include, closes #\<issue number>, in the body of the PR commit message
-* If you want to discuss changes, you can also bring it up in [#dev-talk](https://discordapp.com/channels/354974912613449730/757585807061155840) in our [Discord server](https://discord.gg/YWrKVTn)
+* If you want to discuss changes, you can also bring it up in [#dev-talk](https://discordapp.com/channels/354974912613449730/757585807061155840) in our [Discord server](https://linuxserver.io/discord)
 
 ## Common files
 

.github/ISSUE_TEMPLATE/config.yml

@@ -1,7 +1,7 @@
 blank_issues_enabled: false
 contact_links:
   - name: Discord chat support
-    url: https://discord.gg/YWrKVTn
+    url: https://linuxserver.io/discord
     about: Realtime support / chat with the community and the team.
 
   - name: Discourse discussion forum

.github/workflows/call_issue_pr_tracker.yml

@@ -8,6 +8,9 @@ on:
   pull_request_review:
     types: [submitted,edited,dismissed]
 
+permissions:
+  contents: read
+
 jobs:
   manage-project:
     permissions:

.github/workflows/call_issues_cron.yml

@@ -4,6 +4,9 @@ on:
     - cron:  '11 7 * * *'
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 jobs:
   stale:
     permissions:

.github/workflows/external_trigger.yml

@@ -3,6 +3,9 @@ name: External Trigger Main
 on:
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 jobs:
   external-trigger-alpine321:
     runs-on: ubuntu-latest
@@ -15,7 +18,10 @@ jobs:
           SKIP_EXTERNAL_TRIGGER: ${{ vars.SKIP_EXTERNAL_TRIGGER }}
         run: |
           printf "# External trigger for docker-python\n\n" >> $GITHUB_STEP_SUMMARY
-          if grep -q "^python_alpine321" <<< "${SKIP_EXTERNAL_TRIGGER}"; then
+          if grep -q "^python_alpine321_" <<< "${SKIP_EXTERNAL_TRIGGER}"; then
+            echo "> [!NOTE]" >> $GITHUB_STEP_SUMMARY
+            echo "> Github organizational variable \`SKIP_EXTERNAL_TRIGGER\` contains \`python_alpine321_\`; will skip trigger if version matches." >> $GITHUB_STEP_SUMMARY
+          elif grep -q "^python_alpine321" <<< "${SKIP_EXTERNAL_TRIGGER}"; then
             echo "> [!WARNING]" >> $GITHUB_STEP_SUMMARY
             echo "> Github organizational variable \`SKIP_EXTERNAL_TRIGGER\` contains \`python_alpine321\`; skipping trigger." >> $GITHUB_STEP_SUMMARY
             exit 0
@@ -25,6 +31,11 @@ jobs:
           printf "\n## Retrieving external version\n\n" >> $GITHUB_STEP_SUMMARY
           EXT_RELEASE=$(curl -u ${{ secrets.CR_USER }}:${{ secrets.CR_PAT }} -sX GET https://api.github.com/repos/python/cpython/tags | jq -r '.[] | select(.name | contains("rc") or contains("a") or contains("b") | not) | .name' | sed 's|^v||g' | sort -rV | head -1)
           echo "Type is \`custom_version_command\`" >> $GITHUB_STEP_SUMMARY
+          if grep -q "^python_alpine321_${EXT_RELEASE}" <<< "${SKIP_EXTERNAL_TRIGGER}"; then
+            echo "> [!WARNING]" >> $GITHUB_STEP_SUMMARY
+            echo "> Github organizational variable \`SKIP_EXTERNAL_TRIGGER\` matches current external release; skipping trigger." >> $GITHUB_STEP_SUMMARY
+            exit 0
+          fi
           if [ -z "${EXT_RELEASE}" ] || [ "${EXT_RELEASE}" == "null" ]; then
             echo "> [!WARNING]" >> $GITHUB_STEP_SUMMARY
             echo "> Can't retrieve external version, exiting" >> $GITHUB_STEP_SUMMARY
@@ -35,8 +46,8 @@ jobs:
               "username": "Github Actions"}' ${{ secrets.DISCORD_WEBHOOK }}
             exit 1
           fi
-          EXT_RELEASE=$(echo ${EXT_RELEASE} | sed 's/[~,%@+;:/]//g')
-          echo "External version: \`${EXT_RELEASE}\`" >> $GITHUB_STEP_SUMMARY
+          EXT_RELEASE_SANITIZED=$(echo ${EXT_RELEASE} | sed 's/[~,%@+;:/]//g')
+          echo "Sanitized external version: \`${EXT_RELEASE_SANITIZED}\`" >> $GITHUB_STEP_SUMMARY
           echo "Retrieving last pushed version" >> $GITHUB_STEP_SUMMARY
           image="linuxserver/python"
           tag="alpine321"
@@ -48,13 +59,30 @@ jobs:
             --header "Accept: application/vnd.oci.image.index.v1+json" \
             --header "Authorization: Bearer ${token}" \
             "https://ghcr.io/v2/${image}/manifests/${tag}")
-          multidigest=$(jq -r ".manifests[] | select(.platform.architecture == \"amd64\").digest?" <<< "${multidigest}")
-          digest=$(curl -s \
-            --header "Accept: application/vnd.docker.distribution.manifest.v2+json" \
-            --header "Accept: application/vnd.oci.image.manifest.v1+json" \
-            --header "Authorization: Bearer ${token}" \
-            "https://ghcr.io/v2/${image}/manifests/${multidigest}" \
-            | jq -r '.config.digest')
+          if jq -e '.layers // empty' <<< "${multidigest}" >/dev/null 2>&1; then
+            # If there's a layer element it's a single-arch manifest so just get that digest
+            digest=$(jq -r '.config.digest' <<< "${multidigest}")
+          else
+            # Otherwise it's multi-arch or has manifest annotations
+            if jq -e '.manifests[]?.annotations // empty' <<< "${multidigest}" >/dev/null 2>&1; then
+              # Check for manifest annotations and delete if found
+              multidigest=$(jq 'del(.manifests[] | select(.annotations))' <<< "${multidigest}")
+            fi
+            if [[ $(jq '.manifests | length' <<< "${multidigest}") -gt 1 ]]; then
+              # If there's still more than one digest, it's multi-arch
+              multidigest=$(jq -r ".manifests[] | select(.platform.architecture == \"amd64\").digest?" <<< "${multidigest}")
+            else
+              # Otherwise it's single arch
+              multidigest=$(jq -r ".manifests[].digest?" <<< "${multidigest}")
+            fi
+            if digest=$(curl -s \
+              --header "Accept: application/vnd.docker.distribution.manifest.v2+json" \
+              --header "Accept: application/vnd.oci.image.manifest.v1+json" \
+              --header "Authorization: Bearer ${token}" \
+              "https://ghcr.io/v2/${image}/manifests/${multidigest}"); then
+              digest=$(jq -r '.config.digest' <<< "${digest}");
+            fi
+          fi
           image_info=$(curl -sL \
             --header "Authorization: Bearer ${token}" \
             "https://ghcr.io/v2/${image}/blobs/${digest}")
@@ -75,8 +103,8 @@ jobs:
             exit 1
           fi
           echo "Last pushed version: \`${IMAGE_VERSION}\`" >> $GITHUB_STEP_SUMMARY
-          if [ "${EXT_RELEASE}" == "${IMAGE_VERSION}" ]; then
-            echo "Version \`${EXT_RELEASE}\` already pushed, exiting" >> $GITHUB_STEP_SUMMARY
+          if [ "${EXT_RELEASE_SANITIZED}" == "${IMAGE_VERSION}" ]; then
+            echo "Sanitized version \`${EXT_RELEASE_SANITIZED}\` already pushed, exiting" >> $GITHUB_STEP_SUMMARY
             exit 0
           elif [ $(curl -s https://ci.linuxserver.io/job/Docker-Pipeline-Builders/job/docker-python/job/alpine321/lastBuild/api/json | jq -r '.building') == "true" ]; then
             echo "New version \`${EXT_RELEASE}\` found; but there already seems to be an active build on Jenkins; exiting" >> $GITHUB_STEP_SUMMARY
@@ -91,8 +119,8 @@ jobs:
                 "username": "Github Actions"}' ${{ secrets.DISCORD_WEBHOOK }}
             else
               printf "\n## Trigger new build\n\n" >> $GITHUB_STEP_SUMMARY
-              echo "New version \`${EXT_RELEASE}\` found; old version was \`${IMAGE_VERSION}\`. Triggering new build" >> $GITHUB_STEP_SUMMARY
-              if "${artifacts_found}" == "true" ]]; then
+              echo "New sanitized version \`${EXT_RELEASE_SANITIZED}\` found; old version was \`${IMAGE_VERSION}\`. Triggering new build" >> $GITHUB_STEP_SUMMARY
+              if [[ "${artifacts_found}" == "true" ]]; then
                 echo "All artifacts seem to be uploaded." >> $GITHUB_STEP_SUMMARY
               fi
               response=$(curl -iX POST \
@@ -111,7 +139,7 @@ jobs:
                 --data-urlencode "description=GHA external trigger https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
                 --data-urlencode "Submit=Submit"
               echo "**** Notifying Discord ****"
-              TRIGGER_REASON="A version change was detected for python tag alpine321. Old version:${IMAGE_VERSION} New version:${EXT_RELEASE}"
+              TRIGGER_REASON="A version change was detected for python tag alpine321. Old version:${IMAGE_VERSION} New version:${EXT_RELEASE_SANITIZED}"
               curl -X POST -H "Content-Type: application/json" --data '{"avatar_url": "https://cdn.discordapp.com/avatars/354986384542662657/df91181b3f1cf0ef1592fbe18e0962d7.png","embeds": [{"color": 9802903,
                 "description": "**Build Triggered** \n**Reason:** '"${TRIGGER_REASON}"' \n**Build URL:** '"${buildurl}display/redirect"' \n"}],
                 "username": "Github Actions"}' ${{ secrets.DISCORD_WEBHOOK }}

.github/workflows/external_trigger_scheduler.yml

@@ -5,6 +5,9 @@ on:
     - cron:  '19 * * * *'
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 jobs:
   external-trigger-scheduler:
     runs-on: ubuntu-latest

.github/workflows/greetings.yml

@@ -2,8 +2,14 @@ name: Greetings
 
 on: [pull_request_target, issues]
 
+permissions:
+  contents: read
+
 jobs:
   greeting:
+    permissions:
+      issues: write
+      pull-requests: write
     runs-on: ubuntu-latest
     steps:
     - uses: actions/first-interaction@v1

.github/workflows/package_trigger_scheduler.yml

@@ -5,6 +5,9 @@ on:
     - cron:  '22 7 * * 2'
   workflow_dispatch:
 
+permissions:
+  contents: read
+
 jobs:
   package-trigger-scheduler:
     runs-on: ubuntu-latest
@@ -27,9 +30,18 @@ jobs:
             fi
             printf "\n## Evaluating \`%s\`\n\n" ${br} >> $GITHUB_STEP_SUMMARY
             JENKINS_VARS=$(curl -sX GET https://raw.githubusercontent.com/linuxserver/docker-python/${br}/jenkins-vars.yml)
-            if [[ "${br}" == $(yq -r '.ls_branch' <<< "${JENKINS_VARS}") ]]; then
+            if ! curl -sfX GET https://raw.githubusercontent.com/linuxserver/docker-python/${br}/Jenkinsfile >/dev/null 2>&1; then
+              echo "> [!WARNING]" >> $GITHUB_STEP_SUMMARY
+              echo "> No Jenkinsfile found. Branch is either deprecated or is an early dev branch." >> $GITHUB_STEP_SUMMARY
+              skipped_branches="${skipped_branches}${br} "
+            elif [[ "${br}" == $(yq -r '.ls_branch' <<< "${JENKINS_VARS}") ]]; then
               echo "Branch appears to be live; checking workflow." >> $GITHUB_STEP_SUMMARY
-              if [[ $(yq -r '.skip_package_check' <<< "${JENKINS_VARS}") == "true" ]]; then
+              README_VARS=$(curl -sX GET https://raw.githubusercontent.com/linuxserver/docker-python/${br}/readme-vars.yml)
+              if [[ $(yq -r '.project_deprecation_status' <<< "${README_VARS}") == "true" ]]; then
+                echo "> [!WARNING]" >> $GITHUB_STEP_SUMMARY
+                echo "> Branch appears to be deprecated; skipping trigger." >> $GITHUB_STEP_SUMMARY
+                skipped_branches="${skipped_branches}${br} "
+              elif [[ $(yq -r '.skip_package_check' <<< "${JENKINS_VARS}") == "true" ]]; then
                 echo "> [!WARNING]" >> $GITHUB_STEP_SUMMARY
                 echo "> Skipping branch ${br} due to \`skip_package_check\` being set in \`jenkins-vars.yml\`." >> $GITHUB_STEP_SUMMARY
                 skipped_branches="${skipped_branches}${br} "
@@ -37,7 +49,7 @@ jobs:
                 echo "> [!WARNING]" >> $GITHUB_STEP_SUMMARY
                 echo "> Github organizational variable \`SKIP_PACKAGE_TRIGGER\` contains \`python_${br}\`; skipping trigger." >> $GITHUB_STEP_SUMMARY
                 skipped_branches="${skipped_branches}${br} "
-              elif [ $(curl -s https://ci.linuxserver.io/job/Docker-Pipeline-Builders/job/docker-python/job/${br}/lastBuild/api/json | jq -r '.building') == "true" ]; then
+              elif [ $(curl -s https://ci.linuxserver.io/job/Docker-Pipeline-Builders/job/docker-python/job/${br}/lastBuild/api/json | jq -r '.building' 2>/dev/null) == "true" ]; then
                 echo "> [!WARNING]" >> $GITHUB_STEP_SUMMARY
                 echo "> There already seems to be an active build on Jenkins; skipping package trigger for ${br}" >> $GITHUB_STEP_SUMMARY
                 skipped_branches="${skipped_branches}${br} "
@@ -49,6 +61,11 @@ jobs:
                 response=$(curl -iX POST \
                   https://ci.linuxserver.io/job/Docker-Pipeline-Builders/job/docker-python/job/${br}/buildWithParameters?PACKAGE_CHECK=true \
                   --user ${{ secrets.JENKINS_USER }}:${{ secrets.JENKINS_TOKEN }} | grep -i location | sed "s|^[L|l]ocation: \(.*\)|\1|")
+                if [[ -z "${response}" ]]; then
+                  echo "> [!WARNING]" >> $GITHUB_STEP_SUMMARY
+                  echo "> Jenkins build could not be triggered. Skipping branch."
+                  continue
+                fi
                 echo "Jenkins [job queue url](${response%$'\r'})" >> $GITHUB_STEP_SUMMARY
                 echo "Sleeping 10 seconds until job starts" >> $GITHUB_STEP_SUMMARY
                 sleep 10
@@ -56,11 +73,14 @@ jobs:
                 buildurl="${buildurl%$'\r'}"
                 echo "Jenkins job [build url](${buildurl})" >> $GITHUB_STEP_SUMMARY
                 echo "Attempting to change the Jenkins job description" >> $GITHUB_STEP_SUMMARY
-                curl -iX POST \
+                if ! curl -ifX POST \
                   "${buildurl}submitDescription" \
                   --user ${{ secrets.JENKINS_USER }}:${{ secrets.JENKINS_TOKEN }} \
                   --data-urlencode "description=GHA package trigger https://github.com/${{ github.repository }}/actions/runs/${{ github.run_id }}" \
-                  --data-urlencode "Submit=Submit"
+                  --data-urlencode "Submit=Submit"; then
+                  echo "> [!WARNING]" >> $GITHUB_STEP_SUMMARY
+                  echo "> Unable to change the Jenkins job description."
+                fi
                 sleep 20
               fi
             else

.github/workflows/publish_binaries.yml

@@ -9,6 +9,9 @@ jobs:
   publish-binaries:
     runs-on: ubuntu-latest
     steps:
+      - name: Set up QEMU
+        uses: docker/setup-qemu-action@v3
+
       - name: Retrieve and upload artifacts
         run: |
           mkdir -p ./artifacts/amd64 ./artifacts/arm64
@@ -22,7 +25,7 @@ jobs:
             -H "Content-Type: application/octet-stream" \
             "https://uploads.github.com/repos/linuxserver/docker-python/releases/${{ github.event.release.id }}/assets?name=python-amd64.tar.gz" \
             --data-binary "@python-amd64.tar.gz"
-          docker create --name arm64 ghcr.io/linuxserver/python:arm64v8-${{ github.event.release.tag_name }} blah
+          docker create --name arm64 --platform=linux/arm64 ghcr.io/linuxserver/python:arm64v8-${{ github.event.release.tag_name }} blah
           docker cp arm64:/usr/local/. ./artifacts/arm64/
           tar -czvf python-arm64.tar.gz -C ./artifacts/arm64/ .
           echo "**** Uploading arm64 tarball ****"

Dockerfile

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1
 
-FROM ghcr.io/linuxserver/baseimage-alpine:3.21 as buildstage
+FROM ghcr.io/linuxserver/baseimage-alpine:3.21 AS buildstage
 
 # set version label
 ARG PYTHON_VERSION

Dockerfile.aarch64

@@ -1,6 +1,6 @@
 # syntax=docker/dockerfile:1
 
-FROM ghcr.io/linuxserver/baseimage-alpine:arm64v8-3.21 as buildstage
+FROM ghcr.io/linuxserver/baseimage-alpine:arm64v8-3.21 AS buildstage
 
 # set version label
 ARG PYTHON_VERSION

Jenkinsfile

@@ -56,11 +56,23 @@ pipeline {
       steps{
         echo "Running on node: ${NODE_NAME}"
         sh '''#! /bin/bash
-              containers=$(docker ps -aq)
+              echo "Pruning builder"
+              docker builder prune -f --builder container || :
+              containers=$(docker ps -q)
               if [[ -n "${containers}" ]]; then
-                docker stop ${containers}
+                BUILDX_CONTAINER_ID=$(docker ps -qf 'name=buildx_buildkit')
+                for container in ${containers}; do
+                  if [[ "${container}" == "${BUILDX_CONTAINER_ID}" ]]; then
+                    echo "skipping buildx container in docker stop"
+                  else
+                    echo "Stopping container ${container}"
+                    docker stop ${container}
+                  fi
+                done
               fi
-              docker system prune -af --volumes || : '''
+              docker system prune -f --volumes || :
+              docker image prune -af || :
+           '''
         script{
           env.EXIT_STATUS = ''
           env.LS_RELEASE = sh(
@@ -82,7 +94,11 @@ pipeline {
           env.DOCKERHUB_LINK = 'https://hub.docker.com/r/' + env.DOCKERHUB_IMAGE + '/tags/'
           env.PULL_REQUEST = env.CHANGE_ID
           env.TEMPLATED_FILES = 'Jenkinsfile README.md LICENSE .editorconfig ./.github/CONTRIBUTING.md ./.github/FUNDING.yml ./.github/ISSUE_TEMPLATE/config.yml ./.github/ISSUE_TEMPLATE/issue.bug.yml ./.github/ISSUE_TEMPLATE/issue.feature.yml ./.github/PULL_REQUEST_TEMPLATE.md ./.github/workflows/external_trigger_scheduler.yml ./.github/workflows/greetings.yml ./.github/workflows/package_trigger_scheduler.yml ./.github/workflows/call_issue_pr_tracker.yml ./.github/workflows/call_issues_cron.yml ./.github/workflows/permissions.yml ./.github/workflows/external_trigger.yml'
+          if ( env.SYFT_IMAGE_TAG == null ) {
+            env.SYFT_IMAGE_TAG = 'latest'
+          }
         }
+        echo "Using syft image tag ${SYFT_IMAGE_TAG}"
         sh '''#! /bin/bash
               echo "The default github branch detected as ${GH_DEFAULT_BRANCH}" '''
         script{
@@ -192,6 +208,7 @@ pipeline {
           env.META_TAG = 'alpine321-' + env.EXT_RELEASE_CLEAN + '-ls' + env.LS_TAG_NUMBER
           env.EXT_RELEASE_TAG = 'alpine321-version-' + env.EXT_RELEASE_CLEAN
           env.BUILDCACHE = 'docker.io/lsiodev/buildcache,registry.gitlab.com/linuxserver.io/docker-jenkins-builder/lsiodev-buildcache,ghcr.io/linuxserver/lsiodev-buildcache,quay.io/linuxserver.io/lsiodev-buildcache'
+          env.CITEST_IMAGETAG = 'latest'
         }
       }
     }
@@ -217,6 +234,7 @@ pipeline {
           env.EXT_RELEASE_TAG = 'alpine321-version-' + env.EXT_RELEASE_CLEAN
           env.DOCKERHUB_LINK = 'https://hub.docker.com/r/' + env.DEV_DOCKERHUB_IMAGE + '/tags/'
           env.BUILDCACHE = 'docker.io/lsiodev/buildcache,registry.gitlab.com/linuxserver.io/docker-jenkins-builder/lsiodev-buildcache,ghcr.io/linuxserver/lsiodev-buildcache,quay.io/linuxserver.io/lsiodev-buildcache'
+          env.CITEST_IMAGETAG = 'develop'
         }
       }
     }
@@ -242,6 +260,7 @@ pipeline {
           env.CODE_URL = 'https://github.com/' + env.LS_USER + '/' + env.LS_REPO + '/pull/' + env.PULL_REQUEST
           env.DOCKERHUB_LINK = 'https://hub.docker.com/r/' + env.PR_DOCKERHUB_IMAGE + '/tags/'
           env.BUILDCACHE = 'docker.io/lsiodev/buildcache,registry.gitlab.com/linuxserver.io/docker-jenkins-builder/lsiodev-buildcache,ghcr.io/linuxserver/lsiodev-buildcache,quay.io/linuxserver.io/lsiodev-buildcache'
+          env.CITEST_IMAGETAG = 'develop'
         }
       }
     }
@@ -264,7 +283,7 @@ pipeline {
                   -v ${WORKSPACE}:/mnt \
                   -e AWS_ACCESS_KEY_ID=\"${S3_KEY}\" \
                   -e AWS_SECRET_ACCESS_KEY=\"${S3_SECRET}\" \
-                  ghcr.io/linuxserver/baseimage-alpine:3.20 s6-envdir -fn -- /var/run/s6/container_environment /bin/bash -c "\
+                  ghcr.io/linuxserver/baseimage-alpine:3 s6-envdir -fn -- /var/run/s6/container_environment /bin/bash -c "\
                     apk add --no-cache python3 && \
                     python3 -m venv /lsiopy && \
                     pip install --no-cache-dir -U pip && \
@@ -512,7 +531,7 @@ pipeline {
           --label \"org.opencontainers.image.title=Python\" \
           --label \"org.opencontainers.image.description=python image by linuxserver.io\" \
           --no-cache --pull -t ${IMAGE}:${META_TAG} --platform=linux/amd64 \
-          --provenance=false --sbom=false --builder=container --load \
+          --provenance=true --sbom=true --builder=container --load \
           --build-arg ${BUILD_VERSION_ARG}=${EXT_RELEASE} --build-arg VERSION=\"${VERSION_TAG}\" --build-arg BUILD_DATE=${GITHUB_DATE} ."
         sh '''#! /bin/bash
               set -e
@@ -536,12 +555,17 @@ pipeline {
                     echo $GITHUB_TOKEN | docker login ghcr.io -u LinuxServer-CI --password-stdin
                     echo $GITLAB_TOKEN | docker login registry.gitlab.com -u LinuxServer.io --password-stdin
                     echo $QUAYPASS | docker login quay.io -u $QUAYUSER --password-stdin
+
                     if [[ "${PACKAGE_CHECK}" != "true" ]]; then
+                      declare -A pids
                       IFS=',' read -ra CACHE <<< "$BUILDCACHE"
                       for i in "${CACHE[@]}"; do
                         docker push ${i}:amd64-${COMMIT_SHA}-${BUILD_NUMBER} &
+                        pids[$!]="$i"
+                      done
+                      for p in "${!pids[@]}"; do
+                        wait "$p" || { [[ "${pids[$p]}" != *"quay.io"* ]] && exit 1; }
                       done
-                      wait
                     fi
                 '''
           }
@@ -575,7 +599,7 @@ pipeline {
               --label \"org.opencontainers.image.title=Python\" \
               --label \"org.opencontainers.image.description=python image by linuxserver.io\" \
               --no-cache --pull -t ${IMAGE}:amd64-${META_TAG} --platform=linux/amd64 \
-              --provenance=false --sbom=false --builder=container --load \
+              --provenance=true --sbom=true --builder=container --load \
               --build-arg ${BUILD_VERSION_ARG}=${EXT_RELEASE} --build-arg VERSION=\"${VERSION_TAG}\" --build-arg BUILD_DATE=${GITHUB_DATE} ."
             sh '''#! /bin/bash
                   set -e
@@ -599,12 +623,17 @@ pipeline {
                         echo $GITHUB_TOKEN | docker login ghcr.io -u LinuxServer-CI --password-stdin
                         echo $GITLAB_TOKEN | docker login registry.gitlab.com -u LinuxServer.io --password-stdin
                         echo $QUAYPASS | docker login quay.io -u $QUAYUSER --password-stdin
+
                         if [[ "${PACKAGE_CHECK}" != "true" ]]; then
+                          declare -A pids
                           IFS=',' read -ra CACHE <<< "$BUILDCACHE"
                           for i in "${CACHE[@]}"; do
                             docker push ${i}:amd64-${COMMIT_SHA}-${BUILD_NUMBER} &
+                            pids[$!]="$i"
+                          done
+                          for p in "${!pids[@]}"; do
+                            wait "$p" || { [[ "${pids[$p]}" != *"quay.io"* ]] && exit 1; }
                           done
-                          wait
                         fi
                     '''
               }
@@ -632,7 +661,7 @@ pipeline {
               --label \"org.opencontainers.image.title=Python\" \
               --label \"org.opencontainers.image.description=python image by linuxserver.io\" \
               --no-cache --pull -f Dockerfile.aarch64 -t ${IMAGE}:arm64v8-${META_TAG} --platform=linux/arm64 \
-              --provenance=false --sbom=false --builder=container --load \
+              --provenance=true --sbom=true --builder=container --load \
               --build-arg ${BUILD_VERSION_ARG}=${EXT_RELEASE} --build-arg VERSION=\"${VERSION_TAG}\" --build-arg BUILD_DATE=${GITHUB_DATE} ."
             sh '''#! /bin/bash
                   set -e
@@ -657,11 +686,15 @@ pipeline {
                         echo $GITLAB_TOKEN | docker login registry.gitlab.com -u LinuxServer.io --password-stdin
                         echo $QUAYPASS | docker login quay.io -u $QUAYUSER --password-stdin
                         if [[ "${PACKAGE_CHECK}" != "true" ]]; then
+                          declare -A pids
                           IFS=',' read -ra CACHE <<< "$BUILDCACHE"
                           for i in "${CACHE[@]}"; do
                             docker push ${i}:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER} &
+                            pids[$!]="$i"
+                          done
+                          for p in "${!pids[@]}"; do
+                            wait "$p" || { [[ "${pids[$p]}" != *"quay.io"* ]] && exit 1; }
                           done
-                          wait
                         fi
                     '''
               }
@@ -671,7 +704,8 @@ pipeline {
                   if [[ -n "${containers}" ]]; then
                     docker stop ${containers}
                   fi
-                  docker system prune -af --volumes || :
+                  docker system prune -f --volumes || :
+                  docker image prune -af || :
                '''
           }
         }
@@ -705,7 +739,7 @@ pipeline {
                     CI_DOCKERENV="LSIO_FIRST_PARTY=true"
                   fi
                 fi
-                docker pull ghcr.io/linuxserver/ci:latest
+                docker pull ghcr.io/linuxserver/ci:${CITEST_IMAGETAG}
                 if [ "${MULTIARCH}" == "true" ]; then
                   docker pull ghcr.io/linuxserver/lsiodev-buildcache:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER} --platform=arm64
                   docker tag ghcr.io/linuxserver/lsiodev-buildcache:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER} ${IMAGE}:arm64v8-${META_TAG}
@@ -728,7 +762,10 @@ pipeline {
                 -e WEB_AUTH=\"${CI_AUTH}\" \
                 -e WEB_PATH=\"${CI_WEBPATH}\" \
                 -e NODE_NAME=\"${NODE_NAME}\" \
-                -t ghcr.io/linuxserver/ci:latest \
+                -e SYFT_IMAGE_TAG=\"${CI_SYFT_IMAGE_TAG:-${SYFT_IMAGE_TAG}}\" \
+                -e COMMIT_SHA=\"${COMMIT_SHA}\" \
+                -e BUILD_NUMBER=\"${BUILD_NUMBER}\" \
+                -t ghcr.io/linuxserver/ci:${CITEST_IMAGETAG} \
                 python3 test_build.py'''
         }
       }
@@ -754,9 +791,11 @@ pipeline {
                           CACHEIMAGE=${i}
                       fi
                   done
-                  docker buildx imagetools create --prefer-index=false -t ${PUSHIMAGE}:${META_TAG} -t ${PUSHIMAGE}:alpine321 -t ${PUSHIMAGE}:${EXT_RELEASE_TAG} ${CACHEIMAGE}:amd64-${COMMIT_SHA}-${BUILD_NUMBER}
+                  docker buildx imagetools create --prefer-index=false -t ${PUSHIMAGE}:${META_TAG} -t ${PUSHIMAGE}:alpine321 -t ${PUSHIMAGE}:${EXT_RELEASE_TAG} ${CACHEIMAGE}:amd64-${COMMIT_SHA}-${BUILD_NUMBER} || \
+                    { if [[ "${PUSHIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
                   if [ -n "${SEMVER}" ]; then
-                    docker buildx imagetools create --prefer-index=false -t ${PUSHIMAGE}:${SEMVER} ${CACHEIMAGE}:amd64-${COMMIT_SHA}-${BUILD_NUMBER}
+                    docker buildx imagetools create --prefer-index=false -t ${PUSHIMAGE}:${SEMVER} ${CACHEIMAGE}:amd64-${COMMIT_SHA}-${BUILD_NUMBER} || \
+                      { if [[ "${PUSHIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
                   fi
                 done
               '''
@@ -781,20 +820,27 @@ pipeline {
                           CACHEIMAGE=${i}
                       fi
                   done
-                  docker buildx imagetools create --prefer-index=false -t ${MANIFESTIMAGE}:amd64-${META_TAG} -t ${MANIFESTIMAGE}:amd64-alpine321 -t ${MANIFESTIMAGE}:amd64-${EXT_RELEASE_TAG} ${CACHEIMAGE}:amd64-${COMMIT_SHA}-${BUILD_NUMBER}
-                  docker buildx imagetools create --prefer-index=false -t ${MANIFESTIMAGE}:arm64v8-${META_TAG} -t ${MANIFESTIMAGE}:arm64v8-alpine321 -t ${MANIFESTIMAGE}:arm64v8-${EXT_RELEASE_TAG} ${CACHEIMAGE}:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER}
+                  docker buildx imagetools create --prefer-index=false -t ${MANIFESTIMAGE}:amd64-${META_TAG} -t ${MANIFESTIMAGE}:amd64-alpine321 -t ${MANIFESTIMAGE}:amd64-${EXT_RELEASE_TAG} ${CACHEIMAGE}:amd64-${COMMIT_SHA}-${BUILD_NUMBER} || \
+                    { if [[ "${MANIFESTIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
+                  docker buildx imagetools create --prefer-index=false -t ${MANIFESTIMAGE}:arm64v8-${META_TAG} -t ${MANIFESTIMAGE}:arm64v8-alpine321 -t ${MANIFESTIMAGE}:arm64v8-${EXT_RELEASE_TAG} ${CACHEIMAGE}:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER} || \
+                    { if [[ "${MANIFESTIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
                   if [ -n "${SEMVER}" ]; then
-                    docker buildx imagetools create --prefer-index=false -t ${MANIFESTIMAGE}:amd64-${SEMVER} ${CACHEIMAGE}:amd64-${COMMIT_SHA}-${BUILD_NUMBER}
-                    docker buildx imagetools create --prefer-index=false -t ${MANIFESTIMAGE}:arm64v8-${SEMVER} ${CACHEIMAGE}:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER}
+                    docker buildx imagetools create --prefer-index=false -t ${MANIFESTIMAGE}:amd64-${SEMVER} ${CACHEIMAGE}:amd64-${COMMIT_SHA}-${BUILD_NUMBER} || \
+                      { if [[ "${MANIFESTIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
+                    docker buildx imagetools create --prefer-index=false -t ${MANIFESTIMAGE}:arm64v8-${SEMVER} ${CACHEIMAGE}:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER} || \
+                      { if [[ "${MANIFESTIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
                   fi
                 done
                 for MANIFESTIMAGE in "${IMAGE}" "${GITLABIMAGE}" "${GITHUBIMAGE}" "${QUAYIMAGE}"; do
-                  docker buildx imagetools create -t ${MANIFESTIMAGE}:alpine321 ${MANIFESTIMAGE}:amd64-alpine321 ${MANIFESTIMAGE}:arm64v8-alpine321
-                  docker buildx imagetools create -t ${MANIFESTIMAGE}:${META_TAG} ${MANIFESTIMAGE}:amd64-${META_TAG} ${MANIFESTIMAGE}:arm64v8-${META_TAG}
-
-                  docker buildx imagetools create -t ${MANIFESTIMAGE}:${EXT_RELEASE_TAG} ${MANIFESTIMAGE}:amd64-${EXT_RELEASE_TAG} ${MANIFESTIMAGE}:arm64v8-${EXT_RELEASE_TAG}
+                  docker buildx imagetools create -t ${MANIFESTIMAGE}:alpine321 ${MANIFESTIMAGE}:amd64-alpine321 ${MANIFESTIMAGE}:arm64v8-alpine321 || \
+                    { if [[ "${MANIFESTIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
+                  docker buildx imagetools create -t ${MANIFESTIMAGE}:${META_TAG} ${MANIFESTIMAGE}:amd64-${META_TAG} ${MANIFESTIMAGE}:arm64v8-${META_TAG} || \
+                    { if [[ "${MANIFESTIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
+                  docker buildx imagetools create -t ${MANIFESTIMAGE}:${EXT_RELEASE_TAG} ${MANIFESTIMAGE}:amd64-${EXT_RELEASE_TAG} ${MANIFESTIMAGE}:arm64v8-${EXT_RELEASE_TAG} || \
+                    { if [[ "${MANIFESTIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
                   if [ -n "${SEMVER}" ]; then
-                    docker buildx imagetools create -t ${MANIFESTIMAGE}:${SEMVER} ${MANIFESTIMAGE}:amd64-${SEMVER} ${MANIFESTIMAGE}:arm64v8-${SEMVER}
+                    docker buildx imagetools create -t ${MANIFESTIMAGE}:${SEMVER} ${MANIFESTIMAGE}:amd64-${SEMVER} ${MANIFESTIMAGE}:arm64v8-${SEMVER} || \
+                      { if [[ "${MANIFESTIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
                   fi
                 done
               '''
@@ -812,23 +858,41 @@ pipeline {
         environment name: 'EXIT_STATUS', value: ''
       }
       steps {
-        echo "Pushing New tag for current commit ${META_TAG}"
-        sh '''curl -H "Authorization: token ${GITHUB_TOKEN}" -X POST https://api.github.com/repos/${LS_USER}/${LS_REPO}/git/tags \
-        -d '{"tag":"'${META_TAG}'",\
-             "object": "'${COMMIT_SHA}'",\
-             "message": "Tagging Release '${EXT_RELEASE_CLEAN}'-ls'${LS_TAG_NUMBER}' to alpine321",\
-             "type": "commit",\
-             "tagger": {"name": "LinuxServer-CI","email": "ci@linuxserver.io","date": "'${GITHUB_DATE}'"}}' '''
-        echo "Pushing New release for Tag"
         sh '''#! /bin/bash
+              echo "Auto-generating release notes"
+              if [ "$(git tag --points-at HEAD)" != "" ]; then
+                echo "Existing tag points to current commit, suggesting no new LS changes"
+                AUTO_RELEASE_NOTES="No changes"
+              else
+                AUTO_RELEASE_NOTES=$(curl -fsL -H "Authorization: token ${GITHUB_TOKEN}" -H "Accept: application/vnd.github+json" -X POST https://api.github.com/repos/${LS_USER}/${LS_REPO}/releases/generate-notes  \
+                  -d '{"tag_name":"'${META_TAG}'",\
+                      "target_commitish": "alpine321"}' \
+                  | jq -r '.body' | sed 's|## What.s Changed||')
+              fi
+              echo "Pushing New tag for current commit ${META_TAG}"
+              curl -H "Authorization: token ${GITHUB_TOKEN}" -X POST https://api.github.com/repos/${LS_USER}/${LS_REPO}/git/tags \
+                -d '{"tag":"'${META_TAG}'",\
+                  "object": "'${COMMIT_SHA}'",\
+                  "message": "Tagging Release '${EXT_RELEASE_CLEAN}'-ls'${LS_TAG_NUMBER}' to alpine321",\
+                  "type": "commit",\
+                  "tagger": {"name": "LinuxServer-CI","email": "ci@linuxserver.io","date": "'${GITHUB_DATE}'"}}'
+              echo "Pushing New release for Tag"
               echo "Updating to ${EXT_RELEASE_CLEAN}" > releasebody.json
-              echo '{"tag_name":"'${META_TAG}'",\
-                     "target_commitish": "alpine321",\
-                     "name": "'${META_TAG}'",\
-                     "body": "**CI Report:**\\n\\n'${CI_URL:-N/A}'\\n\\n**LinuxServer Changes:**\\n\\n'${LS_RELEASE_NOTES}'\\n\\n**Remote Changes:**\\n\\n' > start
-              printf '","draft": false,"prerelease": false}' >> releasebody.json
-              paste -d'\\0' start releasebody.json > releasebody.json.done
-              curl -H "Authorization: token ${GITHUB_TOKEN}" -X POST https://api.github.com/repos/${LS_USER}/${LS_REPO}/releases -d @releasebody.json.done'''
+              jq -n \
+                --arg tag_name "$META_TAG" \
+                --arg target_commitish "alpine321" \
+                --arg ci_url "${CI_URL:-N/A}" \
+                --arg ls_notes "$AUTO_RELEASE_NOTES" \
+                --arg remote_notes "$(cat releasebody.json)" \
+                '{
+                  "tag_name": $tag_name,
+                  "target_commitish": $target_commitish,
+                  "name": $tag_name,
+                  "body": ("**CI Report:**\\n\\n" + $ci_url + "\\n\\n**LinuxServer Changes:**\\n\\n" + $ls_notes + "\\n\\n**Remote Changes:**\\n\\n" + $remote_notes),
+                  "draft": false,
+                  "prerelease": true                }' > releasebody.json.done
+              curl -H "Authorization: token ${GITHUB_TOKEN}" -X POST https://api.github.com/repos/${LS_USER}/${LS_REPO}/releases -d @releasebody.json.done
+        '''
       }
     }
     // Add protection to the release branch
@@ -1003,12 +1067,22 @@ EOF
     }
     cleanup {
       sh '''#! /bin/bash
-            echo "Performing docker system prune!!"
-            containers=$(docker ps -aq)
+            echo "Pruning builder!!"
+            docker builder prune -f --builder container || :
+            containers=$(docker ps -q)
             if [[ -n "${containers}" ]]; then
-              docker stop ${containers}
+              BUILDX_CONTAINER_ID=$(docker ps -qf 'name=buildx_buildkit')
+              for container in ${containers}; do
+                if [[ "${container}" == "${BUILDX_CONTAINER_ID}" ]]; then
+                  echo "skipping buildx container in docker stop"
+                else
+                  echo "Stopping container ${container}"
+                  docker stop ${container}
+                fi
+              done
             fi
-            docker system prune -af --volumes || :
+            docker system prune -f --volumes || :
+            docker image prune -af || :
          '''
       cleanWs()
     }

jenkins-vars.yml

@@ -4,7 +4,7 @@
 project_name: docker-python
 external_type: na
 custom_version_command: "curl -sX GET https://api.github.com/repos/python/cpython/tags | jq -r '.[] | select(.name | contains(\"rc\") or contains(\"a\") or contains(\"b\") | not) | .name' | sed 's|^v||g' | sort -rV | head -1"
-release_type: stable
+release_type: prerelease
 release_tag: alpine321
 ls_branch: alpine321
 skip_package_check: true

readme-vars.yml

@@ -2,6 +2,7 @@
 
 # project information
 project_name: python
+project_categories: "Internal"
 full_custom_readme: |
   [![linuxserver.io](https://raw.githubusercontent.com/linuxserver/docker-templates/master/linuxserver.io/img/linuxserver_medium.png)](https://linuxserver.io)