Bot Updating Package Versions

Download the same chromedriver version as the installed chrome version.

.github/CONTRIBUTING.md

@@ -24,7 +24,7 @@
 ## Readme
 
 If you would like to change our readme, please __**do not**__ directly edit the readme, as it is auto-generated on each commit.
-Instead edit the [readme-vars.yml](https://github.com/linuxserver/docker-ci/edit/develop/readme-vars.yml).
+Instead edit the [readme-vars.yml](https://github.com/linuxserver/docker-ci/edit/master/readme-vars.yml).
 
 These variables are used in a template for our [Jenkins Builder](https://github.com/linuxserver/docker-jenkins-builder) as part of an ansible play.
 Most of these variables are also carried over to [docs.linuxserver.io](https://docs.linuxserver.io)
@@ -115,7 +115,7 @@ Once registered you can define the dockerfile to use with `-f Dockerfile.aarch64
 
 ## Update the changelog
 
-If you are modifying the Dockerfiles or any of the startup scripts in [root](https://github.com/linuxserver/docker-ci/tree/develop/root), add an entry to the changelog
+If you are modifying the Dockerfiles or any of the startup scripts in [root](https://github.com/linuxserver/docker-ci/tree/master/root), add an entry to the changelog
 
 ```yml
 changelogs:

.github/PULL_REQUEST_TEMPLATE.md

@@ -21,7 +21,7 @@
 
 ------------------------------
 
- - [ ] I have read the [contributing](https://github.com/linuxserver/docker-ci/blob/develop/.github/CONTRIBUTING.md) guideline and understand that I have made the correct modifications
+ - [ ] I have read the [contributing](https://github.com/linuxserver/docker-ci/blob/master/.github/CONTRIBUTING.md) guideline and understand that I have made the correct modifications
 
 ------------------------------
 

.github/workflows/external_trigger.yml

@@ -7,13 +7,13 @@ permissions:
   contents: read
 
 jobs:
-  external-trigger-develop:
+  external-trigger-master:
     runs-on: ubuntu-latest
     steps:
       - uses: actions/checkout@v4.1.1
 
       - name: External Trigger
-        if: github.ref == 'refs/heads/develop'
+        if: github.ref == 'refs/heads/master'
         env:
           SKIP_EXTERNAL_TRIGGER: ${{ vars.SKIP_EXTERNAL_TRIGGER }}
         run: |
@@ -21,7 +21,7 @@ jobs:
           echo "Type is \`os\`" >> $GITHUB_STEP_SUMMARY
           echo "No external release, exiting" >> $GITHUB_STEP_SUMMARY
           exit 0
-          if grep -q "^ci_develop_${EXT_RELEASE}" <<< "${SKIP_EXTERNAL_TRIGGER}"; then
+          if grep -q "^ci_master_${EXT_RELEASE}" <<< "${SKIP_EXTERNAL_TRIGGER}"; then
             echo "> [!WARNING]" >> $GITHUB_STEP_SUMMARY
             echo "> Github organizational variable \`SKIP_EXTERNAL_TRIGGER\` matches current external release; skipping trigger." >> $GITHUB_STEP_SUMMARY
             exit 0

.github/workflows/greetings.yml

@@ -15,5 +15,5 @@ jobs:
     - uses: actions/first-interaction@v1
       with:
         issue-message: 'Thanks for opening your first issue here! Be sure to follow the relevant issue templates, or risk having this issue marked as invalid.'
-        pr-message: 'Thanks for opening this pull request! Be sure to follow the [pull request template](https://github.com/linuxserver/docker-ci/blob/develop/.github/PULL_REQUEST_TEMPLATE.md)!'
+        pr-message: 'Thanks for opening this pull request! Be sure to follow the [pull request template](https://github.com/linuxserver/docker-ci/blob/master/.github/PULL_REQUEST_TEMPLATE.md)!'
         repo-token: ${{ secrets.GITHUB_TOKEN }}

Dockerfile

@@ -23,14 +23,20 @@ RUN \
   apt-get update && \
   apt-get install -y --no-install-recommends \
     docker-ce \
+    docker-buildx-plugin \
     google-chrome-stable \
     python3-venv \
     unzip \
     xserver-xephyr \
     xvfb && \
   echo "**** install chrome driver ****" && \
-  CHROME_RELEASE=$(curl -sLk https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_STABLE) && \
+  CHROME_VERSION=$(google-chrome --version | awk '{print $3}') && \
-  echo "Retrieving Chrome driver version ${CHROME_RELEASE}" && \
+  CHROME_MAJOR=${CHROME_VERSION%%.*} && \
+  CHROME_RELEASE=$(curl -fsSLk "https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_${CHROME_MAJOR}" || true) && \
+  if [ -z "${CHROME_RELEASE}" ]; then \
+    CHROME_RELEASE=$(curl -fsSLk https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_STABLE); \
+  fi && \
+  echo "Installed Chrome ${CHROME_VERSION}; retrieving ChromeDriver ${CHROME_RELEASE}" && \
   curl -sk -o \
     /tmp/chrome.zip -L \
     "https://storage.googleapis.com/chrome-for-testing-public/${CHROME_RELEASE}/linux64/chromedriver-linux64.zip" && \

Jenkinsfile

@@ -76,7 +76,7 @@ pipeline {
         script{
           env.EXIT_STATUS = ''
           env.LS_RELEASE = sh(
-            script: '''docker run --rm quay.io/skopeo/stable:v1 inspect docker://ghcr.io/${LS_USER}/${CONTAINER_NAME}:develop 2>/dev/null | jq -r '.Labels.build_version' | awk '{print $3}' | grep '\\-ls' || : ''',
+            script: '''docker run --rm quay.io/skopeo/stable:v1 inspect docker://ghcr.io/${LS_USER}/${CONTAINER_NAME}:latest 2>/dev/null | jq -r '.Labels.build_version' | awk '{print $3}' | grep '\\-ls' || : ''',
             returnStdout: true).trim()
           env.LS_RELEASE_NOTES = sh(
             script: '''cat readme-vars.yml | awk -F \\" '/date: "[0-9][0-9].[0-9][0-9].[0-9][0-9]:/ {print $4;exit;}' | sed -E ':a;N;$!ba;s/\\r{0,1}\\n/\\\\n/g' ''',
@@ -109,7 +109,7 @@ pipeline {
         script{
           env.LS_TAG_NUMBER = sh(
             script: '''#! /bin/bash
-                       tagsha=$(git rev-list -n 1 develop-${LS_RELEASE} 2>/dev/null)
+                       tagsha=$(git rev-list -n 1 ${LS_RELEASE} 2>/dev/null)
                        if [ "${tagsha}" == "${COMMIT_SHA}" ]; then
                          echo ${LS_RELEASE_NUMBER}
                        elif [ -z "${GIT_COMMIT}" ]; then
@@ -185,10 +185,10 @@ pipeline {
         }
       }
     }
-    // If this is a develop build use live docker endpoints
+    // If this is a master build use live docker endpoints
     stage("Set ENV live build"){
       when {
-        branch "develop"
+        branch "master"
         environment name: 'CHANGE_ID', value: ''
       }
       steps {
@@ -198,21 +198,22 @@ pipeline {
           env.GITLABIMAGE = 'registry.gitlab.com/linuxserver.io/' + env.LS_REPO + '/' + env.CONTAINER_NAME
           env.QUAYIMAGE = 'quay.io/linuxserver.io/' + env.CONTAINER_NAME
           if (env.MULTIARCH == 'true') {
-            env.CI_TAGS = 'amd64-develop-' + env.EXT_RELEASE_CLEAN + '-ls' + env.LS_TAG_NUMBER + '|arm64v8-develop-' + env.EXT_RELEASE_CLEAN + '-ls' + env.LS_TAG_NUMBER
+            env.CI_TAGS = 'amd64-' + env.EXT_RELEASE_CLEAN + '-ls' + env.LS_TAG_NUMBER + '|arm64v8-' + env.EXT_RELEASE_CLEAN + '-ls' + env.LS_TAG_NUMBER
           } else {
-            env.CI_TAGS = 'develop-' + env.EXT_RELEASE_CLEAN + '-ls' + env.LS_TAG_NUMBER
+            env.CI_TAGS = env.EXT_RELEASE_CLEAN + '-ls' + env.LS_TAG_NUMBER
           }
           env.VERSION_TAG = env.EXT_RELEASE_CLEAN + '-ls' + env.LS_TAG_NUMBER
-          env.META_TAG = 'develop-' + env.EXT_RELEASE_CLEAN + '-ls' + env.LS_TAG_NUMBER
+          env.META_TAG = env.EXT_RELEASE_CLEAN + '-ls' + env.LS_TAG_NUMBER
-          env.EXT_RELEASE_TAG = 'develop-version-' + env.EXT_RELEASE_CLEAN
+          env.EXT_RELEASE_TAG = 'version-' + env.EXT_RELEASE_CLEAN
           env.BUILDCACHE = 'docker.io/lsiodev/buildcache,registry.gitlab.com/linuxserver.io/docker-jenkins-builder/lsiodev-buildcache,ghcr.io/linuxserver/lsiodev-buildcache,quay.io/linuxserver.io/lsiodev-buildcache'
+          env.CITEST_IMAGETAG = 'latest'
         }
       }
     }
     // If this is a dev build use dev docker endpoints
     stage("Set ENV dev build"){
       when {
-        not {branch "develop"}
+        not {branch "master"}
         environment name: 'CHANGE_ID', value: ''
       }
       steps {
@@ -222,15 +223,16 @@ pipeline {
           env.GITLABIMAGE = 'registry.gitlab.com/linuxserver.io/' + env.LS_REPO + '/lsiodev-' + env.CONTAINER_NAME
           env.QUAYIMAGE = 'quay.io/linuxserver.io/lsiodev-' + env.CONTAINER_NAME
           if (env.MULTIARCH == 'true') {
-            env.CI_TAGS = 'amd64-develop-' + env.EXT_RELEASE_CLEAN + '-pkg-' + env.PACKAGE_TAG + '-dev-' + env.COMMIT_SHA + '|arm64v8-develop-' + env.EXT_RELEASE_CLEAN + '-pkg-' + env.PACKAGE_TAG + '-dev-' + env.COMMIT_SHA
+            env.CI_TAGS = 'amd64-' + env.EXT_RELEASE_CLEAN + '-pkg-' + env.PACKAGE_TAG + '-dev-' + env.COMMIT_SHA + '|arm64v8-' + env.EXT_RELEASE_CLEAN + '-pkg-' + env.PACKAGE_TAG + '-dev-' + env.COMMIT_SHA
           } else {
-            env.CI_TAGS = 'develop-' + env.EXT_RELEASE_CLEAN + '-pkg-' + env.PACKAGE_TAG + '-dev-' + env.COMMIT_SHA
+            env.CI_TAGS = env.EXT_RELEASE_CLEAN + '-pkg-' + env.PACKAGE_TAG + '-dev-' + env.COMMIT_SHA
           }
           env.VERSION_TAG = env.EXT_RELEASE_CLEAN + '-pkg-' + env.PACKAGE_TAG + '-dev-' + env.COMMIT_SHA
-          env.META_TAG = 'develop-' + env.EXT_RELEASE_CLEAN + '-pkg-' + env.PACKAGE_TAG + '-dev-' + env.COMMIT_SHA
+          env.META_TAG = env.EXT_RELEASE_CLEAN + '-pkg-' + env.PACKAGE_TAG + '-dev-' + env.COMMIT_SHA
-          env.EXT_RELEASE_TAG = 'develop-version-' + env.EXT_RELEASE_CLEAN
+          env.EXT_RELEASE_TAG = 'version-' + env.EXT_RELEASE_CLEAN
           env.DOCKERHUB_LINK = 'https://hub.docker.com/r/' + env.DEV_DOCKERHUB_IMAGE + '/tags/'
           env.BUILDCACHE = 'docker.io/lsiodev/buildcache,registry.gitlab.com/linuxserver.io/docker-jenkins-builder/lsiodev-buildcache,ghcr.io/linuxserver/lsiodev-buildcache,quay.io/linuxserver.io/lsiodev-buildcache'
+          env.CITEST_IMAGETAG = 'develop'
         }
       }
     }
@@ -246,16 +248,17 @@ pipeline {
           env.GITLABIMAGE = 'registry.gitlab.com/linuxserver.io/' + env.LS_REPO + '/lspipepr-' + env.CONTAINER_NAME
           env.QUAYIMAGE = 'quay.io/linuxserver.io/lspipepr-' + env.CONTAINER_NAME
           if (env.MULTIARCH == 'true') {
-            env.CI_TAGS = 'amd64-develop-' + env.EXT_RELEASE_CLEAN + '-pkg-' + env.PACKAGE_TAG + '-dev-' + env.COMMIT_SHA + '-pr-' + env.PULL_REQUEST + '|arm64v8-develop-' + env.EXT_RELEASE_CLEAN + '-pkg-' + env.PACKAGE_TAG + '-dev-' + env.COMMIT_SHA + '-pr-' + env.PULL_REQUEST
+            env.CI_TAGS = 'amd64-' + env.EXT_RELEASE_CLEAN + '-pkg-' + env.PACKAGE_TAG + '-dev-' + env.COMMIT_SHA + '-pr-' + env.PULL_REQUEST + '|arm64v8-' + env.EXT_RELEASE_CLEAN + '-pkg-' + env.PACKAGE_TAG + '-dev-' + env.COMMIT_SHA + '-pr-' + env.PULL_REQUEST
           } else {
-            env.CI_TAGS = 'develop-' + env.EXT_RELEASE_CLEAN + '-pkg-' + env.PACKAGE_TAG + '-dev-' + env.COMMIT_SHA + '-pr-' + env.PULL_REQUEST
+            env.CI_TAGS = env.EXT_RELEASE_CLEAN + '-pkg-' + env.PACKAGE_TAG + '-dev-' + env.COMMIT_SHA + '-pr-' + env.PULL_REQUEST
           }
           env.VERSION_TAG = env.EXT_RELEASE_CLEAN + '-pkg-' + env.PACKAGE_TAG + '-dev-' + env.COMMIT_SHA + '-pr-' + env.PULL_REQUEST
-          env.META_TAG = 'develop-' + env.EXT_RELEASE_CLEAN + '-pkg-' + env.PACKAGE_TAG + '-dev-' + env.COMMIT_SHA + '-pr-' + env.PULL_REQUEST
+          env.META_TAG = env.EXT_RELEASE_CLEAN + '-pkg-' + env.PACKAGE_TAG + '-dev-' + env.COMMIT_SHA + '-pr-' + env.PULL_REQUEST
-          env.EXT_RELEASE_TAG = 'develop-version-' + env.EXT_RELEASE_CLEAN
+          env.EXT_RELEASE_TAG = 'version-' + env.EXT_RELEASE_CLEAN
           env.CODE_URL = 'https://github.com/' + env.LS_USER + '/' + env.LS_REPO + '/pull/' + env.PULL_REQUEST
           env.DOCKERHUB_LINK = 'https://hub.docker.com/r/' + env.PR_DOCKERHUB_IMAGE + '/tags/'
           env.BUILDCACHE = 'docker.io/lsiodev/buildcache,registry.gitlab.com/linuxserver.io/docker-jenkins-builder/lsiodev-buildcache,ghcr.io/linuxserver/lsiodev-buildcache,quay.io/linuxserver.io/lsiodev-buildcache'
+          env.CITEST_IMAGETAG = 'develop'
         }
       }
     }
@@ -278,7 +281,7 @@ pipeline {
                   -v ${WORKSPACE}:/mnt \
                   -e AWS_ACCESS_KEY_ID=\"${S3_KEY}\" \
                   -e AWS_SECRET_ACCESS_KEY=\"${S3_SECRET}\" \
-                  ghcr.io/linuxserver/baseimage-alpine:3.20 s6-envdir -fn -- /var/run/s6/container_environment /bin/bash -c "\
+                  ghcr.io/linuxserver/baseimage-alpine:3.23 s6-envdir -fn -- /var/run/s6/container_environment /bin/bash -c "\
                     apk add --no-cache python3 && \
                     python3 -m venv /lsiopy && \
                     pip install --no-cache-dir -U pip && \
@@ -290,7 +293,7 @@ pipeline {
     // Use helper containers to render templated files
     stage('Update-Templates') {
       when {
-        branch "develop"
+        branch "master"
         environment name: 'CHANGE_ID', value: ''
         expression {
           env.CONTAINER_NAME != null
@@ -302,24 +305,24 @@ pipeline {
               TEMPDIR=$(mktemp -d)
               docker pull ghcr.io/linuxserver/jenkins-builder:latest
               # Cloned repo paths for templating:
-              # ${TEMPDIR}/docker-${CONTAINER_NAME}: Cloned branch develop of ${LS_USER}/${LS_REPO} for running the jenkins builder on
+              # ${TEMPDIR}/docker-${CONTAINER_NAME}: Cloned branch master of ${LS_USER}/${LS_REPO} for running the jenkins builder on
-              # ${TEMPDIR}/repo/${LS_REPO}: Cloned branch develop of ${LS_USER}/${LS_REPO} for commiting various templated file changes and pushing back to Github
+              # ${TEMPDIR}/repo/${LS_REPO}: Cloned branch master of ${LS_USER}/${LS_REPO} for commiting various templated file changes and pushing back to Github
               # ${TEMPDIR}/docs/docker-documentation: Cloned docs repo for pushing docs updates to Github
               # ${TEMPDIR}/unraid/docker-templates: Cloned docker-templates repo to check for logos
               # ${TEMPDIR}/unraid/templates: Cloned templates repo for commiting unraid template changes and pushing back to Github
-              git clone --branch develop --depth 1 https://github.com/${LS_USER}/${LS_REPO}.git ${TEMPDIR}/docker-${CONTAINER_NAME}
+              git clone --branch master --depth 1 https://github.com/${LS_USER}/${LS_REPO}.git ${TEMPDIR}/docker-${CONTAINER_NAME}
               docker run --rm -v ${TEMPDIR}/docker-${CONTAINER_NAME}:/tmp -e LOCAL=true -e PUID=$(id -u) -e PGID=$(id -g) ghcr.io/linuxserver/jenkins-builder:latest 
               echo "Starting Stage 1 - Jenkinsfile update"
               if [[ "$(md5sum Jenkinsfile | awk '{ print $1 }')" != "$(md5sum ${TEMPDIR}/docker-${CONTAINER_NAME}/Jenkinsfile | awk '{ print $1 }')" ]]; then
                 mkdir -p ${TEMPDIR}/repo
                 git clone https://github.com/${LS_USER}/${LS_REPO}.git ${TEMPDIR}/repo/${LS_REPO}
                 cd ${TEMPDIR}/repo/${LS_REPO}
-                git checkout -f develop
+                git checkout -f master
                 cp ${TEMPDIR}/docker-${CONTAINER_NAME}/Jenkinsfile ${TEMPDIR}/repo/${LS_REPO}/
                 git add Jenkinsfile
                 git commit -m 'Bot Updating Templated Files'
-                git pull https://LinuxServer-CI:${GITHUB_TOKEN}@github.com/${LS_USER}/${LS_REPO}.git develop
+                git pull https://LinuxServer-CI:${GITHUB_TOKEN}@github.com/${LS_USER}/${LS_REPO}.git master
-                git push https://LinuxServer-CI:${GITHUB_TOKEN}@github.com/${LS_USER}/${LS_REPO}.git develop
+                git push https://LinuxServer-CI:${GITHUB_TOKEN}@github.com/${LS_USER}/${LS_REPO}.git master
                 echo "true" > /tmp/${COMMIT_SHA}-${BUILD_NUMBER}
                 echo "Updating Jenkinsfile and exiting build, new one will trigger based on commit"
                 rm -Rf ${TEMPDIR}
@@ -338,13 +341,13 @@ pipeline {
                 mkdir -p ${TEMPDIR}/repo
                 git clone https://github.com/${LS_USER}/${LS_REPO}.git ${TEMPDIR}/repo/${LS_REPO}
                 cd ${TEMPDIR}/repo/${LS_REPO}
-                git checkout -f develop
+                git checkout -f master
                 for i in ${TEMPLATES_TO_DELETE}; do
                   git rm "${i}"
                 done
                 git commit -m 'Bot Updating Templated Files'
-                git pull https://LinuxServer-CI:${GITHUB_TOKEN}@github.com/${LS_USER}/${LS_REPO}.git develop
+                git pull https://LinuxServer-CI:${GITHUB_TOKEN}@github.com/${LS_USER}/${LS_REPO}.git master
-                git push https://LinuxServer-CI:${GITHUB_TOKEN}@github.com/${LS_USER}/${LS_REPO}.git develop
+                git push https://LinuxServer-CI:${GITHUB_TOKEN}@github.com/${LS_USER}/${LS_REPO}.git master
                 echo "true" > /tmp/${COMMIT_SHA}-${BUILD_NUMBER}
                 echo "Deleting old/deprecated templates and exiting build, new one will trigger based on commit"
                 rm -Rf ${TEMPDIR}
@@ -360,7 +363,7 @@ pipeline {
                 mkdir -p ${TEMPDIR}/repo
                 git clone https://github.com/${LS_USER}/${LS_REPO}.git ${TEMPDIR}/repo/${LS_REPO}
                 cd ${TEMPDIR}/repo/${LS_REPO}
-                git checkout -f develop
+                git checkout -f master
                 cd ${TEMPDIR}/docker-${CONTAINER_NAME}
                 mkdir -p ${TEMPDIR}/repo/${LS_REPO}/.github/workflows
                 mkdir -p ${TEMPDIR}/repo/${LS_REPO}/.github/ISSUE_TEMPLATE
@@ -373,8 +376,8 @@ pipeline {
                 fi
                 git add readme-vars.yml ${TEMPLATED_FILES}
                 git commit -m 'Bot Updating Templated Files'
-                git pull https://LinuxServer-CI:${GITHUB_TOKEN}@github.com/${LS_USER}/${LS_REPO}.git develop
+                git pull https://LinuxServer-CI:${GITHUB_TOKEN}@github.com/${LS_USER}/${LS_REPO}.git master
-                git push https://LinuxServer-CI:${GITHUB_TOKEN}@github.com/${LS_USER}/${LS_REPO}.git develop
+                git push https://LinuxServer-CI:${GITHUB_TOKEN}@github.com/${LS_USER}/${LS_REPO}.git master
                 echo "true" > /tmp/${COMMIT_SHA}-${BUILD_NUMBER}
                 echo "Updating templates and exiting build, new one will trigger based on commit"
                 rm -Rf ${TEMPDIR}
@@ -441,7 +444,7 @@ pipeline {
     // Exit the build if the Templated files were just updated
     stage('Template-exit') {
       when {
-        branch "develop"
+        branch "master"
         environment name: 'CHANGE_ID', value: ''
         environment name: 'FILES_UPDATED', value: 'true'
         expression {
@@ -454,10 +457,10 @@ pipeline {
         }
       }
     }
-    // If this is a develop build check the S6 service file perms
+    // If this is a master build check the S6 service file perms
     stage("Check S6 Service file Permissions"){
       when {
-        branch "develop"
+        branch "master"
         environment name: 'CHANGE_ID', value: ''
         environment name: 'EXIT_STATUS', value: ''
       }
@@ -550,13 +553,16 @@ pipeline {
                     echo $GITHUB_TOKEN | docker login ghcr.io -u LinuxServer-CI --password-stdin
                     echo $GITLAB_TOKEN | docker login registry.gitlab.com -u LinuxServer.io --password-stdin
                     echo $QUAYPASS | docker login quay.io -u $QUAYUSER --password-stdin
+
                     if [[ "${PACKAGE_CHECK}" != "true" ]]; then
+                      declare -A pids
                       IFS=',' read -ra CACHE <<< "$BUILDCACHE"
                       for i in "${CACHE[@]}"; do
                         docker push ${i}:amd64-${COMMIT_SHA}-${BUILD_NUMBER} &
+                        pids[$!]="$i"
                       done
-                      for p in $(jobs -p); do
+                      for p in "${!pids[@]}"; do
-                        wait "$p" || { echo "job $p failed" >&2; exit 1; }
+                        wait "$p" || { [[ "${pids[$p]}" != *"quay.io"* ]] && exit 1; }
                       done
                     fi
                 '''
@@ -616,13 +622,16 @@ pipeline {
                         echo $GITHUB_TOKEN | docker login ghcr.io -u LinuxServer-CI --password-stdin
                         echo $GITLAB_TOKEN | docker login registry.gitlab.com -u LinuxServer.io --password-stdin
                         echo $QUAYPASS | docker login quay.io -u $QUAYUSER --password-stdin
+
                         if [[ "${PACKAGE_CHECK}" != "true" ]]; then
+                          declare -A pids
                           IFS=',' read -ra CACHE <<< "$BUILDCACHE"
                           for i in "${CACHE[@]}"; do
                             docker push ${i}:amd64-${COMMIT_SHA}-${BUILD_NUMBER} &
+                            pids[$!]="$i"
                           done
-                          for p in $(jobs -p); do
+                          for p in "${!pids[@]}"; do
-                            wait "$p" || { echo "job $p failed" >&2; exit 1; }
+                            wait "$p" || { [[ "${pids[$p]}" != *"quay.io"* ]] && exit 1; }
                           done
                         fi
                     '''
@@ -676,12 +685,14 @@ pipeline {
                         echo $GITLAB_TOKEN | docker login registry.gitlab.com -u LinuxServer.io --password-stdin
                         echo $QUAYPASS | docker login quay.io -u $QUAYUSER --password-stdin
                         if [[ "${PACKAGE_CHECK}" != "true" ]]; then
+                          declare -A pids
                           IFS=',' read -ra CACHE <<< "$BUILDCACHE"
                           for i in "${CACHE[@]}"; do
                             docker push ${i}:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER} &
+                            pids[$!]="$i"
                           done
-                          for p in $(jobs -p); do
+                          for p in "${!pids[@]}"; do
-                            wait "$p" || { echo "job $p failed" >&2; exit 1; }
+                            wait "$p" || { [[ "${pids[$p]}" != *"quay.io"* ]] && exit 1; }
                           done
                         fi
                     '''
@@ -702,7 +713,7 @@ pipeline {
     // Take the image we just built and dump package versions for comparison
     stage('Update-packages') {
       when {
-        branch "develop"
+        branch "master"
         environment name: 'CHANGE_ID', value: ''
         environment name: 'EXIT_STATUS', value: ''
       }
@@ -725,14 +736,14 @@ pipeline {
               echo "Package tag sha from current packages in buit container is ${NEW_PACKAGE_TAG} comparing to old ${PACKAGE_TAG} from github"
               if [ "${NEW_PACKAGE_TAG}" != "${PACKAGE_TAG}" ]; then
                 git clone https://github.com/${LS_USER}/${LS_REPO}.git ${TEMPDIR}/${LS_REPO}
-                git --git-dir ${TEMPDIR}/${LS_REPO}/.git checkout -f develop
+                git --git-dir ${TEMPDIR}/${LS_REPO}/.git checkout -f master
                 cp ${TEMPDIR}/package_versions.txt ${TEMPDIR}/${LS_REPO}/
                 cd ${TEMPDIR}/${LS_REPO}/
                 wait
                 git add package_versions.txt
                 git commit -m 'Bot Updating Package Versions'
-                git pull https://LinuxServer-CI:${GITHUB_TOKEN}@github.com/${LS_USER}/${LS_REPO}.git develop
+                git pull https://LinuxServer-CI:${GITHUB_TOKEN}@github.com/${LS_USER}/${LS_REPO}.git master
-                git push https://LinuxServer-CI:${GITHUB_TOKEN}@github.com/${LS_USER}/${LS_REPO}.git develop
+                git push https://LinuxServer-CI:${GITHUB_TOKEN}@github.com/${LS_USER}/${LS_REPO}.git master
                 echo "true" > /tmp/packages-${COMMIT_SHA}-${BUILD_NUMBER}
                 echo "Package tag updated, stopping build process"
               else
@@ -750,7 +761,7 @@ pipeline {
     // Exit the build if the package file was just updated
     stage('PACKAGE-exit') {
       when {
-        branch "develop"
+        branch "master"
         environment name: 'CHANGE_ID', value: ''
         environment name: 'PACKAGE_UPDATED', value: 'true'
         environment name: 'EXIT_STATUS', value: ''
@@ -764,7 +775,7 @@ pipeline {
     // Exit the build if this is just a package check and there are no changes to push
     stage('PACKAGECHECK-exit') {
       when {
-        branch "develop"
+        branch "master"
         environment name: 'CHANGE_ID', value: ''
         environment name: 'PACKAGE_UPDATED', value: 'false'
         environment name: 'EXIT_STATUS', value: ''
@@ -806,7 +817,7 @@ pipeline {
                     CI_DOCKERENV="LSIO_FIRST_PARTY=true"
                   fi
                 fi
-                docker pull ghcr.io/linuxserver/ci:latest
+                docker pull ghcr.io/linuxserver/ci:${CITEST_IMAGETAG}
                 if [ "${MULTIARCH}" == "true" ]; then
                   docker pull ghcr.io/linuxserver/lsiodev-buildcache:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER} --platform=arm64
                   docker tag ghcr.io/linuxserver/lsiodev-buildcache:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER} ${IMAGE}:arm64v8-${META_TAG}
@@ -818,7 +829,7 @@ pipeline {
                 -e DOCKER_LOGS_TIMEOUT=\"${CI_DELAY}\" \
                 -e TAGS=\"${CI_TAGS}\" \
                 -e META_TAG=\"${META_TAG}\" \
-                -e RELEASE_TAG=\"develop\" \
+                -e RELEASE_TAG=\"latest\" \
                 -e PORT=\"${CI_PORT}\" \
                 -e SSL=\"${CI_SSL}\" \
                 -e BASE=\"${DIST_IMAGE}\" \
@@ -830,7 +841,9 @@ pipeline {
                 -e WEB_PATH=\"${CI_WEBPATH}\" \
                 -e NODE_NAME=\"${NODE_NAME}\" \
                 -e SYFT_IMAGE_TAG=\"${CI_SYFT_IMAGE_TAG:-${SYFT_IMAGE_TAG}}\" \
-                -t ghcr.io/linuxserver/ci:latest \
+                -e COMMIT_SHA=\"${COMMIT_SHA}\" \
+                -e BUILD_NUMBER=\"${BUILD_NUMBER}\" \
+                -t ghcr.io/linuxserver/ci:${CITEST_IMAGETAG} \
                 python3 test_build.py'''
         }
       }
@@ -856,9 +869,11 @@ pipeline {
                           CACHEIMAGE=${i}
                       fi
                   done
-                  docker buildx imagetools create --prefer-index=false -t ${PUSHIMAGE}:${META_TAG} -t ${PUSHIMAGE}:develop -t ${PUSHIMAGE}:${EXT_RELEASE_TAG} ${CACHEIMAGE}:amd64-${COMMIT_SHA}-${BUILD_NUMBER}
+                  docker buildx imagetools create --prefer-index=false -t ${PUSHIMAGE}:${META_TAG} -t ${PUSHIMAGE}:latest -t ${PUSHIMAGE}:${EXT_RELEASE_TAG} ${CACHEIMAGE}:amd64-${COMMIT_SHA}-${BUILD_NUMBER} || \
+                    { if [[ "${PUSHIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
                   if [ -n "${SEMVER}" ]; then
-                    docker buildx imagetools create --prefer-index=false -t ${PUSHIMAGE}:${SEMVER} ${CACHEIMAGE}:amd64-${COMMIT_SHA}-${BUILD_NUMBER}
+                    docker buildx imagetools create --prefer-index=false -t ${PUSHIMAGE}:${SEMVER} ${CACHEIMAGE}:amd64-${COMMIT_SHA}-${BUILD_NUMBER} || \
+                      { if [[ "${PUSHIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
                   fi
                 done
               '''
@@ -883,20 +898,27 @@ pipeline {
                           CACHEIMAGE=${i}
                       fi
                   done
-                  docker buildx imagetools create --prefer-index=false -t ${MANIFESTIMAGE}:amd64-${META_TAG} -t ${MANIFESTIMAGE}:amd64-develop -t ${MANIFESTIMAGE}:amd64-${EXT_RELEASE_TAG} ${CACHEIMAGE}:amd64-${COMMIT_SHA}-${BUILD_NUMBER}
+                  docker buildx imagetools create --prefer-index=false -t ${MANIFESTIMAGE}:amd64-${META_TAG} -t ${MANIFESTIMAGE}:amd64-latest -t ${MANIFESTIMAGE}:amd64-${EXT_RELEASE_TAG} ${CACHEIMAGE}:amd64-${COMMIT_SHA}-${BUILD_NUMBER} || \
-                  docker buildx imagetools create --prefer-index=false -t ${MANIFESTIMAGE}:arm64v8-${META_TAG} -t ${MANIFESTIMAGE}:arm64v8-develop -t ${MANIFESTIMAGE}:arm64v8-${EXT_RELEASE_TAG} ${CACHEIMAGE}:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER}
+                    { if [[ "${MANIFESTIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
+                  docker buildx imagetools create --prefer-index=false -t ${MANIFESTIMAGE}:arm64v8-${META_TAG} -t ${MANIFESTIMAGE}:arm64v8-latest -t ${MANIFESTIMAGE}:arm64v8-${EXT_RELEASE_TAG} ${CACHEIMAGE}:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER} || \
+                    { if [[ "${MANIFESTIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
                   if [ -n "${SEMVER}" ]; then
-                    docker buildx imagetools create --prefer-index=false -t ${MANIFESTIMAGE}:amd64-${SEMVER} ${CACHEIMAGE}:amd64-${COMMIT_SHA}-${BUILD_NUMBER}
+                    docker buildx imagetools create --prefer-index=false -t ${MANIFESTIMAGE}:amd64-${SEMVER} ${CACHEIMAGE}:amd64-${COMMIT_SHA}-${BUILD_NUMBER} || \
-                    docker buildx imagetools create --prefer-index=false -t ${MANIFESTIMAGE}:arm64v8-${SEMVER} ${CACHEIMAGE}:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER}
+                      { if [[ "${MANIFESTIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
+                    docker buildx imagetools create --prefer-index=false -t ${MANIFESTIMAGE}:arm64v8-${SEMVER} ${CACHEIMAGE}:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER} || \
+                      { if [[ "${MANIFESTIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
                   fi
                 done
                 for MANIFESTIMAGE in "${IMAGE}" "${GITLABIMAGE}" "${GITHUBIMAGE}" "${QUAYIMAGE}"; do
-                  docker buildx imagetools create -t ${MANIFESTIMAGE}:develop ${MANIFESTIMAGE}:amd64-develop ${MANIFESTIMAGE}:arm64v8-develop
+                  docker buildx imagetools create -t ${MANIFESTIMAGE}:latest ${MANIFESTIMAGE}:amd64-latest ${MANIFESTIMAGE}:arm64v8-latest || \
-                  docker buildx imagetools create -t ${MANIFESTIMAGE}:${META_TAG} ${MANIFESTIMAGE}:amd64-${META_TAG} ${MANIFESTIMAGE}:arm64v8-${META_TAG}
+                    { if [[ "${MANIFESTIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
-
+                  docker buildx imagetools create -t ${MANIFESTIMAGE}:${META_TAG} ${MANIFESTIMAGE}:amd64-${META_TAG} ${MANIFESTIMAGE}:arm64v8-${META_TAG} || \
-                  docker buildx imagetools create -t ${MANIFESTIMAGE}:${EXT_RELEASE_TAG} ${MANIFESTIMAGE}:amd64-${EXT_RELEASE_TAG} ${MANIFESTIMAGE}:arm64v8-${EXT_RELEASE_TAG}
+                    { if [[ "${MANIFESTIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
+                  docker buildx imagetools create -t ${MANIFESTIMAGE}:${EXT_RELEASE_TAG} ${MANIFESTIMAGE}:amd64-${EXT_RELEASE_TAG} ${MANIFESTIMAGE}:arm64v8-${EXT_RELEASE_TAG} || \
+                    { if [[ "${MANIFESTIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
                   if [ -n "${SEMVER}" ]; then
-                    docker buildx imagetools create -t ${MANIFESTIMAGE}:${SEMVER} ${MANIFESTIMAGE}:amd64-${SEMVER} ${MANIFESTIMAGE}:arm64v8-${SEMVER}
+                    docker buildx imagetools create -t ${MANIFESTIMAGE}:${SEMVER} ${MANIFESTIMAGE}:amd64-${SEMVER} ${MANIFESTIMAGE}:arm64v8-${SEMVER} || \
+                      { if [[ "${MANIFESTIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
                   fi
                 done
               '''
@@ -906,7 +928,7 @@ pipeline {
     // If this is a public release tag it in the LS Github
     stage('Github-Tag-Push-Release') {
       when {
-        branch "develop"
+        branch "master"
         expression {
           env.LS_RELEASE != env.EXT_RELEASE_CLEAN + '-ls' + env.LS_TAG_NUMBER
         }
@@ -914,36 +936,54 @@ pipeline {
         environment name: 'EXIT_STATUS', value: ''
       }
       steps {
-        echo "Pushing New tag for current commit ${META_TAG}"
-        sh '''curl -H "Authorization: token ${GITHUB_TOKEN}" -X POST https://api.github.com/repos/${LS_USER}/${LS_REPO}/git/tags \
-        -d '{"tag":"'${META_TAG}'",\
-             "object": "'${COMMIT_SHA}'",\
-             "message": "Tagging Release '${EXT_RELEASE_CLEAN}'-ls'${LS_TAG_NUMBER}' to develop",\
-             "type": "commit",\
-             "tagger": {"name": "LinuxServer-CI","email": "ci@linuxserver.io","date": "'${GITHUB_DATE}'"}}' '''
-        echo "Pushing New release for Tag"
         sh '''#! /bin/bash
+              echo "Auto-generating release notes"
+              if [ "$(git tag --points-at HEAD)" != "" ]; then
+                echo "Existing tag points to current commit, suggesting no new LS changes"
+                AUTO_RELEASE_NOTES="No changes"
+              else
+                AUTO_RELEASE_NOTES=$(curl -fsL -H "Authorization: token ${GITHUB_TOKEN}" -H "Accept: application/vnd.github+json" -X POST https://api.github.com/repos/${LS_USER}/${LS_REPO}/releases/generate-notes  \
+                  -d '{"tag_name":"'${META_TAG}'",\
+                      "target_commitish": "master"}' \
+                  | jq -r '.body' | sed 's|## What.s Changed||')
+              fi
+              echo "Pushing New tag for current commit ${META_TAG}"
+              curl -H "Authorization: token ${GITHUB_TOKEN}" -X POST https://api.github.com/repos/${LS_USER}/${LS_REPO}/git/tags \
+                -d '{"tag":"'${META_TAG}'",\
+                  "object": "'${COMMIT_SHA}'",\
+                  "message": "Tagging Release '${EXT_RELEASE_CLEAN}'-ls'${LS_TAG_NUMBER}' to master",\
+                  "type": "commit",\
+                  "tagger": {"name": "LinuxServer-CI","email": "ci@linuxserver.io","date": "'${GITHUB_DATE}'"}}'
+              echo "Pushing New release for Tag"
               echo "Updating base packages to ${PACKAGE_TAG}" > releasebody.json
-              echo '{"tag_name":"'${META_TAG}'",\
+              jq -n \
-                     "target_commitish": "develop",\
+                --arg tag_name "$META_TAG" \
-                     "name": "'${META_TAG}'",\
+                --arg target_commitish "master" \
-                     "body": "**CI Report:**\\n\\n'${CI_URL:-N/A}'\\n\\n**LinuxServer Changes:**\\n\\n'${LS_RELEASE_NOTES}'\\n\\n**Remote Changes:**\\n\\n' > start
+                --arg ci_url "${CI_URL:-N/A}" \
-              printf '","draft": false,"prerelease": true}' >> releasebody.json
+                --arg ls_notes "$AUTO_RELEASE_NOTES" \
-              paste -d'\\0' start releasebody.json > releasebody.json.done
+                --arg remote_notes "$(cat releasebody.json)" \
-              curl -H "Authorization: token ${GITHUB_TOKEN}" -X POST https://api.github.com/repos/${LS_USER}/${LS_REPO}/releases -d @releasebody.json.done'''
+                '{
+                  "tag_name": $tag_name,
+                  "target_commitish": $target_commitish,
+                  "name": $tag_name,
+                  "body": ("**CI Report:**\\n\\n" + $ci_url + "\\n\\n**LinuxServer Changes:**\\n\\n" + $ls_notes + "\\n\\n**Remote Changes:**\\n\\n" + $remote_notes),
+                  "draft": false,
+                  "prerelease": false                }' > releasebody.json.done
+              curl -H "Authorization: token ${GITHUB_TOKEN}" -X POST https://api.github.com/repos/${LS_USER}/${LS_REPO}/releases -d @releasebody.json.done
+        '''
       }
     }
     // Add protection to the release branch
     stage('Github-Release-Branch-Protection') {
       when {
-        branch "develop"
+        branch "master"
         environment name: 'CHANGE_ID', value: ''
         environment name: 'EXIT_STATUS', value: ''
       }
       steps {
-        echo "Setting up protection for release branch develop"
+        echo "Setting up protection for release branch master"
         sh '''#! /bin/bash
-          curl -H "Authorization: token ${GITHUB_TOKEN}" -X PUT https://api.github.com/repos/${LS_USER}/${LS_REPO}/branches/develop/protection \
+          curl -H "Authorization: token ${GITHUB_TOKEN}" -X PUT https://api.github.com/repos/${LS_USER}/${LS_REPO}/branches/master/protection \
           -d $(jq -c .  << EOF
             {
               "required_status_checks": null,

README.md

@@ -1,5 +1,5 @@
 <!-- DO NOT EDIT THIS FILE MANUALLY -->
-<!-- Please read https://github.com/linuxserver/docker-ci/blob/develop/.github/CONTRIBUTING.md -->
+<!-- Please read https://github.com/linuxserver/docker-ci/blob/master/.github/CONTRIBUTING.md -->
 [linuxserverurl]: https://linuxserver.io
 [forumurl]: https://discourse.linuxserver.io
 [huburl]: https://hub.docker.com/r/linuxserver/ci/

ci/ci.py

@@ -10,11 +10,13 @@ import logging
 from logging import Logger
 import mimetypes
 import json
+import subprocess
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 from functools import wraps
 from typing import Callable, Any, Literal
 from textwrap import dedent
+import enum
 
 import boto3
 import requests
@@ -62,6 +64,42 @@ def deprecated(reason: str):
         return wrapper
     return deprecated_decorator
 
+class CITests(enum.Enum):
+    """Enum for the different CI tests"""
+    CONTAINER_START = "Container Start"
+    CREATE_SBOM = "Create SBOM"
+    CREATE_BUILDX_SBOM = "Create Buildx SBOM"
+    CREATE_SYFT_SBOM = "Create Syft SBOM"
+    CAPTURE_SCREENSHOT = "Capture Screenshot"
+    GET_BUILD_INFO = "Get Build Info"
+
+class CITestResult(enum.Enum):
+    """Enum for the different CI test results"""
+    PASS = "PASS"
+    FAIL = "FAIL"
+    ERROR = "ERROR"
+
+class CIReportResult(enum.Enum):
+    """Enum for the different CI report results"""
+    PASS = "PASS"
+    FAIL = "FAIL"
+    ERROR = "ERROR"
+
+class Platform(enum.Enum):
+    """Enum for the different platforms"""
+    AMD64 = "amd64"
+    ARM64 = "arm64"
+    ARM32 = "arm"
+    RISCV64 = "riscv64"
+    UNKNOWN = "amd64"
+
+class BuildCacheTag(enum.Enum):
+    """Enum for the different build cache tags"""
+    AMD64 = "amd64"
+    ARM64 = "arm64v8"
+    RISCV64 = "riscv64"
+    UNKNOWN = "amd64"
+
 class SetEnvs():
     """Simple helper class that sets up the ENVs"""
     def __init__(self) -> None:
@@ -88,6 +126,9 @@ class SetEnvs():
         self.bucket: str = os.environ.get("S3_BUCKET", "ci-tests.linuxserver.io")
         self.release_tag: str = os.environ.get("RELEASE_TAG", "latest")
         self.syft_image_tag: str = os.environ.get("SYFT_IMAGE_TAG", "v1.26.1")
+        self.commit_sha: str = os.environ.get("COMMIT_SHA", "")
+        self.build_number: str = os.environ.get("BUILD_NUMBER", "")
+        self.build_cache_registry: str = os.environ.get("BUILD_CACHE_REGISTRY", "ghcr.io/linuxserver/lsiodev-buildcache")
 
         if os.environ.get("DELAY_START"):
             self.logger.warning("DELAY_START env is obsolete, and not in use anymore")
@@ -138,6 +179,9 @@ class SetEnvs():
         S3_REGION:              '{os.environ.get("S3_REGION")}'
         S3_BUCKET:              '{os.environ.get("S3_BUCKET")}'
         SYFT_IMAGE_TAG:         '{os.environ.get("SYFT_IMAGE_TAG")}'
+        COMMIT_SHA:             '{os.environ.get("COMMIT_SHA")}'
+        BUILD_NUMBER:           '{os.environ.get("BUILD_NUMBER")}'
+        BUILD_CACHE_REGISTRY:   '{os.environ.get("BUILD_CACHE_REGISTRY")}'
         Docker Engine Version:  '{self.get_docker_engine_version()}'
         """)
         self.logger.info(env_data)
@@ -269,7 +313,7 @@ class CI(SetEnvs):
         self.tags = list(self.tags_env.split("|"))
         self.tag_report_tests:dict[str,dict[str,dict]] = {tag: {"test":{}} for tag in self.tags} # Adds all the tags as keys with an empty dict as value to the dict
         self.report_containers: dict[str,dict[str,dict]] = {}
-        self.report_status = "PASS"
+        self.report_status = CIReportResult.PASS
         self.outdir: str = f"{os.path.dirname(os.path.realpath(__file__))}/output/{self.image}/{self.meta_tag}"
         os.makedirs(self.outdir, exist_ok=True)
         self.s3_client = self.create_s3_client()
@@ -320,31 +364,31 @@ class CI(SetEnvs):
 
         # Run these tests in parallel so the runtime data is more accurate.
         with ThreadPoolExecutor(max_workers=2,thread_name_prefix=thread_name) as executor:
-            future_sbom: Future[str] = executor.submit(self.generate_sbom, tag)
+            future_sbom: Future[str|CITestResult] = executor.submit(self.make_sbom, tag)
             future_logs: Future[bool] = executor.submit(self.watch_container_logs, container, tag)
 
-        sbom: str = future_sbom.result(self.sbom_timeout + 5) # Set a thread timeout if the function for some reason hangs
+        sbom: str|CITestResult = future_sbom.result(self.sbom_timeout + 5) # Set a thread timeout if the function for some reason hangs
         logsfound: bool = future_logs.result(self.logs_timeout + 5) # Set a thread timeout if the function for some reason hangs
-        build_info: dict = self.get_build_info(container,tag) # Get the image build info
+        build_info: dict[str,str] = self.get_build_info(container,tag) # Get the image build info
 
         if not logsfound:
             self.logger.error("Test of %s FAILED after %.2f seconds", tag, time.time() - start_time)
             self._endtest(container, tag, build_info, sbom, False, start_time)
             return
 
-        if build_info["version"] == "ERROR":
+        if build_info["version"] == CITestResult.ERROR.value:
             self.logger.error("Test of %s FAILED after %.2f seconds", tag, time.time() - start_time)
             self._endtest(container, tag, build_info, sbom, False, start_time)
             return
 
-        if sbom == "ERROR":
+        if sbom == CITestResult.ERROR:
             self.logger.error("Test of %s FAILED after %.2f seconds", tag, time.time() - start_time)
             self._endtest(container, tag, build_info, sbom, False, start_time)
             return
 
         # Screenshot the web interface and check connectivity
         screenshot_success, browser_logs = self.take_screenshot(container, tag)
-        if not screenshot_success and self.get_platform(tag) == "amd64":
+        if not screenshot_success and self.get_platform(tag) == Platform.AMD64.value:
             self.logger.error("Test of %s FAILED after %.2f seconds", tag, time.time() - start_time)
             self._endtest(container, tag, build_info, sbom, False, start_time, browser_logs)
             return
@@ -353,7 +397,7 @@ class CI(SetEnvs):
         self.logger.success("Test of %s PASSED after %.2f seconds", tag, time.time() - start_time)
         return
 
-    def _endtest(self, container:Container, tag:str, build_info:dict[str,str], packages:str, test_success: bool, start_time:float|int = 0.0, browser_logs: str = "") -> None:
+    def _endtest(self, container:Container, tag:str, build_info:dict[str,str], packages:str|CITestResult, test_success: bool, start_time:float|int = 0.0, browser_logs: str = "") -> None:
         """End the test with as much info as we have and append to the report.
 
         Args:
@@ -369,10 +413,12 @@ class CI(SetEnvs):
             runtime = "-"
         if isinstance(start_time,(float, int)):
             runtime = f"{time.time() - start_time:.2f}s"
+        if isinstance(packages, CITestResult):
+            packages = packages.value
         logblob: str = container.logs(timestamps=True).decode("utf-8")
         self.create_html_ansi_file(logblob, tag, "log") # Generate an html container log file based on the latest logs
         try:
-            container.remove(force="true")
+            container.remove(force=True)
         except APIError:
             self.logger.exception("Failed to remove container %s",tag)
         warning_texts: dict[str, str] = {
@@ -428,15 +474,15 @@ class CI(SetEnvs):
         platform: str = tag[:5]
         match platform:
             case "amd64":
-                return "amd64"
+                return Platform.AMD64.value
             case "arm64":
-                return "arm64"
+                return Platform.ARM64.value
             case "arm32":
-                return "arm"
+                return Platform.ARM32.value
             case "riscv":
-                return "riscv64"
+                return Platform.RISCV64.value
             case _:
-                return "amd64"
+                return Platform.UNKNOWN.value
 
     @deprecated(reason="Use generate_sbom instead")
     def export_package_info(self, container:Container, tag:str) -> str:
@@ -475,7 +521,8 @@ class CI(SetEnvs):
                 "message":str(error)}.items())))
             self.report_status = "FAIL"
         return packages
-
+    
+    @deprecated(reason="Use make_sbom instead")
     def generate_sbom(self, tag:str) -> str:
         """Generate the SBOM for the image tag.
 
@@ -522,6 +569,188 @@ class CI(SetEnvs):
             self.logger.exception("Failed to remove the syft container, %s",tag)
         return "ERROR"
 
+    def make_sbom(self, tag: str) -> "str|CITestResult":
+        """Generate the SBOM for the image tag. It will first try to use buildx imagetools inspect, if that fails it will fallback to using syft.
+
+        Creates the output file in `{self.outdir}/{tag}.sbom.html`
+
+        Args:
+            tag (str): The tag we are testing
+        Returns:
+            str: Return the output if successful otherwise "ERROR".
+        """
+        start_time = time.time()
+        sbom: str | CITestResult = self.get_sbom_buildx_blob(tag)
+        if isinstance(sbom, str):
+            packages: list[dict[str, str]] = self.parse_buildx_sbom(sbom)
+            formatted_table: str | CITestResult = self.format_package_table(packages=packages)
+            if packages and formatted_table != CITestResult.ERROR:
+                self.create_html_ansi_file(formatted_table, tag, "sbom")
+                self._add_test_result(tag, CITests.CREATE_BUILDX_SBOM, CITestResult.PASS, "-", start_time)
+                self.logger.success("Create SBOM %s: PASSED after %.2f seconds", tag, time.time() - start_time)
+                return formatted_table
+            self.logger.error("No packages found in buildx SBOM for tag %s", tag)
+            self.report_status = CIReportResult.FAIL
+            self._add_test_result(tag, CITests.CREATE_BUILDX_SBOM, CITestResult.FAIL, "No packages found in buildx SBOM", start_time)
+        else:
+            self.logger.warning("Falling back to Syft for SBOM generation on tag %s", tag)
+        
+        # Fallback to syft if buildx failed
+        sbom = self.get_sbom_syft(tag)
+        if sbom != CITestResult.ERROR:
+            self._add_test_result(tag, CITests.CREATE_SBOM, CITestResult.PASS, "-", start_time)
+            self.logger.success("Create SBOM %s: PASSED after %.2f seconds", tag, time.time() - start_time)
+            self.create_html_ansi_file(str(sbom),tag,"sbom")
+            return sbom
+        self.report_status = CIReportResult.FAIL
+        self._add_test_result(tag, CITests.CREATE_SBOM, CITestResult.FAIL, "Failed to generate SBOM with both buildx and syft", start_time)
+        return CITestResult.ERROR
+    
+    def get_sbom_syft(self, tag: str) -> str | CITestResult:
+        """Get the SBOM for the image tag using Syft.
+
+        Args:
+            tag (str): The tag we are testing
+        Returns:
+            str: SBOM output if successful, otherwise "ERROR".
+        """
+        start_time = time.time()
+        platform: str = self.get_platform(tag)
+        syft:Container = self.client.containers.run(image=f"ghcr.io/anchore/syft:{self.syft_image_tag}",command=f"{self.image}:{tag} --platform=linux/{platform}",
+            detach=True, volumes={"/var/run/docker.sock": {"bind": "/var/run/docker.sock", "mode": "rw"}})
+        self.logger.info("Creating SBOM package list on %s with syft version %s",tag,self.syft_image_tag)
+        logblob: str = ""
+        t_end: float = time.time() + self.sbom_timeout
+        self.logger.info("Tailing the Syft container logs for %s seconds looking the 'VERSION' message on tag: %s",self.sbom_timeout,tag)
+        while time.time() < t_end:
+            time.sleep(5)
+            try:
+                logblob: str = syft.logs().decode("utf-8")
+                if "VERSION" in logblob:
+                    self.logger.info("Get Syft package versions for %s completed", tag)
+                    self.logger.success("%s package list %s: PASSED after %.2f seconds", CITests.CREATE_SYFT_SBOM.value, tag, time.time() - start_time)
+                    try:
+                        syft.remove(force=True)
+                    except Exception:
+                        self.logger.exception("Failed to remove the Syft container, %s",tag)
+                    return logblob
+            except (APIError,ContainerError,ImageNotFound):
+                self.logger.exception("Creating Syft SBOM package list on %s: FAIL", tag)
+        self.logger.error("Failed to generate Syft SBOM output on tag %s. SBOM output:\n%s",tag, logblob)
+        try:
+            syft.remove(force=True)
+        except Exception:
+            self.logger.exception("Failed to remove the Syft container, %s",tag)
+        return CITestResult.ERROR
+
+    def get_build_cache_url(self, tag: str) -> str:
+        """Get the build cache URL for the given tag.
+
+        Args:
+            tag (str): The tag we are testing
+        Returns:
+            str: The build cache URL ex `ghcr.io/linuxserver/lsiodev-buildcache:arm64v8-<COMMIT_SHA>-<BUILD_NUMBER>`
+        """
+        platform: str = self.get_build_cache_platform(tag)
+        build_cache_url: str = f"{self.build_cache_registry}:{platform}-{self.commit_sha}-{self.build_number}"
+        return build_cache_url
+
+    def get_build_cache_platform(self, tag: str) -> str:
+        """Get the build cache platform for the given tag.
+
+        Args:
+            tag (str): The tag we are testing
+
+        Returns:
+            str: The build cache platform
+        """
+        
+        platform = self.get_platform(tag)
+        match platform:
+            case Platform.AMD64.value:
+                return BuildCacheTag.AMD64.value
+            case Platform.ARM64.value:
+                return BuildCacheTag.ARM64.value
+            case Platform.RISCV64.value:
+                return BuildCacheTag.RISCV64.value
+            case _:
+                return BuildCacheTag.UNKNOWN.value
+
+    def get_sbom_buildx_blob(self, tag: str) -> str | CITestResult:
+        """Get the SBOM for the image tag using docker buildx imagetools inspect.
+
+        Args:
+            tag (str): The tag we are testing
+
+        Returns:
+            str: SBOM output if successful, otherwise "CITestResult.ERROR".
+        """
+        try:
+            image_ref:str = self.get_build_cache_url(tag)
+            self.logger.info("Generating SBOM for %s using buildx imagetools inspect", image_ref)
+            cmd = f'docker buildx imagetools inspect {image_ref} --format "{{{{ json (index .SBOM).SPDX.packages }}}}"'
+            result: subprocess.CompletedProcess[str] = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=self.sbom_timeout, check=False)
+            if result.returncode == 0 and result.stdout.strip():
+                self.logger.success("SBOM generated for %s using buildx imagetools inspect.", tag)
+                return result.stdout.strip()
+            self.logger.error("Failed to generate buildx SBOM for %s: %s", tag, result.stderr)
+            return CITestResult.ERROR
+        except Exception:
+            self.logger.exception("Exception while running buildx imagetools inspect for %s", tag)
+            return CITestResult.ERROR
+
+    def parse_buildx_sbom(self, sbom: str) -> list[dict[str, str]]:
+        """Parse the buildx imagetools inspect SBOM string and extract package information.
+
+        Args:
+            sbom (str): The SBOM in JSON format.
+        Returns:
+            list[dict[str, str]]: A list of dictionaries containing package information.
+        """
+        try:
+            sbom_data: dict[str, Any] = json.loads(sbom)
+            packages: list[dict[str, str]] = []
+            for item in sbom_data:
+                package_info = {
+                    "name": item.get("name", ""),
+                    "version": item.get("versionInfo", ""),
+                }
+                if package_info in packages:
+                    continue
+                packages.append(package_info)
+            return packages
+        except json.JSONDecodeError:
+            self.logger.error("Failed to parse buildx imagetools inspect SBOM output.")
+            return []
+
+    def format_package_table(self, packages: list[dict[str, str]]) -> str | CITestResult:
+        """Format a list of package dicts into a padded string table.
+
+        Args:
+            packages (list[dict[str, str]]): List of package dicts with 'name' and 'version' keys.
+
+        Returns:
+            str: Padded string table of package names and versions.
+        """
+        try:
+            if not packages:
+                self.logger.error("No packages found to format.")
+                return CITestResult.ERROR
+            # Dynamically determine column widths based on longest name/version
+            name_width = max(len(pkg.get('name', '')) for pkg in packages) + 10
+            version_width = max(len(pkg.get('version', '')) for pkg in packages) + 10
+            lines: list[str] = []
+            header = f"{'NAME'.ljust(name_width)}{'VERSION'.ljust(version_width)}"
+            lines.append(header)
+            for pkg in packages:
+                name = pkg.get('name', '')[:name_width]
+                version = pkg.get('version', '')[:version_width]
+                lines.append(f"{name.ljust(name_width)}{version.ljust(version_width)}")
+            return "\n".join(lines)
+        except Exception:
+            self.logger.exception("Failed to format package table.")
+            return CITestResult.ERROR
+
     @deprecated(reason="Use get_build_info instead")
     def get_build_version(self,container:Container,tag:str) -> str:
         """Fetch the build version from the container object attributes.
@@ -568,7 +797,7 @@ class CI(SetEnvs):
             case _:
                 return self.image
 
-    def get_build_url(self, tag) -> str:
+    def get_build_url(self, tag: str) -> str:
         """Get the build url from the IMAGE env.
 
         Args:
@@ -610,7 +839,6 @@ class CI(SetEnvs):
             }
             ```
         """
-        test = "Get build info"
         start_time = time.time()
         try:
             self.logger.info("Fetching build info on tag: %s",tag)
@@ -623,15 +851,15 @@ class CI(SetEnvs):
                 "tag": tag,
                 "image": self.get_image_name()
             }
-            self._add_test_result(tag, test, "PASS", "-", start_time)
+            self._add_test_result(tag, CITests.GET_BUILD_INFO, CITestResult.PASS, "-", start_time)
             self.logger.success("Get build info on tag '%s': PASS", tag)
         except (APIError,KeyError) as error:
             self.logger.exception("Get build info on tag '%s': FAIL", tag)
-            build_info = {"version": "ERROR", "created": "ERROR", "size": "ERROR", "maintainer": "ERROR"}
+            build_info = {"version": CITestResult.ERROR.value, "created": CITestResult.ERROR.value, "size": CITestResult.ERROR.value, "maintainer": CITestResult.ERROR.value}
             if isinstance(error,KeyError):
                 error: str = f"KeyError: {error}"
-            self._add_test_result(tag, test, "FAIL", str(error), start_time)
+            self._add_test_result(tag, CITests.GET_BUILD_INFO, CITestResult.FAIL, str(error), start_time)
-            self.report_status = "FAIL"
+            self.report_status = CIReportResult.FAIL
         return build_info
 
     def watch_container_logs(self, container:Container, tag:str) -> bool:
@@ -645,7 +873,6 @@ class CI(SetEnvs):
         Returns:
             bool: Return True if the "done" message is found, otherwise False.
         """
-        test = "Container startup"
         start_time = time.time()
         t_end: float = time.time() + self.logs_timeout
         self.logger.info("Tailing the %s logs for %s seconds looking for the 'done' message", tag, self.logs_timeout)
@@ -653,20 +880,20 @@ class CI(SetEnvs):
             try:
                 logblob: str = container.logs().decode("utf-8")
                 if "[services.d] done." in logblob or "[ls.io-init] done." in logblob:
-                    self.logger.info("%s completed for %s",test, tag)
+                    self.logger.info("%s completed for %s",CITests.CONTAINER_START.value, tag)
-                    self._add_test_result(tag, test, "PASS", "-", start_time)
+                    self._add_test_result(tag, CITests.CONTAINER_START, CITestResult.PASS, "-", start_time)
-                    self.logger.success("%s %s: PASSED after %.2f seconds", test, tag, time.time() - start_time)
+                    self.logger.success("%s %s: PASSED after %.2f seconds", CITests.CONTAINER_START.value, tag, time.time() - start_time)
                     return True
                 time.sleep(1)
             except APIError as error:
-                self.logger.exception("%s %s: FAIL - INIT NOT FINISHED", test, tag)
+                self.logger.exception("%s %s: FAIL - INIT NOT FINISHED", CITests.CONTAINER_START.value, tag)
-                self._add_test_result(tag, test, "FAIL", f"INIT NOT FINISHED: {str(error)}", start_time)
+                self._add_test_result(tag, CITests.CONTAINER_START, CITestResult.FAIL, f"INIT NOT FINISHED: {str(error)}", start_time)
-                self.report_status = "FAIL"
+                self.report_status = CIReportResult.FAIL
                 return False
-        self.logger.error("%s failed for %s", test, tag)
+        self.logger.error("%s failed for %s", CITests.CONTAINER_START.value, tag)
-        self._add_test_result(tag, test, "FAIL", "INIT NOT FINISHED", start_time)
+        self._add_test_result(tag, CITests.CONTAINER_START, CITestResult.FAIL, "INIT NOT FINISHED", start_time)
-        self.logger.error("%s %s: FAIL - INIT NOT FINISHED", test, tag)
+        self.logger.error("%s %s: FAIL - INIT NOT FINISHED", CITests.CONTAINER_START.value, tag)
-        self.report_status = "FAIL"
+        self.report_status = CIReportResult.FAIL
         return False
 
     def report_render(self) -> None:
@@ -679,7 +906,7 @@ class CI(SetEnvs):
         with open(f"{self.outdir}/index.html", mode="w", encoding="utf-8") as file_:
             file_.write(template.render(
             report_containers=self.report_containers,
-            report_status=self.report_status,
+            report_status=self.report_status.value,
             meta_tag=self.meta_tag,
             image=self.get_image_name(),
             bucket=self.bucket,
@@ -692,11 +919,11 @@ class CI(SetEnvs):
         """Render the badge file for upload"""
         self.logger.info("Creating badge")
         try:
-            badge = anybadge.Badge("CI", self.report_status, thresholds={
+            badge = anybadge.Badge("CI", self.report_status.value, thresholds={
                                    "PASS": "green", "FAIL": "red"})
             badge.write_badge(f"{self.outdir}/badge.svg", overwrite=True)
             with open(f"{self.outdir}/ci-status.yml", "w", encoding="utf-8") as file:
-                file.write(f"CI: '{self.report_status}'")
+                file.write(f"CI: '{self.report_status.value}'")
         except (ValueError,RuntimeError,FileNotFoundError,OSError):
             self.logger.exception("Failed to render badge file!")
 
@@ -737,7 +964,7 @@ class CI(SetEnvs):
                 raise CIError(f"Upload Error: {error}") from error
         self.logger.info("Report available on https://%s/%s/%s/index.html",self.bucket, self.image, self.meta_tag)
 
-    def create_html_ansi_file(self, blob:str, tag:str, name:str, full:bool = True) -> None:
+    def create_html_ansi_file(self, blob:str|CITestResult, tag:str, name:str, full:bool = True) -> None:
         """Creates an HTML file in the "self.outdir" directory that we upload to S3
 
         Args:
@@ -748,6 +975,8 @@ class CI(SetEnvs):
 
         """
         try:
+            if isinstance(blob,CITestResult):
+                blob = blob.value
             self.logger.info("Creating %s.%s.html", tag, name)
             converter = Ansi2HTMLConverter(title=f"{tag}-{name}")
             html_logs: str = converter.convert(blob,full=full)
@@ -793,26 +1022,28 @@ class CI(SetEnvs):
         except (S3UploadFailedError, ClientError, FileNotFoundError) as e:
             self.logger.exception(f"Failed to upload the CI logs! Error: {e}")
 
-    def _add_test_result(self, tag:str, test:str, status:str, message:str, start_time:float|int = 0.0) -> None:
+    def _add_test_result(self, tag:str, test:CITests, status:CITestResult, message:str, start_time:float|int = 0.0) -> None:
         """Add a test result to the report
 
         Args:
             tag (str): The tag we are testing
-            test (str): The test we are running
+            test (CITests): The test we are running
-            status (str): The status of the test
+            status (CITestResult): The status of the test
             message (str): The message of the test
             start_time (str, optional): The start time of the test. Defaults to 0.0. Used to calculate the runtime of the test.
         """
-        if status not in ["PASS","FAIL"]:
+        if not isinstance(test, CITests):
-            raise ValueError("Status must be either PASS or FAIL")
+            raise ValueError("test must be an instance of CITests")
+        if not isinstance(status, CITestResult):
+            raise ValueError("status must be an instance of CITestResult")
         if tag not in self.tags:
             raise ValueError("Tag not in the list of tags")
         if not start_time:
             runtime = "-"
         if isinstance(start_time,(float, int)):
             runtime: str = f"{time.time() - start_time:.2f}s"
-        self.tag_report_tests[tag]["test"][test] = (dict(sorted({
+        self.tag_report_tests[tag]["test"][test.value] = (dict(sorted({
-            "status":status,
+            "status":status.value,
             "message":message,
             "runtime": runtime}.items())))
 
@@ -832,7 +1063,6 @@ class CI(SetEnvs):
             return True, ""
         proto: Literal["https", "http"] = "https" if self.ssl.upper() == "TRUE" else "http"
         screenshot_timeout = time.time() + self.screenshot_timeout
-        test = "Get screenshot"
         start_time = time.time()
         driver: WebDriver | None = None
         browser_logs: str = ""
@@ -853,7 +1083,7 @@ class CI(SetEnvs):
                     driver.get_screenshot_as_file(f"{self.outdir}/{tag}.png")
                     if not os.path.isfile(f"{self.outdir}/{tag}.png"):
                         raise FileNotFoundError(f"Screenshot '{self.outdir}/{tag}.png' not found")
-                    self._add_test_result(tag, test, "PASS", "-", start_time)
+                    self._add_test_result(tag, CITests.CAPTURE_SCREENSHOT, CITestResult.PASS, "-", start_time)
                     self.logger.success("Screenshot %s: PASSED after %.2f seconds", tag, time.time() - start_time)
                     return True, self._get_browser_logs(driver, tag)
                 except Exception as error:
@@ -864,23 +1094,23 @@ class CI(SetEnvs):
                         raise error
             raise TimeoutException("Timeout taking screenshot")
         except (requests.Timeout, requests.ConnectionError, KeyError) as error:
-            self._add_test_result(tag, test, "FAIL", f"CONNECTION ERROR: {str(error)}", start_time)
+            self._add_test_result(tag, CITests.CAPTURE_SCREENSHOT, CITestResult.FAIL, f"CONNECTION ERROR: {str(error)}", start_time)
             self.logger.exception("Screenshot %s FAIL CONNECTION ERROR", tag)
-            self.report_status = "FAIL"
+            self.report_status = CIReportResult.FAIL
             if driver:
                 browser_logs = self._get_browser_logs(driver, tag)
             return False, browser_logs
         except TimeoutException as error:
-            self._add_test_result(tag, test, "FAIL", f"TIMEOUT: {str(error)}", start_time)
+            self._add_test_result(tag, CITests.CAPTURE_SCREENSHOT, CITestResult.FAIL, f"TIMEOUT: {str(error)}", start_time)
             self.logger.exception("Screenshot %s FAIL TIMEOUT", tag)
-            self.report_status = "FAIL"
+            self.report_status = CIReportResult.FAIL
             if driver:
                 browser_logs = self._get_browser_logs(driver, tag)
             return False, browser_logs
         except (WebDriverException, Exception) as error:
-            self._add_test_result(tag, test, "FAIL", f"UNKNOWN: {str(error)}", start_time)
+            self._add_test_result(tag, CITests.CAPTURE_SCREENSHOT, CITestResult.FAIL, f"UNKNOWN: {str(error)}", start_time)
             self.logger.exception("Screenshot %s FAIL UNKNOWN", tag)
-            self.report_status = "FAIL"
+            self.report_status = CIReportResult.FAIL
             if driver:
                 browser_logs = self._get_browser_logs(driver, tag)
             return False, browser_logs

jenkins-vars.yml

@@ -3,9 +3,9 @@
 # jenkins variables
 project_name: docker-ci
 external_type: os
-release_type: prerelease
+release_type: stable
-release_tag: develop
+release_tag: latest
-ls_branch: develop
+ls_branch: master
 repo_vars:
   - BUILD_VERSION_ARG = 'OS'
   - LS_USER = 'linuxserver'

test_build.py

@@ -1,8 +1,7 @@
 #!/usr/bin/env python3
 import os
-import time
 from logging import Logger
-from ci.ci import CI, CIError
+from ci.ci import CI, CIError, Platform, CIReportResult
 from ci.logger import configure_logging
 
 def run_test() -> None:
@@ -10,15 +9,15 @@ def run_test() -> None:
     ci.run(ci.tags)
     # Don't set the whole report as failed if any of the ARM tag fails.
     for tag in ci.report_containers.keys():
-        if tag.startswith("amd64") and ci.report_containers[tag]['test_success'] == True:
+        if tag.startswith(Platform.AMD64.value) and ci.report_containers[tag]['test_success'] == True:
-            ci.report_status = 'PASS' # Override the report_status if an ARM tag failed, but the amd64 tag passed.
+            ci.report_status = CIReportResult.PASS # Override the report_status if an ARM tag failed, but the amd64 tag passed.
-    if ci.report_status == 'PASS':
+    if ci.report_status == CIReportResult.PASS:
         logger.success('All tests PASSED after %.2f seconds', ci.total_runtime)
     ci.report_render()
     ci.badge_render()
     ci.json_render()
     ci.report_upload()
-    if ci.report_status == 'PASS':  # Exit based on test results
+    if ci.report_status == CIReportResult.PASS:  # Exit based on test results
         ci.log_upload()
         return
     logger.error('Tests FAILED')

tests/sbom_buildx_formatted_blob.txt

@@ -0,0 +1,149 @@
+NAME                                                                       VERSION                  
+adduser                                                                    3.137ubuntu1             
+apt                                                                        2.8.3                    
+apt-utils                                                                  2.8.3                    
+base-files                                                                 13ubuntu10.3             
+base-passwd                                                                3.6.3build1              
+bash                                                                       5.2.21-2ubuntu4          
+bsdutils                                                                   1:2.39.3-9ubuntu6.3      
+ca-certificates                                                            20240203                 
+catatonit                                                                  0.1.7-1                  
+coreutils                                                                  9.4-3ubuntu6.1           
+cron                                                                       3.0pl1-184ubuntu2        
+cron-daemon-common                                                         3.0pl1-184ubuntu2        
+curl                                                                       8.5.0-2ubuntu10.6        
+dash                                                                       0.5.12-6ubuntu5          
+debconf                                                                    1.5.86ubuntu1            
+debianutils                                                                5.17build1               
+diffutils                                                                  1:3.10-1build1           
+dirmngr                                                                    2.4.4-2ubuntu17.3        
+dpkg                                                                       1.22.6ubuntu6.2          
+e2fsprogs                                                                  1.47.0-2.4~exp1ubuntu4.1 
+findutils                                                                  4.9.0-5build1            
+gcc-14-base                                                                14.2.0-4ubuntu2~24.04    
+gnupg                                                                      2.4.4-2ubuntu17.3        
+gnupg-l10n                                                                 2.4.4-2ubuntu17.3        
+gnupg-utils                                                                2.4.4-2ubuntu17.3        
+gpg                                                                        2.4.4-2ubuntu17.3        
+gpg-agent                                                                  2.4.4-2ubuntu17.3        
+gpg-wks-client                                                             2.4.4-2ubuntu17.3        
+gpgconf                                                                    2.4.4-2ubuntu17.3        
+gpgsm                                                                      2.4.4-2ubuntu17.3        
+gpgv                                                                       2.4.4-2ubuntu17.3        
+grep                                                                       3.11-4build1             
+gzip                                                                       1.12-1ubuntu3.1          
+hostname                                                                   3.23+nmu2ubuntu2         
+init-system-helpers                                                        1.66ubuntu1              
+jq                                                                         1.7.1-3ubuntu0.24.04.1   
+keyboxd                                                                    2.4.4-2ubuntu17.3        
+krb5-locales                                                               1.20.1-6ubuntu2.6        
+libacl1                                                                    2.3.2-1build1.1          
+libapt-pkg6.0t64                                                           2.8.3                    
+libassuan0                                                                 2.5.6-1build1            
+libattr1                                                                   1:2.5.2-1build1.1        
+libaudit-common                                                            1:3.1.2-2.1build1.1      
+libaudit1                                                                  1:3.1.2-2.1build1.1      
+libblkid1                                                                  2.39.3-9ubuntu6.3        
+libbrotli1                                                                 1.1.0-2build2            
+libbsd0                                                                    0.12.1-1build1.1         
+libbz2-1.0                                                                 1.0.8-5.1build0.1        
+libc-bin                                                                   2.39-0ubuntu8.6          
+libc6                                                                      2.39-0ubuntu8.6          
+libcap-ng0                                                                 0.8.4-2build2            
+libcap2                                                                    1:2.66-5ubuntu2.2        
+libcom-err2                                                                1.47.0-2.4~exp1ubuntu4.1 
+libcrypt1                                                                  1:4.4.36-4build1         
+libcurl4t64                                                                8.5.0-2ubuntu10.6        
+libdb5.3t64                                                                5.3.28+dfsg2-7           
+libdebconfclient0                                                          0.271ubuntu3             
+libext2fs2t64                                                              1.47.0-2.4~exp1ubuntu4.1 
+libffi8                                                                    3.4.6-1build1            
+libgcc-s1                                                                  14.2.0-4ubuntu2~24.04    
+libgcrypt20                                                                1.10.3-2build1           
+libgmp10                                                                   2:6.3.0+dfsg-2ubuntu6.1  
+libgnutls30t64                                                             3.8.3-1.1ubuntu3.4       
+libgpg-error0                                                              1.47-3build2.1           
+libgssapi-krb5-2                                                           1.20.1-6ubuntu2.6        
+libhogweed6t64                                                             3.9.1-2.2build1.1        
+libidn2-0                                                                  2.3.7-2build1.1          
+libjq1                                                                     1.7.1-3ubuntu0.24.04.1   
+libk5crypto3                                                               1.20.1-6ubuntu2.6        
+libkeyutils1                                                               1.6.3-3build1            
+libkmod2                                                                   31+20240202-2ubuntu7.1   
+libkrb5-3                                                                  1.20.1-6ubuntu2.6        
+libkrb5support0                                                            1.20.1-6ubuntu2.6        
+libksba8                                                                   1.6.6-1build1            
+libldap-common                                                             2.6.7+dfsg-1~exp1ubuntu8.
+libldap2                                                                   2.6.7+dfsg-1~exp1ubuntu8.
+liblz4-1                                                                   1.9.4-1build1.1          
+liblzma5                                                                   5.6.1+really5.4.5-1ubuntu
+libmd0                                                                     1.1.0-2build1.1          
+libmount1                                                                  2.39.3-9ubuntu6.3        
+libncursesw6                                                               6.4+20240113-1ubuntu2    
+libnettle8t64                                                              3.9.1-2.2build1.1        
+libnghttp2-14                                                              1.59.0-1ubuntu0.2        
+libnpth0t64                                                                1.6-3.1build1            
+libonig5                                                                   6.9.9-1build1            
+libp11-kit0                                                                0.25.3-4ubuntu2.1        
+libpam-modules                                                             1.5.3-5ubuntu5.5         
+libpam-modules-bin                                                         1.5.3-5ubuntu5.5         
+libpam-runtime                                                             1.5.3-5ubuntu5.5         
+libpam0g                                                                   1.5.3-5ubuntu5.5         
+libpcre2-8-0                                                               10.42-4ubuntu2.1         
+libproc2-0                                                                 2:4.0.4-4ubuntu3.2       
+libpsl5t64                                                                 0.21.2-1.1build1         
+libreadline8t64                                                            8.2-4build1              
+librtmp1                                                                   2.4+20151223.gitfa8646d.1
+libsasl2-2                                                                 2.1.28+dfsg1-5ubuntu3.1  
+libsasl2-modules                                                           2.1.28+dfsg1-5ubuntu3.1  
+libsasl2-modules-db                                                        2.1.28+dfsg1-5ubuntu3.1  
+libseccomp2                                                                2.5.5-1ubuntu3.1         
+libselinux1                                                                3.5-2ubuntu2.1           
+libsemanage-common                                                         3.5-1build5              
+libsemanage2                                                               3.5-1build5              
+libsepol2                                                                  3.5-2build1              
+libsmartcols1                                                              2.39.3-9ubuntu6.3        
+libsqlite3-0                                                               3.45.1-1ubuntu2.5        
+libss2                                                                     1.47.0-2.4~exp1ubuntu4.1 
+libssh-4                                                                   0.10.6-2ubuntu0.1        
+libssl3t64                                                                 3.0.13-0ubuntu3.5        
+libstdc++6                                                                 14.2.0-4ubuntu2~24.04    
+libsystemd0                                                                255.4-1ubuntu8.10        
+libtasn1-6                                                                 4.19.0-3ubuntu0.24.04.1  
+libtinfo6                                                                  6.4+20240113-1ubuntu2    
+libudev1                                                                   255.4-1ubuntu8.10        
+libunistring5                                                              1.1-2build1.1            
+libuuid1                                                                   2.39.3-9ubuntu6.3        
+libxxhash0                                                                 0.8.2-2build1            
+libzstd1                                                                   1.5.5+dfsg2-2build1.1    
+locales                                                                    2.39-0ubuntu8.6          
+login                                                                      1:4.13+dfsg1-4ubuntu3.2  
+logsave                                                                    1.47.0-2.4~exp1ubuntu4.1 
+mawk                                                                       1.3.4.20240123-1build1   
+mount                                                                      2.39.3-9ubuntu6.3        
+ncurses-base                                                               6.4+20240113-1ubuntu2    
+ncurses-bin                                                                6.4+20240113-1ubuntu2    
+netcat-openbsd                                                             1.226-1ubuntu2           
+openssl                                                                    3.0.13-0ubuntu3.5        
+passwd                                                                     1:4.13+dfsg1-4ubuntu3.2  
+perl-base                                                                  5.38.2-3.2ubuntu0.2      
+pinentry-curses                                                            1.2.1-3ubuntu5           
+plexmediaserver                                                            1.42.2.10156-f737b826c   
+procps                                                                     2:4.0.4-4ubuntu3.2       
+publicsuffix                                                               20231001.0357-0.1        
+readline-common                                                            8.2-4build1              
+sed                                                                        4.9-2build1              
+sensible-utils                                                             0.0.22                   
+systemd-dev                                                                255.4-1ubuntu8.10        
+systemd-hwe-hwdb                                                           255.1.5                  
+systemd-standalone-sysusers                                                255.4-1ubuntu8.10        
+sysvinit-utils                                                             3.08-6ubuntu3            
+tar                                                                        1.35+dfsg-3build1        
+tzdata                                                                     2025b-0ubuntu0.24.04.1   
+ubuntu-keyring                                                             2023.11.28.1             
+udev                                                                       255.4-1ubuntu8.10        
+unminimize                                                                 0.2.1                    
+util-linux                                                                 2.39.3-9ubuntu6.3        
+wget                                                                       1.21.4-1ubuntu4.1        
+zlib1g                                                                     1:1.3.dfsg-3.1ubuntu2.1  
+sbom                                                                                                

tests/test_ci.py

@@ -1,5 +1,5 @@
 import os
-from unittest.mock import Mock
+from unittest.mock import Mock, patch
 import json
 
 import pytest
@@ -8,7 +8,7 @@ import chromedriver_autoinstaller
 from docker import DockerClient
 from moto import mock_aws
 
-from ci.ci import CI, SetEnvs
+from ci.ci import CI, SetEnvs, CITestResult, CITests, Platform, BuildCacheTag
 
 os.environ["DRY_RUN"] = "false"
 os.environ["IMAGE"] = "linuxserver/test"
@@ -23,12 +23,26 @@ os.environ["SSL"] = "true"
 os.environ["PORT"] = "443"
 os.environ["WEB_SCREENSHOT"] = "true"
 os.environ["WEB_AUTH"] = ""
+os.environ["COMMIT_SHA"] = "test-commit-sha"
+os.environ["BUILD_NUMBER"] = "1234"
 
 @pytest.fixture
 def sbom_blob() -> bytes:
     with open("tests/sbom_blob.txt", "rb") as f:
         yield f.read()
 
+@pytest.fixture
+def sbom_buildx_blob() -> str:
+    with open("tests/sbom_buildx_blob.txt", "rb") as f:
+        data = f.read()
+    yield data.decode("utf-8").strip()
+
+@pytest.fixture
+def sbom_buildx_formatted_blob() -> str:
+    with open("tests/sbom_buildx_formatted_blob.txt", "rb") as f:
+        data = f.read()
+    yield data.decode("utf-8").strip()
+
 @pytest.fixture
 def syft_mock_container(sbom_blob:bytes) -> Mock:
     container = Mock(spec=Container)
@@ -38,11 +52,12 @@ def syft_mock_container(sbom_blob:bytes) -> Mock:
     yield container
 
 @pytest.fixture
-def ci(tmpdir, syft_mock_container: Mock) -> CI:
+def ci(tmpdir, syft_mock_container: Mock, sbom_buildx_blob: str) -> CI:
     ci = CI()
     ci.client = Mock(DockerClient)
     ci.client.containers = Mock()
     ci.client.containers.run = Mock(return_value=syft_mock_container)
+    ci.get_sbom_buildx_blob = Mock(return_value=sbom_buildx_blob)
     ci.outdir = tmpdir
     yield ci
 
@@ -108,8 +123,8 @@ def test_convert_env(set_envs: SetEnvs):
 
 def test_add_test_result(ci: CI):
     for tag in ci.tags:
-        ci._add_test_result(tag=tag, test=f"test-{tag}", status="PASS", message="-", start_time="")
+        ci._add_test_result(tag=tag, test=CITests.CONTAINER_START, status=CITestResult.PASS, message="-", start_time="")
-        assert ci.tag_report_tests[tag] == {'test': {f"test-{tag}": {"status": "PASS", "message": "-", "runtime": "-"}}}
+        assert ci.tag_report_tests[tag] == {'test': {CITests.CONTAINER_START.value: {"status": "PASS", "message": "-", "runtime": "-"}}}
 
 def test_get_build_info(ci: CI, mock_container: Mock):
     info: dict[str, str] = ci.get_build_info(mock_container,ci.tags[0])
@@ -125,20 +140,20 @@ def test_get_build_info(ci: CI, mock_container: Mock):
     assert info == mock_info
 
 def test_get_platform(ci: CI):
-    assert ci.get_platform(ci.tags[0]) == "amd64"
+    assert ci.get_platform(ci.tags[0]) == Platform.AMD64.value
-    assert ci.get_platform(ci.tags[1]) == "arm64"
+    assert ci.get_platform(ci.tags[1]) == Platform.ARM64.value
 
 def test_watch_container_logs(ci: CI, mock_container: Mock):
     ci.watch_container_logs(mock_container, ci.tags[0])
-    assert ci.tag_report_tests[ci.tags[0]]["test"]["Container startup"]["status"] == "PASS"
+    assert ci.tag_report_tests[ci.tags[0]]["test"][CITests.CONTAINER_START.value]["status"] == CITestResult.PASS.value
 
 def test_take_screenshot(ci:CI,mock_container: Mock):
     screenshot: bool = ci.take_screenshot(mock_container, ci.tags[0])
     if screenshot:
         assert os.path.isfile(os.path.join(ci.outdir, f"{ci.tags[0]}.png")) is True
-        assert ci.tag_report_tests[ci.tags[0]]["test"]["Get screenshot"]["status"] == "PASS"
+        assert ci.tag_report_tests[ci.tags[0]]["test"][CITests.CAPTURE_SCREENSHOT.value]["status"] == CITestResult.PASS.value
     else:
-        assert ci.tag_report_tests[ci.tags[0]]["test"]["Get screenshot"]["status"] == "FAIL"
+        assert ci.tag_report_tests[ci.tags[0]]["test"][CITests.CAPTURE_SCREENSHOT.value]["status"] == CITestResult.FAIL.value
 
 def test_create_html_ansi_file(ci:CI, log_blob:bytes):
     logs = log_blob.decode("utf-8")
@@ -159,10 +174,41 @@ def test_badge_render(ci:CI):
     ci.badge_render()
     assert os.path.isfile(os.path.join(ci.outdir,"ci-status.yml")) is True
 
-def test_generate_sbom(ci:CI, syft_mock_container:Mock, sbom_blob:bytes):
+def test_get_sbom_syft(ci:CI, syft_mock_container:Mock, sbom_blob:bytes):
-    sbom = ci.generate_sbom(ci.tags[0])
+    sbom = ci.get_sbom_syft(ci.tags[0])
     assert "VERSION" in sbom
 
+def test_parse_buildx_sbom(ci:CI, sbom_buildx_blob:str):
+    packages = ci.parse_buildx_sbom(sbom_buildx_blob)
+    assert len(packages) == 148
+    assert packages[0]["name"] == "adduser"
+    assert packages[0]["version"] == "3.137ubuntu1"
+
+def test_format_package_table(ci:CI, sbom_buildx_blob:str, sbom_buildx_formatted_blob: str):
+    packages = ci.parse_buildx_sbom(sbom_buildx_blob)
+    table = ci.format_package_table(packages)
+    assert "VERSION" in table
+    assert "cron" in table
+    assert "3.0pl1-184ubuntu2" in table
+
+def test_get_sbom_buildx_blob(ci: CI, sbom_buildx_blob: str) -> None:
+    expected_output = sbom_buildx_blob
+    mock_completed_process = Mock()
+    mock_completed_process.returncode = 0
+    mock_completed_process.stdout = expected_output
+
+    with patch("subprocess.run", return_value=mock_completed_process):
+        sbom = ci.get_sbom_buildx_blob(ci.tags[0])
+        assert sbom.strip() == expected_output.strip()
+
+def test_make_sbom(ci: CI, sbom_buildx_blob: str, sbom_buildx_formatted_blob: str) -> None:
+    with patch.object(ci, 'get_sbom_buildx_blob', return_value=sbom_buildx_blob):
+        packages = ci.make_sbom(ci.tags[0])
+        assert os.path.isfile(os.path.join(ci.outdir,"amd64-nightly-5.10.1.9109-ls85.sbom.html")) is True
+        assert "VERSION" in packages
+        assert "cron" in packages
+        assert "3.0pl1-184ubuntu2" in packages
+
 def test_create_s3_client(ci:CI):
     with mock_aws():
         ci.s3_client = ci.create_s3_client()
@@ -197,3 +243,13 @@ def test_get_image_name(ci: CI) -> None:
     assert ci.get_image_name() == "linuxserver/lspipepr-plex"
     ci.image = "lsiobase/ubuntu"
     assert ci.get_image_name() == "linuxserver/docker-baseimage-ubuntu"
+
+def test_get_build_cache_url(ci: CI) -> None:
+    for tag in ci.tags:
+        cache_tag = ci.get_build_cache_platform(tag)
+        expected_url = f"{ci.build_cache_registry}:{cache_tag}-{ci.commit_sha}-{ci.build_number}"
+        assert ci.get_build_cache_url(tag) == expected_url
+
+def test_get_build_cache_platform(ci: CI) -> None:
+    assert ci.get_build_cache_platform(ci.tags[0]) == BuildCacheTag.AMD64.value
+    assert ci.get_build_cache_platform(ci.tags[1]) == BuildCacheTag.ARM64.value