mirror of
https://github.com/linuxserver/docker-ci.git
synced 2026-02-20 02:53:44 +08:00
Compare commits
No commits in common. "master" and "402c1905-ls26" have entirely different histories.
master
...
402c1905-l
10
Dockerfile
10
Dockerfile
@ -23,20 +23,14 @@ RUN \
|
|||||||
apt-get update && \
|
apt-get update && \
|
||||||
apt-get install -y --no-install-recommends \
|
apt-get install -y --no-install-recommends \
|
||||||
docker-ce \
|
docker-ce \
|
||||||
docker-buildx-plugin \
|
|
||||||
google-chrome-stable \
|
google-chrome-stable \
|
||||||
python3-venv \
|
python3-venv \
|
||||||
unzip \
|
unzip \
|
||||||
xserver-xephyr \
|
xserver-xephyr \
|
||||||
xvfb && \
|
xvfb && \
|
||||||
echo "**** install chrome driver ****" && \
|
echo "**** install chrome driver ****" && \
|
||||||
CHROME_VERSION=$(google-chrome --version | awk '{print $3}') && \
|
CHROME_RELEASE=$(curl -sLk https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_STABLE) && \
|
||||||
CHROME_MAJOR=${CHROME_VERSION%%.*} && \
|
echo "Retrieving Chrome driver version ${CHROME_RELEASE}" && \
|
||||||
CHROME_RELEASE=$(curl -fsSLk "https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_${CHROME_MAJOR}" || true) && \
|
|
||||||
if [ -z "${CHROME_RELEASE}" ]; then \
|
|
||||||
CHROME_RELEASE=$(curl -fsSLk https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_STABLE); \
|
|
||||||
fi && \
|
|
||||||
echo "Installed Chrome ${CHROME_VERSION}; retrieving ChromeDriver ${CHROME_RELEASE}" && \
|
|
||||||
curl -sk -o \
|
curl -sk -o \
|
||||||
/tmp/chrome.zip -L \
|
/tmp/chrome.zip -L \
|
||||||
"https://storage.googleapis.com/chrome-for-testing-public/${CHROME_RELEASE}/linux64/chromedriver-linux64.zip" && \
|
"https://storage.googleapis.com/chrome-for-testing-public/${CHROME_RELEASE}/linux64/chromedriver-linux64.zip" && \
|
||||||
|
|||||||
110
Jenkinsfile
vendored
110
Jenkinsfile
vendored
@ -206,7 +206,6 @@ pipeline {
|
|||||||
env.META_TAG = env.EXT_RELEASE_CLEAN + '-ls' + env.LS_TAG_NUMBER
|
env.META_TAG = env.EXT_RELEASE_CLEAN + '-ls' + env.LS_TAG_NUMBER
|
||||||
env.EXT_RELEASE_TAG = 'version-' + env.EXT_RELEASE_CLEAN
|
env.EXT_RELEASE_TAG = 'version-' + env.EXT_RELEASE_CLEAN
|
||||||
env.BUILDCACHE = 'docker.io/lsiodev/buildcache,registry.gitlab.com/linuxserver.io/docker-jenkins-builder/lsiodev-buildcache,ghcr.io/linuxserver/lsiodev-buildcache,quay.io/linuxserver.io/lsiodev-buildcache'
|
env.BUILDCACHE = 'docker.io/lsiodev/buildcache,registry.gitlab.com/linuxserver.io/docker-jenkins-builder/lsiodev-buildcache,ghcr.io/linuxserver/lsiodev-buildcache,quay.io/linuxserver.io/lsiodev-buildcache'
|
||||||
env.CITEST_IMAGETAG = 'latest'
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -232,7 +231,6 @@ pipeline {
|
|||||||
env.EXT_RELEASE_TAG = 'version-' + env.EXT_RELEASE_CLEAN
|
env.EXT_RELEASE_TAG = 'version-' + env.EXT_RELEASE_CLEAN
|
||||||
env.DOCKERHUB_LINK = 'https://hub.docker.com/r/' + env.DEV_DOCKERHUB_IMAGE + '/tags/'
|
env.DOCKERHUB_LINK = 'https://hub.docker.com/r/' + env.DEV_DOCKERHUB_IMAGE + '/tags/'
|
||||||
env.BUILDCACHE = 'docker.io/lsiodev/buildcache,registry.gitlab.com/linuxserver.io/docker-jenkins-builder/lsiodev-buildcache,ghcr.io/linuxserver/lsiodev-buildcache,quay.io/linuxserver.io/lsiodev-buildcache'
|
env.BUILDCACHE = 'docker.io/lsiodev/buildcache,registry.gitlab.com/linuxserver.io/docker-jenkins-builder/lsiodev-buildcache,ghcr.io/linuxserver/lsiodev-buildcache,quay.io/linuxserver.io/lsiodev-buildcache'
|
||||||
env.CITEST_IMAGETAG = 'develop'
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -258,7 +256,6 @@ pipeline {
|
|||||||
env.CODE_URL = 'https://github.com/' + env.LS_USER + '/' + env.LS_REPO + '/pull/' + env.PULL_REQUEST
|
env.CODE_URL = 'https://github.com/' + env.LS_USER + '/' + env.LS_REPO + '/pull/' + env.PULL_REQUEST
|
||||||
env.DOCKERHUB_LINK = 'https://hub.docker.com/r/' + env.PR_DOCKERHUB_IMAGE + '/tags/'
|
env.DOCKERHUB_LINK = 'https://hub.docker.com/r/' + env.PR_DOCKERHUB_IMAGE + '/tags/'
|
||||||
env.BUILDCACHE = 'docker.io/lsiodev/buildcache,registry.gitlab.com/linuxserver.io/docker-jenkins-builder/lsiodev-buildcache,ghcr.io/linuxserver/lsiodev-buildcache,quay.io/linuxserver.io/lsiodev-buildcache'
|
env.BUILDCACHE = 'docker.io/lsiodev/buildcache,registry.gitlab.com/linuxserver.io/docker-jenkins-builder/lsiodev-buildcache,ghcr.io/linuxserver/lsiodev-buildcache,quay.io/linuxserver.io/lsiodev-buildcache'
|
||||||
env.CITEST_IMAGETAG = 'develop'
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -281,7 +278,7 @@ pipeline {
|
|||||||
-v ${WORKSPACE}:/mnt \
|
-v ${WORKSPACE}:/mnt \
|
||||||
-e AWS_ACCESS_KEY_ID=\"${S3_KEY}\" \
|
-e AWS_ACCESS_KEY_ID=\"${S3_KEY}\" \
|
||||||
-e AWS_SECRET_ACCESS_KEY=\"${S3_SECRET}\" \
|
-e AWS_SECRET_ACCESS_KEY=\"${S3_SECRET}\" \
|
||||||
ghcr.io/linuxserver/baseimage-alpine:3.23 s6-envdir -fn -- /var/run/s6/container_environment /bin/bash -c "\
|
ghcr.io/linuxserver/baseimage-alpine:3.20 s6-envdir -fn -- /var/run/s6/container_environment /bin/bash -c "\
|
||||||
apk add --no-cache python3 && \
|
apk add --no-cache python3 && \
|
||||||
python3 -m venv /lsiopy && \
|
python3 -m venv /lsiopy && \
|
||||||
pip install --no-cache-dir -U pip && \
|
pip install --no-cache-dir -U pip && \
|
||||||
@ -553,16 +550,13 @@ pipeline {
|
|||||||
echo $GITHUB_TOKEN | docker login ghcr.io -u LinuxServer-CI --password-stdin
|
echo $GITHUB_TOKEN | docker login ghcr.io -u LinuxServer-CI --password-stdin
|
||||||
echo $GITLAB_TOKEN | docker login registry.gitlab.com -u LinuxServer.io --password-stdin
|
echo $GITLAB_TOKEN | docker login registry.gitlab.com -u LinuxServer.io --password-stdin
|
||||||
echo $QUAYPASS | docker login quay.io -u $QUAYUSER --password-stdin
|
echo $QUAYPASS | docker login quay.io -u $QUAYUSER --password-stdin
|
||||||
|
|
||||||
if [[ "${PACKAGE_CHECK}" != "true" ]]; then
|
if [[ "${PACKAGE_CHECK}" != "true" ]]; then
|
||||||
declare -A pids
|
|
||||||
IFS=',' read -ra CACHE <<< "$BUILDCACHE"
|
IFS=',' read -ra CACHE <<< "$BUILDCACHE"
|
||||||
for i in "${CACHE[@]}"; do
|
for i in "${CACHE[@]}"; do
|
||||||
docker push ${i}:amd64-${COMMIT_SHA}-${BUILD_NUMBER} &
|
docker push ${i}:amd64-${COMMIT_SHA}-${BUILD_NUMBER} &
|
||||||
pids[$!]="$i"
|
|
||||||
done
|
done
|
||||||
for p in "${!pids[@]}"; do
|
for p in $(jobs -p); do
|
||||||
wait "$p" || { [[ "${pids[$p]}" != *"quay.io"* ]] && exit 1; }
|
wait "$p" || { echo "job $p failed" >&2; exit 1; }
|
||||||
done
|
done
|
||||||
fi
|
fi
|
||||||
'''
|
'''
|
||||||
@ -622,16 +616,13 @@ pipeline {
|
|||||||
echo $GITHUB_TOKEN | docker login ghcr.io -u LinuxServer-CI --password-stdin
|
echo $GITHUB_TOKEN | docker login ghcr.io -u LinuxServer-CI --password-stdin
|
||||||
echo $GITLAB_TOKEN | docker login registry.gitlab.com -u LinuxServer.io --password-stdin
|
echo $GITLAB_TOKEN | docker login registry.gitlab.com -u LinuxServer.io --password-stdin
|
||||||
echo $QUAYPASS | docker login quay.io -u $QUAYUSER --password-stdin
|
echo $QUAYPASS | docker login quay.io -u $QUAYUSER --password-stdin
|
||||||
|
|
||||||
if [[ "${PACKAGE_CHECK}" != "true" ]]; then
|
if [[ "${PACKAGE_CHECK}" != "true" ]]; then
|
||||||
declare -A pids
|
|
||||||
IFS=',' read -ra CACHE <<< "$BUILDCACHE"
|
IFS=',' read -ra CACHE <<< "$BUILDCACHE"
|
||||||
for i in "${CACHE[@]}"; do
|
for i in "${CACHE[@]}"; do
|
||||||
docker push ${i}:amd64-${COMMIT_SHA}-${BUILD_NUMBER} &
|
docker push ${i}:amd64-${COMMIT_SHA}-${BUILD_NUMBER} &
|
||||||
pids[$!]="$i"
|
|
||||||
done
|
done
|
||||||
for p in "${!pids[@]}"; do
|
for p in $(jobs -p); do
|
||||||
wait "$p" || { [[ "${pids[$p]}" != *"quay.io"* ]] && exit 1; }
|
wait "$p" || { echo "job $p failed" >&2; exit 1; }
|
||||||
done
|
done
|
||||||
fi
|
fi
|
||||||
'''
|
'''
|
||||||
@ -685,14 +676,12 @@ pipeline {
|
|||||||
echo $GITLAB_TOKEN | docker login registry.gitlab.com -u LinuxServer.io --password-stdin
|
echo $GITLAB_TOKEN | docker login registry.gitlab.com -u LinuxServer.io --password-stdin
|
||||||
echo $QUAYPASS | docker login quay.io -u $QUAYUSER --password-stdin
|
echo $QUAYPASS | docker login quay.io -u $QUAYUSER --password-stdin
|
||||||
if [[ "${PACKAGE_CHECK}" != "true" ]]; then
|
if [[ "${PACKAGE_CHECK}" != "true" ]]; then
|
||||||
declare -A pids
|
|
||||||
IFS=',' read -ra CACHE <<< "$BUILDCACHE"
|
IFS=',' read -ra CACHE <<< "$BUILDCACHE"
|
||||||
for i in "${CACHE[@]}"; do
|
for i in "${CACHE[@]}"; do
|
||||||
docker push ${i}:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER} &
|
docker push ${i}:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER} &
|
||||||
pids[$!]="$i"
|
|
||||||
done
|
done
|
||||||
for p in "${!pids[@]}"; do
|
for p in $(jobs -p); do
|
||||||
wait "$p" || { [[ "${pids[$p]}" != *"quay.io"* ]] && exit 1; }
|
wait "$p" || { echo "job $p failed" >&2; exit 1; }
|
||||||
done
|
done
|
||||||
fi
|
fi
|
||||||
'''
|
'''
|
||||||
@ -817,7 +806,7 @@ pipeline {
|
|||||||
CI_DOCKERENV="LSIO_FIRST_PARTY=true"
|
CI_DOCKERENV="LSIO_FIRST_PARTY=true"
|
||||||
fi
|
fi
|
||||||
fi
|
fi
|
||||||
docker pull ghcr.io/linuxserver/ci:${CITEST_IMAGETAG}
|
docker pull ghcr.io/linuxserver/ci:latest
|
||||||
if [ "${MULTIARCH}" == "true" ]; then
|
if [ "${MULTIARCH}" == "true" ]; then
|
||||||
docker pull ghcr.io/linuxserver/lsiodev-buildcache:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER} --platform=arm64
|
docker pull ghcr.io/linuxserver/lsiodev-buildcache:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER} --platform=arm64
|
||||||
docker tag ghcr.io/linuxserver/lsiodev-buildcache:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER} ${IMAGE}:arm64v8-${META_TAG}
|
docker tag ghcr.io/linuxserver/lsiodev-buildcache:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER} ${IMAGE}:arm64v8-${META_TAG}
|
||||||
@ -841,9 +830,7 @@ pipeline {
|
|||||||
-e WEB_PATH=\"${CI_WEBPATH}\" \
|
-e WEB_PATH=\"${CI_WEBPATH}\" \
|
||||||
-e NODE_NAME=\"${NODE_NAME}\" \
|
-e NODE_NAME=\"${NODE_NAME}\" \
|
||||||
-e SYFT_IMAGE_TAG=\"${CI_SYFT_IMAGE_TAG:-${SYFT_IMAGE_TAG}}\" \
|
-e SYFT_IMAGE_TAG=\"${CI_SYFT_IMAGE_TAG:-${SYFT_IMAGE_TAG}}\" \
|
||||||
-e COMMIT_SHA=\"${COMMIT_SHA}\" \
|
-t ghcr.io/linuxserver/ci:latest \
|
||||||
-e BUILD_NUMBER=\"${BUILD_NUMBER}\" \
|
|
||||||
-t ghcr.io/linuxserver/ci:${CITEST_IMAGETAG} \
|
|
||||||
python3 test_build.py'''
|
python3 test_build.py'''
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@ -869,11 +856,9 @@ pipeline {
|
|||||||
CACHEIMAGE=${i}
|
CACHEIMAGE=${i}
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
docker buildx imagetools create --prefer-index=false -t ${PUSHIMAGE}:${META_TAG} -t ${PUSHIMAGE}:latest -t ${PUSHIMAGE}:${EXT_RELEASE_TAG} ${CACHEIMAGE}:amd64-${COMMIT_SHA}-${BUILD_NUMBER} || \
|
docker buildx imagetools create --prefer-index=false -t ${PUSHIMAGE}:${META_TAG} -t ${PUSHIMAGE}:latest -t ${PUSHIMAGE}:${EXT_RELEASE_TAG} ${CACHEIMAGE}:amd64-${COMMIT_SHA}-${BUILD_NUMBER}
|
||||||
{ if [[ "${PUSHIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
|
|
||||||
if [ -n "${SEMVER}" ]; then
|
if [ -n "${SEMVER}" ]; then
|
||||||
docker buildx imagetools create --prefer-index=false -t ${PUSHIMAGE}:${SEMVER} ${CACHEIMAGE}:amd64-${COMMIT_SHA}-${BUILD_NUMBER} || \
|
docker buildx imagetools create --prefer-index=false -t ${PUSHIMAGE}:${SEMVER} ${CACHEIMAGE}:amd64-${COMMIT_SHA}-${BUILD_NUMBER}
|
||||||
{ if [[ "${PUSHIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
|
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
'''
|
'''
|
||||||
@ -898,27 +883,20 @@ pipeline {
|
|||||||
CACHEIMAGE=${i}
|
CACHEIMAGE=${i}
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
docker buildx imagetools create --prefer-index=false -t ${MANIFESTIMAGE}:amd64-${META_TAG} -t ${MANIFESTIMAGE}:amd64-latest -t ${MANIFESTIMAGE}:amd64-${EXT_RELEASE_TAG} ${CACHEIMAGE}:amd64-${COMMIT_SHA}-${BUILD_NUMBER} || \
|
docker buildx imagetools create --prefer-index=false -t ${MANIFESTIMAGE}:amd64-${META_TAG} -t ${MANIFESTIMAGE}:amd64-latest -t ${MANIFESTIMAGE}:amd64-${EXT_RELEASE_TAG} ${CACHEIMAGE}:amd64-${COMMIT_SHA}-${BUILD_NUMBER}
|
||||||
{ if [[ "${MANIFESTIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
|
docker buildx imagetools create --prefer-index=false -t ${MANIFESTIMAGE}:arm64v8-${META_TAG} -t ${MANIFESTIMAGE}:arm64v8-latest -t ${MANIFESTIMAGE}:arm64v8-${EXT_RELEASE_TAG} ${CACHEIMAGE}:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER}
|
||||||
docker buildx imagetools create --prefer-index=false -t ${MANIFESTIMAGE}:arm64v8-${META_TAG} -t ${MANIFESTIMAGE}:arm64v8-latest -t ${MANIFESTIMAGE}:arm64v8-${EXT_RELEASE_TAG} ${CACHEIMAGE}:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER} || \
|
|
||||||
{ if [[ "${MANIFESTIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
|
|
||||||
if [ -n "${SEMVER}" ]; then
|
if [ -n "${SEMVER}" ]; then
|
||||||
docker buildx imagetools create --prefer-index=false -t ${MANIFESTIMAGE}:amd64-${SEMVER} ${CACHEIMAGE}:amd64-${COMMIT_SHA}-${BUILD_NUMBER} || \
|
docker buildx imagetools create --prefer-index=false -t ${MANIFESTIMAGE}:amd64-${SEMVER} ${CACHEIMAGE}:amd64-${COMMIT_SHA}-${BUILD_NUMBER}
|
||||||
{ if [[ "${MANIFESTIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
|
docker buildx imagetools create --prefer-index=false -t ${MANIFESTIMAGE}:arm64v8-${SEMVER} ${CACHEIMAGE}:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER}
|
||||||
docker buildx imagetools create --prefer-index=false -t ${MANIFESTIMAGE}:arm64v8-${SEMVER} ${CACHEIMAGE}:arm64v8-${COMMIT_SHA}-${BUILD_NUMBER} || \
|
|
||||||
{ if [[ "${MANIFESTIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
|
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
for MANIFESTIMAGE in "${IMAGE}" "${GITLABIMAGE}" "${GITHUBIMAGE}" "${QUAYIMAGE}"; do
|
for MANIFESTIMAGE in "${IMAGE}" "${GITLABIMAGE}" "${GITHUBIMAGE}" "${QUAYIMAGE}"; do
|
||||||
docker buildx imagetools create -t ${MANIFESTIMAGE}:latest ${MANIFESTIMAGE}:amd64-latest ${MANIFESTIMAGE}:arm64v8-latest || \
|
docker buildx imagetools create -t ${MANIFESTIMAGE}:latest ${MANIFESTIMAGE}:amd64-latest ${MANIFESTIMAGE}:arm64v8-latest
|
||||||
{ if [[ "${MANIFESTIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
|
docker buildx imagetools create -t ${MANIFESTIMAGE}:${META_TAG} ${MANIFESTIMAGE}:amd64-${META_TAG} ${MANIFESTIMAGE}:arm64v8-${META_TAG}
|
||||||
docker buildx imagetools create -t ${MANIFESTIMAGE}:${META_TAG} ${MANIFESTIMAGE}:amd64-${META_TAG} ${MANIFESTIMAGE}:arm64v8-${META_TAG} || \
|
|
||||||
{ if [[ "${MANIFESTIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
|
docker buildx imagetools create -t ${MANIFESTIMAGE}:${EXT_RELEASE_TAG} ${MANIFESTIMAGE}:amd64-${EXT_RELEASE_TAG} ${MANIFESTIMAGE}:arm64v8-${EXT_RELEASE_TAG}
|
||||||
docker buildx imagetools create -t ${MANIFESTIMAGE}:${EXT_RELEASE_TAG} ${MANIFESTIMAGE}:amd64-${EXT_RELEASE_TAG} ${MANIFESTIMAGE}:arm64v8-${EXT_RELEASE_TAG} || \
|
|
||||||
{ if [[ "${MANIFESTIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
|
|
||||||
if [ -n "${SEMVER}" ]; then
|
if [ -n "${SEMVER}" ]; then
|
||||||
docker buildx imagetools create -t ${MANIFESTIMAGE}:${SEMVER} ${MANIFESTIMAGE}:amd64-${SEMVER} ${MANIFESTIMAGE}:arm64v8-${SEMVER} || \
|
docker buildx imagetools create -t ${MANIFESTIMAGE}:${SEMVER} ${MANIFESTIMAGE}:amd64-${SEMVER} ${MANIFESTIMAGE}:arm64v8-${SEMVER}
|
||||||
{ if [[ "${MANIFESTIMAGE}" != "${QUAYIMAGE}" ]]; then exit 1; fi; }
|
|
||||||
fi
|
fi
|
||||||
done
|
done
|
||||||
'''
|
'''
|
||||||
@ -936,41 +914,23 @@ pipeline {
|
|||||||
environment name: 'EXIT_STATUS', value: ''
|
environment name: 'EXIT_STATUS', value: ''
|
||||||
}
|
}
|
||||||
steps {
|
steps {
|
||||||
|
echo "Pushing New tag for current commit ${META_TAG}"
|
||||||
|
sh '''curl -H "Authorization: token ${GITHUB_TOKEN}" -X POST https://api.github.com/repos/${LS_USER}/${LS_REPO}/git/tags \
|
||||||
|
-d '{"tag":"'${META_TAG}'",\
|
||||||
|
"object": "'${COMMIT_SHA}'",\
|
||||||
|
"message": "Tagging Release '${EXT_RELEASE_CLEAN}'-ls'${LS_TAG_NUMBER}' to master",\
|
||||||
|
"type": "commit",\
|
||||||
|
"tagger": {"name": "LinuxServer-CI","email": "ci@linuxserver.io","date": "'${GITHUB_DATE}'"}}' '''
|
||||||
|
echo "Pushing New release for Tag"
|
||||||
sh '''#! /bin/bash
|
sh '''#! /bin/bash
|
||||||
echo "Auto-generating release notes"
|
|
||||||
if [ "$(git tag --points-at HEAD)" != "" ]; then
|
|
||||||
echo "Existing tag points to current commit, suggesting no new LS changes"
|
|
||||||
AUTO_RELEASE_NOTES="No changes"
|
|
||||||
else
|
|
||||||
AUTO_RELEASE_NOTES=$(curl -fsL -H "Authorization: token ${GITHUB_TOKEN}" -H "Accept: application/vnd.github+json" -X POST https://api.github.com/repos/${LS_USER}/${LS_REPO}/releases/generate-notes \
|
|
||||||
-d '{"tag_name":"'${META_TAG}'",\
|
|
||||||
"target_commitish": "master"}' \
|
|
||||||
| jq -r '.body' | sed 's|## What.s Changed||')
|
|
||||||
fi
|
|
||||||
echo "Pushing New tag for current commit ${META_TAG}"
|
|
||||||
curl -H "Authorization: token ${GITHUB_TOKEN}" -X POST https://api.github.com/repos/${LS_USER}/${LS_REPO}/git/tags \
|
|
||||||
-d '{"tag":"'${META_TAG}'",\
|
|
||||||
"object": "'${COMMIT_SHA}'",\
|
|
||||||
"message": "Tagging Release '${EXT_RELEASE_CLEAN}'-ls'${LS_TAG_NUMBER}' to master",\
|
|
||||||
"type": "commit",\
|
|
||||||
"tagger": {"name": "LinuxServer-CI","email": "ci@linuxserver.io","date": "'${GITHUB_DATE}'"}}'
|
|
||||||
echo "Pushing New release for Tag"
|
|
||||||
echo "Updating base packages to ${PACKAGE_TAG}" > releasebody.json
|
echo "Updating base packages to ${PACKAGE_TAG}" > releasebody.json
|
||||||
jq -n \
|
echo '{"tag_name":"'${META_TAG}'",\
|
||||||
--arg tag_name "$META_TAG" \
|
"target_commitish": "master",\
|
||||||
--arg target_commitish "master" \
|
"name": "'${META_TAG}'",\
|
||||||
--arg ci_url "${CI_URL:-N/A}" \
|
"body": "**CI Report:**\\n\\n'${CI_URL:-N/A}'\\n\\n**LinuxServer Changes:**\\n\\n'${LS_RELEASE_NOTES}'\\n\\n**Remote Changes:**\\n\\n' > start
|
||||||
--arg ls_notes "$AUTO_RELEASE_NOTES" \
|
printf '","draft": false,"prerelease": false}' >> releasebody.json
|
||||||
--arg remote_notes "$(cat releasebody.json)" \
|
paste -d'\\0' start releasebody.json > releasebody.json.done
|
||||||
'{
|
curl -H "Authorization: token ${GITHUB_TOKEN}" -X POST https://api.github.com/repos/${LS_USER}/${LS_REPO}/releases -d @releasebody.json.done'''
|
||||||
"tag_name": $tag_name,
|
|
||||||
"target_commitish": $target_commitish,
|
|
||||||
"name": $tag_name,
|
|
||||||
"body": ("**CI Report:**\\n\\n" + $ci_url + "\\n\\n**LinuxServer Changes:**\\n\\n" + $ls_notes + "\\n\\n**Remote Changes:**\\n\\n" + $remote_notes),
|
|
||||||
"draft": false,
|
|
||||||
"prerelease": false }' > releasebody.json.done
|
|
||||||
curl -H "Authorization: token ${GITHUB_TOKEN}" -X POST https://api.github.com/repos/${LS_USER}/${LS_REPO}/releases -d @releasebody.json.done
|
|
||||||
'''
|
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// Add protection to the release branch
|
// Add protection to the release branch
|
||||||
|
|||||||
332
ci/ci.py
332
ci/ci.py
@ -10,13 +10,11 @@ import logging
|
|||||||
from logging import Logger
|
from logging import Logger
|
||||||
import mimetypes
|
import mimetypes
|
||||||
import json
|
import json
|
||||||
import subprocess
|
|
||||||
from requests.adapters import HTTPAdapter
|
from requests.adapters import HTTPAdapter
|
||||||
from urllib3.util.retry import Retry
|
from urllib3.util.retry import Retry
|
||||||
from functools import wraps
|
from functools import wraps
|
||||||
from typing import Callable, Any, Literal
|
from typing import Callable, Any, Literal
|
||||||
from textwrap import dedent
|
from textwrap import dedent
|
||||||
import enum
|
|
||||||
|
|
||||||
import boto3
|
import boto3
|
||||||
import requests
|
import requests
|
||||||
@ -64,42 +62,6 @@ def deprecated(reason: str):
|
|||||||
return wrapper
|
return wrapper
|
||||||
return deprecated_decorator
|
return deprecated_decorator
|
||||||
|
|
||||||
class CITests(enum.Enum):
|
|
||||||
"""Enum for the different CI tests"""
|
|
||||||
CONTAINER_START = "Container Start"
|
|
||||||
CREATE_SBOM = "Create SBOM"
|
|
||||||
CREATE_BUILDX_SBOM = "Create Buildx SBOM"
|
|
||||||
CREATE_SYFT_SBOM = "Create Syft SBOM"
|
|
||||||
CAPTURE_SCREENSHOT = "Capture Screenshot"
|
|
||||||
GET_BUILD_INFO = "Get Build Info"
|
|
||||||
|
|
||||||
class CITestResult(enum.Enum):
|
|
||||||
"""Enum for the different CI test results"""
|
|
||||||
PASS = "PASS"
|
|
||||||
FAIL = "FAIL"
|
|
||||||
ERROR = "ERROR"
|
|
||||||
|
|
||||||
class CIReportResult(enum.Enum):
|
|
||||||
"""Enum for the different CI report results"""
|
|
||||||
PASS = "PASS"
|
|
||||||
FAIL = "FAIL"
|
|
||||||
ERROR = "ERROR"
|
|
||||||
|
|
||||||
class Platform(enum.Enum):
|
|
||||||
"""Enum for the different platforms"""
|
|
||||||
AMD64 = "amd64"
|
|
||||||
ARM64 = "arm64"
|
|
||||||
ARM32 = "arm"
|
|
||||||
RISCV64 = "riscv64"
|
|
||||||
UNKNOWN = "amd64"
|
|
||||||
|
|
||||||
class BuildCacheTag(enum.Enum):
|
|
||||||
"""Enum for the different build cache tags"""
|
|
||||||
AMD64 = "amd64"
|
|
||||||
ARM64 = "arm64v8"
|
|
||||||
RISCV64 = "riscv64"
|
|
||||||
UNKNOWN = "amd64"
|
|
||||||
|
|
||||||
class SetEnvs():
|
class SetEnvs():
|
||||||
"""Simple helper class that sets up the ENVs"""
|
"""Simple helper class that sets up the ENVs"""
|
||||||
def __init__(self) -> None:
|
def __init__(self) -> None:
|
||||||
@ -126,9 +88,6 @@ class SetEnvs():
|
|||||||
self.bucket: str = os.environ.get("S3_BUCKET", "ci-tests.linuxserver.io")
|
self.bucket: str = os.environ.get("S3_BUCKET", "ci-tests.linuxserver.io")
|
||||||
self.release_tag: str = os.environ.get("RELEASE_TAG", "latest")
|
self.release_tag: str = os.environ.get("RELEASE_TAG", "latest")
|
||||||
self.syft_image_tag: str = os.environ.get("SYFT_IMAGE_TAG", "v1.26.1")
|
self.syft_image_tag: str = os.environ.get("SYFT_IMAGE_TAG", "v1.26.1")
|
||||||
self.commit_sha: str = os.environ.get("COMMIT_SHA", "")
|
|
||||||
self.build_number: str = os.environ.get("BUILD_NUMBER", "")
|
|
||||||
self.build_cache_registry: str = os.environ.get("BUILD_CACHE_REGISTRY", "ghcr.io/linuxserver/lsiodev-buildcache")
|
|
||||||
|
|
||||||
if os.environ.get("DELAY_START"):
|
if os.environ.get("DELAY_START"):
|
||||||
self.logger.warning("DELAY_START env is obsolete, and not in use anymore")
|
self.logger.warning("DELAY_START env is obsolete, and not in use anymore")
|
||||||
@ -179,9 +138,6 @@ class SetEnvs():
|
|||||||
S3_REGION: '{os.environ.get("S3_REGION")}'
|
S3_REGION: '{os.environ.get("S3_REGION")}'
|
||||||
S3_BUCKET: '{os.environ.get("S3_BUCKET")}'
|
S3_BUCKET: '{os.environ.get("S3_BUCKET")}'
|
||||||
SYFT_IMAGE_TAG: '{os.environ.get("SYFT_IMAGE_TAG")}'
|
SYFT_IMAGE_TAG: '{os.environ.get("SYFT_IMAGE_TAG")}'
|
||||||
COMMIT_SHA: '{os.environ.get("COMMIT_SHA")}'
|
|
||||||
BUILD_NUMBER: '{os.environ.get("BUILD_NUMBER")}'
|
|
||||||
BUILD_CACHE_REGISTRY: '{os.environ.get("BUILD_CACHE_REGISTRY")}'
|
|
||||||
Docker Engine Version: '{self.get_docker_engine_version()}'
|
Docker Engine Version: '{self.get_docker_engine_version()}'
|
||||||
""")
|
""")
|
||||||
self.logger.info(env_data)
|
self.logger.info(env_data)
|
||||||
@ -313,7 +269,7 @@ class CI(SetEnvs):
|
|||||||
self.tags = list(self.tags_env.split("|"))
|
self.tags = list(self.tags_env.split("|"))
|
||||||
self.tag_report_tests:dict[str,dict[str,dict]] = {tag: {"test":{}} for tag in self.tags} # Adds all the tags as keys with an empty dict as value to the dict
|
self.tag_report_tests:dict[str,dict[str,dict]] = {tag: {"test":{}} for tag in self.tags} # Adds all the tags as keys with an empty dict as value to the dict
|
||||||
self.report_containers: dict[str,dict[str,dict]] = {}
|
self.report_containers: dict[str,dict[str,dict]] = {}
|
||||||
self.report_status = CIReportResult.PASS
|
self.report_status = "PASS"
|
||||||
self.outdir: str = f"{os.path.dirname(os.path.realpath(__file__))}/output/{self.image}/{self.meta_tag}"
|
self.outdir: str = f"{os.path.dirname(os.path.realpath(__file__))}/output/{self.image}/{self.meta_tag}"
|
||||||
os.makedirs(self.outdir, exist_ok=True)
|
os.makedirs(self.outdir, exist_ok=True)
|
||||||
self.s3_client = self.create_s3_client()
|
self.s3_client = self.create_s3_client()
|
||||||
@ -364,31 +320,31 @@ class CI(SetEnvs):
|
|||||||
|
|
||||||
# Run these tests in parallel so the runtime data is more accurate.
|
# Run these tests in parallel so the runtime data is more accurate.
|
||||||
with ThreadPoolExecutor(max_workers=2,thread_name_prefix=thread_name) as executor:
|
with ThreadPoolExecutor(max_workers=2,thread_name_prefix=thread_name) as executor:
|
||||||
future_sbom: Future[str|CITestResult] = executor.submit(self.make_sbom, tag)
|
future_sbom: Future[str] = executor.submit(self.generate_sbom, tag)
|
||||||
future_logs: Future[bool] = executor.submit(self.watch_container_logs, container, tag)
|
future_logs: Future[bool] = executor.submit(self.watch_container_logs, container, tag)
|
||||||
|
|
||||||
sbom: str|CITestResult = future_sbom.result(self.sbom_timeout + 5) # Set a thread timeout if the function for some reason hangs
|
sbom: str = future_sbom.result(self.sbom_timeout + 5) # Set a thread timeout if the function for some reason hangs
|
||||||
logsfound: bool = future_logs.result(self.logs_timeout + 5) # Set a thread timeout if the function for some reason hangs
|
logsfound: bool = future_logs.result(self.logs_timeout + 5) # Set a thread timeout if the function for some reason hangs
|
||||||
build_info: dict[str,str] = self.get_build_info(container,tag) # Get the image build info
|
build_info: dict = self.get_build_info(container,tag) # Get the image build info
|
||||||
|
|
||||||
if not logsfound:
|
if not logsfound:
|
||||||
self.logger.error("Test of %s FAILED after %.2f seconds", tag, time.time() - start_time)
|
self.logger.error("Test of %s FAILED after %.2f seconds", tag, time.time() - start_time)
|
||||||
self._endtest(container, tag, build_info, sbom, False, start_time)
|
self._endtest(container, tag, build_info, sbom, False, start_time)
|
||||||
return
|
return
|
||||||
|
|
||||||
if build_info["version"] == CITestResult.ERROR.value:
|
if build_info["version"] == "ERROR":
|
||||||
self.logger.error("Test of %s FAILED after %.2f seconds", tag, time.time() - start_time)
|
self.logger.error("Test of %s FAILED after %.2f seconds", tag, time.time() - start_time)
|
||||||
self._endtest(container, tag, build_info, sbom, False, start_time)
|
self._endtest(container, tag, build_info, sbom, False, start_time)
|
||||||
return
|
return
|
||||||
|
|
||||||
if sbom == CITestResult.ERROR:
|
if sbom == "ERROR":
|
||||||
self.logger.error("Test of %s FAILED after %.2f seconds", tag, time.time() - start_time)
|
self.logger.error("Test of %s FAILED after %.2f seconds", tag, time.time() - start_time)
|
||||||
self._endtest(container, tag, build_info, sbom, False, start_time)
|
self._endtest(container, tag, build_info, sbom, False, start_time)
|
||||||
return
|
return
|
||||||
|
|
||||||
# Screenshot the web interface and check connectivity
|
# Screenshot the web interface and check connectivity
|
||||||
screenshot_success, browser_logs = self.take_screenshot(container, tag)
|
screenshot_success, browser_logs = self.take_screenshot(container, tag)
|
||||||
if not screenshot_success and self.get_platform(tag) == Platform.AMD64.value:
|
if not screenshot_success and self.get_platform(tag) == "amd64":
|
||||||
self.logger.error("Test of %s FAILED after %.2f seconds", tag, time.time() - start_time)
|
self.logger.error("Test of %s FAILED after %.2f seconds", tag, time.time() - start_time)
|
||||||
self._endtest(container, tag, build_info, sbom, False, start_time, browser_logs)
|
self._endtest(container, tag, build_info, sbom, False, start_time, browser_logs)
|
||||||
return
|
return
|
||||||
@ -397,7 +353,7 @@ class CI(SetEnvs):
|
|||||||
self.logger.success("Test of %s PASSED after %.2f seconds", tag, time.time() - start_time)
|
self.logger.success("Test of %s PASSED after %.2f seconds", tag, time.time() - start_time)
|
||||||
return
|
return
|
||||||
|
|
||||||
def _endtest(self, container:Container, tag:str, build_info:dict[str,str], packages:str|CITestResult, test_success: bool, start_time:float|int = 0.0, browser_logs: str = "") -> None:
|
def _endtest(self, container:Container, tag:str, build_info:dict[str,str], packages:str, test_success: bool, start_time:float|int = 0.0, browser_logs: str = "") -> None:
|
||||||
"""End the test with as much info as we have and append to the report.
|
"""End the test with as much info as we have and append to the report.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@ -413,12 +369,10 @@ class CI(SetEnvs):
|
|||||||
runtime = "-"
|
runtime = "-"
|
||||||
if isinstance(start_time,(float, int)):
|
if isinstance(start_time,(float, int)):
|
||||||
runtime = f"{time.time() - start_time:.2f}s"
|
runtime = f"{time.time() - start_time:.2f}s"
|
||||||
if isinstance(packages, CITestResult):
|
|
||||||
packages = packages.value
|
|
||||||
logblob: str = container.logs(timestamps=True).decode("utf-8")
|
logblob: str = container.logs(timestamps=True).decode("utf-8")
|
||||||
self.create_html_ansi_file(logblob, tag, "log") # Generate an html container log file based on the latest logs
|
self.create_html_ansi_file(logblob, tag, "log") # Generate an html container log file based on the latest logs
|
||||||
try:
|
try:
|
||||||
container.remove(force=True)
|
container.remove(force="true")
|
||||||
except APIError:
|
except APIError:
|
||||||
self.logger.exception("Failed to remove container %s",tag)
|
self.logger.exception("Failed to remove container %s",tag)
|
||||||
warning_texts: dict[str, str] = {
|
warning_texts: dict[str, str] = {
|
||||||
@ -474,15 +428,15 @@ class CI(SetEnvs):
|
|||||||
platform: str = tag[:5]
|
platform: str = tag[:5]
|
||||||
match platform:
|
match platform:
|
||||||
case "amd64":
|
case "amd64":
|
||||||
return Platform.AMD64.value
|
return "amd64"
|
||||||
case "arm64":
|
case "arm64":
|
||||||
return Platform.ARM64.value
|
return "arm64"
|
||||||
case "arm32":
|
case "arm32":
|
||||||
return Platform.ARM32.value
|
return "arm"
|
||||||
case "riscv":
|
case "riscv":
|
||||||
return Platform.RISCV64.value
|
return "riscv64"
|
||||||
case _:
|
case _:
|
||||||
return Platform.UNKNOWN.value
|
return "amd64"
|
||||||
|
|
||||||
@deprecated(reason="Use generate_sbom instead")
|
@deprecated(reason="Use generate_sbom instead")
|
||||||
def export_package_info(self, container:Container, tag:str) -> str:
|
def export_package_info(self, container:Container, tag:str) -> str:
|
||||||
@ -521,8 +475,7 @@ class CI(SetEnvs):
|
|||||||
"message":str(error)}.items())))
|
"message":str(error)}.items())))
|
||||||
self.report_status = "FAIL"
|
self.report_status = "FAIL"
|
||||||
return packages
|
return packages
|
||||||
|
|
||||||
@deprecated(reason="Use make_sbom instead")
|
|
||||||
def generate_sbom(self, tag:str) -> str:
|
def generate_sbom(self, tag:str) -> str:
|
||||||
"""Generate the SBOM for the image tag.
|
"""Generate the SBOM for the image tag.
|
||||||
|
|
||||||
@ -569,188 +522,6 @@ class CI(SetEnvs):
|
|||||||
self.logger.exception("Failed to remove the syft container, %s",tag)
|
self.logger.exception("Failed to remove the syft container, %s",tag)
|
||||||
return "ERROR"
|
return "ERROR"
|
||||||
|
|
||||||
def make_sbom(self, tag: str) -> "str|CITestResult":
|
|
||||||
"""Generate the SBOM for the image tag. It will first try to use buildx imagetools inspect, if that fails it will fallback to using syft.
|
|
||||||
|
|
||||||
Creates the output file in `{self.outdir}/{tag}.sbom.html`
|
|
||||||
|
|
||||||
Args:
|
|
||||||
tag (str): The tag we are testing
|
|
||||||
Returns:
|
|
||||||
str: Return the output if successful otherwise "ERROR".
|
|
||||||
"""
|
|
||||||
start_time = time.time()
|
|
||||||
sbom: str | CITestResult = self.get_sbom_buildx_blob(tag)
|
|
||||||
if isinstance(sbom, str):
|
|
||||||
packages: list[dict[str, str]] = self.parse_buildx_sbom(sbom)
|
|
||||||
formatted_table: str | CITestResult = self.format_package_table(packages=packages)
|
|
||||||
if packages and formatted_table != CITestResult.ERROR:
|
|
||||||
self.create_html_ansi_file(formatted_table, tag, "sbom")
|
|
||||||
self._add_test_result(tag, CITests.CREATE_BUILDX_SBOM, CITestResult.PASS, "-", start_time)
|
|
||||||
self.logger.success("Create SBOM %s: PASSED after %.2f seconds", tag, time.time() - start_time)
|
|
||||||
return formatted_table
|
|
||||||
self.logger.error("No packages found in buildx SBOM for tag %s", tag)
|
|
||||||
self.report_status = CIReportResult.FAIL
|
|
||||||
self._add_test_result(tag, CITests.CREATE_BUILDX_SBOM, CITestResult.FAIL, "No packages found in buildx SBOM", start_time)
|
|
||||||
else:
|
|
||||||
self.logger.warning("Falling back to Syft for SBOM generation on tag %s", tag)
|
|
||||||
|
|
||||||
# Fallback to syft if buildx failed
|
|
||||||
sbom = self.get_sbom_syft(tag)
|
|
||||||
if sbom != CITestResult.ERROR:
|
|
||||||
self._add_test_result(tag, CITests.CREATE_SBOM, CITestResult.PASS, "-", start_time)
|
|
||||||
self.logger.success("Create SBOM %s: PASSED after %.2f seconds", tag, time.time() - start_time)
|
|
||||||
self.create_html_ansi_file(str(sbom),tag,"sbom")
|
|
||||||
return sbom
|
|
||||||
self.report_status = CIReportResult.FAIL
|
|
||||||
self._add_test_result(tag, CITests.CREATE_SBOM, CITestResult.FAIL, "Failed to generate SBOM with both buildx and syft", start_time)
|
|
||||||
return CITestResult.ERROR
|
|
||||||
|
|
||||||
def get_sbom_syft(self, tag: str) -> str | CITestResult:
|
|
||||||
"""Get the SBOM for the image tag using Syft.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
tag (str): The tag we are testing
|
|
||||||
Returns:
|
|
||||||
str: SBOM output if successful, otherwise "ERROR".
|
|
||||||
"""
|
|
||||||
start_time = time.time()
|
|
||||||
platform: str = self.get_platform(tag)
|
|
||||||
syft:Container = self.client.containers.run(image=f"ghcr.io/anchore/syft:{self.syft_image_tag}",command=f"{self.image}:{tag} --platform=linux/{platform}",
|
|
||||||
detach=True, volumes={"/var/run/docker.sock": {"bind": "/var/run/docker.sock", "mode": "rw"}})
|
|
||||||
self.logger.info("Creating SBOM package list on %s with syft version %s",tag,self.syft_image_tag)
|
|
||||||
logblob: str = ""
|
|
||||||
t_end: float = time.time() + self.sbom_timeout
|
|
||||||
self.logger.info("Tailing the Syft container logs for %s seconds looking the 'VERSION' message on tag: %s",self.sbom_timeout,tag)
|
|
||||||
while time.time() < t_end:
|
|
||||||
time.sleep(5)
|
|
||||||
try:
|
|
||||||
logblob: str = syft.logs().decode("utf-8")
|
|
||||||
if "VERSION" in logblob:
|
|
||||||
self.logger.info("Get Syft package versions for %s completed", tag)
|
|
||||||
self.logger.success("%s package list %s: PASSED after %.2f seconds", CITests.CREATE_SYFT_SBOM.value, tag, time.time() - start_time)
|
|
||||||
try:
|
|
||||||
syft.remove(force=True)
|
|
||||||
except Exception:
|
|
||||||
self.logger.exception("Failed to remove the Syft container, %s",tag)
|
|
||||||
return logblob
|
|
||||||
except (APIError,ContainerError,ImageNotFound):
|
|
||||||
self.logger.exception("Creating Syft SBOM package list on %s: FAIL", tag)
|
|
||||||
self.logger.error("Failed to generate Syft SBOM output on tag %s. SBOM output:\n%s",tag, logblob)
|
|
||||||
try:
|
|
||||||
syft.remove(force=True)
|
|
||||||
except Exception:
|
|
||||||
self.logger.exception("Failed to remove the Syft container, %s",tag)
|
|
||||||
return CITestResult.ERROR
|
|
||||||
|
|
||||||
def get_build_cache_url(self, tag: str) -> str:
|
|
||||||
"""Get the build cache URL for the given tag.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
tag (str): The tag we are testing
|
|
||||||
Returns:
|
|
||||||
str: The build cache URL ex `ghcr.io/linuxserver/lsiodev-buildcache:arm64v8-<COMMIT_SHA>-<BUILD_NUMBER>`
|
|
||||||
"""
|
|
||||||
platform: str = self.get_build_cache_platform(tag)
|
|
||||||
build_cache_url: str = f"{self.build_cache_registry}:{platform}-{self.commit_sha}-{self.build_number}"
|
|
||||||
return build_cache_url
|
|
||||||
|
|
||||||
def get_build_cache_platform(self, tag: str) -> str:
|
|
||||||
"""Get the build cache platform for the given tag.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
tag (str): The tag we are testing
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
str: The build cache platform
|
|
||||||
"""
|
|
||||||
|
|
||||||
platform = self.get_platform(tag)
|
|
||||||
match platform:
|
|
||||||
case Platform.AMD64.value:
|
|
||||||
return BuildCacheTag.AMD64.value
|
|
||||||
case Platform.ARM64.value:
|
|
||||||
return BuildCacheTag.ARM64.value
|
|
||||||
case Platform.RISCV64.value:
|
|
||||||
return BuildCacheTag.RISCV64.value
|
|
||||||
case _:
|
|
||||||
return BuildCacheTag.UNKNOWN.value
|
|
||||||
|
|
||||||
def get_sbom_buildx_blob(self, tag: str) -> str | CITestResult:
|
|
||||||
"""Get the SBOM for the image tag using docker buildx imagetools inspect.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
tag (str): The tag we are testing
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
str: SBOM output if successful, otherwise "CITestResult.ERROR".
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
image_ref:str = self.get_build_cache_url(tag)
|
|
||||||
self.logger.info("Generating SBOM for %s using buildx imagetools inspect", image_ref)
|
|
||||||
cmd = f'docker buildx imagetools inspect {image_ref} --format "{{{{ json (index .SBOM).SPDX.packages }}}}"'
|
|
||||||
result: subprocess.CompletedProcess[str] = subprocess.run(cmd, shell=True, capture_output=True, text=True, timeout=self.sbom_timeout, check=False)
|
|
||||||
if result.returncode == 0 and result.stdout.strip():
|
|
||||||
self.logger.success("SBOM generated for %s using buildx imagetools inspect.", tag)
|
|
||||||
return result.stdout.strip()
|
|
||||||
self.logger.error("Failed to generate buildx SBOM for %s: %s", tag, result.stderr)
|
|
||||||
return CITestResult.ERROR
|
|
||||||
except Exception:
|
|
||||||
self.logger.exception("Exception while running buildx imagetools inspect for %s", tag)
|
|
||||||
return CITestResult.ERROR
|
|
||||||
|
|
||||||
def parse_buildx_sbom(self, sbom: str) -> list[dict[str, str]]:
|
|
||||||
"""Parse the buildx imagetools inspect SBOM string and extract package information.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
sbom (str): The SBOM in JSON format.
|
|
||||||
Returns:
|
|
||||||
list[dict[str, str]]: A list of dictionaries containing package information.
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
sbom_data: dict[str, Any] = json.loads(sbom)
|
|
||||||
packages: list[dict[str, str]] = []
|
|
||||||
for item in sbom_data:
|
|
||||||
package_info = {
|
|
||||||
"name": item.get("name", ""),
|
|
||||||
"version": item.get("versionInfo", ""),
|
|
||||||
}
|
|
||||||
if package_info in packages:
|
|
||||||
continue
|
|
||||||
packages.append(package_info)
|
|
||||||
return packages
|
|
||||||
except json.JSONDecodeError:
|
|
||||||
self.logger.error("Failed to parse buildx imagetools inspect SBOM output.")
|
|
||||||
return []
|
|
||||||
|
|
||||||
def format_package_table(self, packages: list[dict[str, str]]) -> str | CITestResult:
|
|
||||||
"""Format a list of package dicts into a padded string table.
|
|
||||||
|
|
||||||
Args:
|
|
||||||
packages (list[dict[str, str]]): List of package dicts with 'name' and 'version' keys.
|
|
||||||
|
|
||||||
Returns:
|
|
||||||
str: Padded string table of package names and versions.
|
|
||||||
"""
|
|
||||||
try:
|
|
||||||
if not packages:
|
|
||||||
self.logger.error("No packages found to format.")
|
|
||||||
return CITestResult.ERROR
|
|
||||||
# Dynamically determine column widths based on longest name/version
|
|
||||||
name_width = max(len(pkg.get('name', '')) for pkg in packages) + 10
|
|
||||||
version_width = max(len(pkg.get('version', '')) for pkg in packages) + 10
|
|
||||||
lines: list[str] = []
|
|
||||||
header = f"{'NAME'.ljust(name_width)}{'VERSION'.ljust(version_width)}"
|
|
||||||
lines.append(header)
|
|
||||||
for pkg in packages:
|
|
||||||
name = pkg.get('name', '')[:name_width]
|
|
||||||
version = pkg.get('version', '')[:version_width]
|
|
||||||
lines.append(f"{name.ljust(name_width)}{version.ljust(version_width)}")
|
|
||||||
return "\n".join(lines)
|
|
||||||
except Exception:
|
|
||||||
self.logger.exception("Failed to format package table.")
|
|
||||||
return CITestResult.ERROR
|
|
||||||
|
|
||||||
@deprecated(reason="Use get_build_info instead")
|
@deprecated(reason="Use get_build_info instead")
|
||||||
def get_build_version(self,container:Container,tag:str) -> str:
|
def get_build_version(self,container:Container,tag:str) -> str:
|
||||||
"""Fetch the build version from the container object attributes.
|
"""Fetch the build version from the container object attributes.
|
||||||
@ -797,7 +568,7 @@ class CI(SetEnvs):
|
|||||||
case _:
|
case _:
|
||||||
return self.image
|
return self.image
|
||||||
|
|
||||||
def get_build_url(self, tag: str) -> str:
|
def get_build_url(self, tag) -> str:
|
||||||
"""Get the build url from the IMAGE env.
|
"""Get the build url from the IMAGE env.
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@ -839,6 +610,7 @@ class CI(SetEnvs):
|
|||||||
}
|
}
|
||||||
```
|
```
|
||||||
"""
|
"""
|
||||||
|
test = "Get build info"
|
||||||
start_time = time.time()
|
start_time = time.time()
|
||||||
try:
|
try:
|
||||||
self.logger.info("Fetching build info on tag: %s",tag)
|
self.logger.info("Fetching build info on tag: %s",tag)
|
||||||
@ -851,15 +623,15 @@ class CI(SetEnvs):
|
|||||||
"tag": tag,
|
"tag": tag,
|
||||||
"image": self.get_image_name()
|
"image": self.get_image_name()
|
||||||
}
|
}
|
||||||
self._add_test_result(tag, CITests.GET_BUILD_INFO, CITestResult.PASS, "-", start_time)
|
self._add_test_result(tag, test, "PASS", "-", start_time)
|
||||||
self.logger.success("Get build info on tag '%s': PASS", tag)
|
self.logger.success("Get build info on tag '%s': PASS", tag)
|
||||||
except (APIError,KeyError) as error:
|
except (APIError,KeyError) as error:
|
||||||
self.logger.exception("Get build info on tag '%s': FAIL", tag)
|
self.logger.exception("Get build info on tag '%s': FAIL", tag)
|
||||||
build_info = {"version": CITestResult.ERROR.value, "created": CITestResult.ERROR.value, "size": CITestResult.ERROR.value, "maintainer": CITestResult.ERROR.value}
|
build_info = {"version": "ERROR", "created": "ERROR", "size": "ERROR", "maintainer": "ERROR"}
|
||||||
if isinstance(error,KeyError):
|
if isinstance(error,KeyError):
|
||||||
error: str = f"KeyError: {error}"
|
error: str = f"KeyError: {error}"
|
||||||
self._add_test_result(tag, CITests.GET_BUILD_INFO, CITestResult.FAIL, str(error), start_time)
|
self._add_test_result(tag, test, "FAIL", str(error), start_time)
|
||||||
self.report_status = CIReportResult.FAIL
|
self.report_status = "FAIL"
|
||||||
return build_info
|
return build_info
|
||||||
|
|
||||||
def watch_container_logs(self, container:Container, tag:str) -> bool:
|
def watch_container_logs(self, container:Container, tag:str) -> bool:
|
||||||
@ -873,6 +645,7 @@ class CI(SetEnvs):
|
|||||||
Returns:
|
Returns:
|
||||||
bool: Return True if the "done" message is found, otherwise False.
|
bool: Return True if the "done" message is found, otherwise False.
|
||||||
"""
|
"""
|
||||||
|
test = "Container startup"
|
||||||
start_time = time.time()
|
start_time = time.time()
|
||||||
t_end: float = time.time() + self.logs_timeout
|
t_end: float = time.time() + self.logs_timeout
|
||||||
self.logger.info("Tailing the %s logs for %s seconds looking for the 'done' message", tag, self.logs_timeout)
|
self.logger.info("Tailing the %s logs for %s seconds looking for the 'done' message", tag, self.logs_timeout)
|
||||||
@ -880,20 +653,20 @@ class CI(SetEnvs):
|
|||||||
try:
|
try:
|
||||||
logblob: str = container.logs().decode("utf-8")
|
logblob: str = container.logs().decode("utf-8")
|
||||||
if "[services.d] done." in logblob or "[ls.io-init] done." in logblob:
|
if "[services.d] done." in logblob or "[ls.io-init] done." in logblob:
|
||||||
self.logger.info("%s completed for %s",CITests.CONTAINER_START.value, tag)
|
self.logger.info("%s completed for %s",test, tag)
|
||||||
self._add_test_result(tag, CITests.CONTAINER_START, CITestResult.PASS, "-", start_time)
|
self._add_test_result(tag, test, "PASS", "-", start_time)
|
||||||
self.logger.success("%s %s: PASSED after %.2f seconds", CITests.CONTAINER_START.value, tag, time.time() - start_time)
|
self.logger.success("%s %s: PASSED after %.2f seconds", test, tag, time.time() - start_time)
|
||||||
return True
|
return True
|
||||||
time.sleep(1)
|
time.sleep(1)
|
||||||
except APIError as error:
|
except APIError as error:
|
||||||
self.logger.exception("%s %s: FAIL - INIT NOT FINISHED", CITests.CONTAINER_START.value, tag)
|
self.logger.exception("%s %s: FAIL - INIT NOT FINISHED", test, tag)
|
||||||
self._add_test_result(tag, CITests.CONTAINER_START, CITestResult.FAIL, f"INIT NOT FINISHED: {str(error)}", start_time)
|
self._add_test_result(tag, test, "FAIL", f"INIT NOT FINISHED: {str(error)}", start_time)
|
||||||
self.report_status = CIReportResult.FAIL
|
self.report_status = "FAIL"
|
||||||
return False
|
return False
|
||||||
self.logger.error("%s failed for %s", CITests.CONTAINER_START.value, tag)
|
self.logger.error("%s failed for %s", test, tag)
|
||||||
self._add_test_result(tag, CITests.CONTAINER_START, CITestResult.FAIL, "INIT NOT FINISHED", start_time)
|
self._add_test_result(tag, test, "FAIL", "INIT NOT FINISHED", start_time)
|
||||||
self.logger.error("%s %s: FAIL - INIT NOT FINISHED", CITests.CONTAINER_START.value, tag)
|
self.logger.error("%s %s: FAIL - INIT NOT FINISHED", test, tag)
|
||||||
self.report_status = CIReportResult.FAIL
|
self.report_status = "FAIL"
|
||||||
return False
|
return False
|
||||||
|
|
||||||
def report_render(self) -> None:
|
def report_render(self) -> None:
|
||||||
@ -906,7 +679,7 @@ class CI(SetEnvs):
|
|||||||
with open(f"{self.outdir}/index.html", mode="w", encoding="utf-8") as file_:
|
with open(f"{self.outdir}/index.html", mode="w", encoding="utf-8") as file_:
|
||||||
file_.write(template.render(
|
file_.write(template.render(
|
||||||
report_containers=self.report_containers,
|
report_containers=self.report_containers,
|
||||||
report_status=self.report_status.value,
|
report_status=self.report_status,
|
||||||
meta_tag=self.meta_tag,
|
meta_tag=self.meta_tag,
|
||||||
image=self.get_image_name(),
|
image=self.get_image_name(),
|
||||||
bucket=self.bucket,
|
bucket=self.bucket,
|
||||||
@ -919,11 +692,11 @@ class CI(SetEnvs):
|
|||||||
"""Render the badge file for upload"""
|
"""Render the badge file for upload"""
|
||||||
self.logger.info("Creating badge")
|
self.logger.info("Creating badge")
|
||||||
try:
|
try:
|
||||||
badge = anybadge.Badge("CI", self.report_status.value, thresholds={
|
badge = anybadge.Badge("CI", self.report_status, thresholds={
|
||||||
"PASS": "green", "FAIL": "red"})
|
"PASS": "green", "FAIL": "red"})
|
||||||
badge.write_badge(f"{self.outdir}/badge.svg", overwrite=True)
|
badge.write_badge(f"{self.outdir}/badge.svg", overwrite=True)
|
||||||
with open(f"{self.outdir}/ci-status.yml", "w", encoding="utf-8") as file:
|
with open(f"{self.outdir}/ci-status.yml", "w", encoding="utf-8") as file:
|
||||||
file.write(f"CI: '{self.report_status.value}'")
|
file.write(f"CI: '{self.report_status}'")
|
||||||
except (ValueError,RuntimeError,FileNotFoundError,OSError):
|
except (ValueError,RuntimeError,FileNotFoundError,OSError):
|
||||||
self.logger.exception("Failed to render badge file!")
|
self.logger.exception("Failed to render badge file!")
|
||||||
|
|
||||||
@ -964,7 +737,7 @@ class CI(SetEnvs):
|
|||||||
raise CIError(f"Upload Error: {error}") from error
|
raise CIError(f"Upload Error: {error}") from error
|
||||||
self.logger.info("Report available on https://%s/%s/%s/index.html",self.bucket, self.image, self.meta_tag)
|
self.logger.info("Report available on https://%s/%s/%s/index.html",self.bucket, self.image, self.meta_tag)
|
||||||
|
|
||||||
def create_html_ansi_file(self, blob:str|CITestResult, tag:str, name:str, full:bool = True) -> None:
|
def create_html_ansi_file(self, blob:str, tag:str, name:str, full:bool = True) -> None:
|
||||||
"""Creates an HTML file in the "self.outdir" directory that we upload to S3
|
"""Creates an HTML file in the "self.outdir" directory that we upload to S3
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
@ -975,8 +748,6 @@ class CI(SetEnvs):
|
|||||||
|
|
||||||
"""
|
"""
|
||||||
try:
|
try:
|
||||||
if isinstance(blob,CITestResult):
|
|
||||||
blob = blob.value
|
|
||||||
self.logger.info("Creating %s.%s.html", tag, name)
|
self.logger.info("Creating %s.%s.html", tag, name)
|
||||||
converter = Ansi2HTMLConverter(title=f"{tag}-{name}")
|
converter = Ansi2HTMLConverter(title=f"{tag}-{name}")
|
||||||
html_logs: str = converter.convert(blob,full=full)
|
html_logs: str = converter.convert(blob,full=full)
|
||||||
@ -1022,28 +793,26 @@ class CI(SetEnvs):
|
|||||||
except (S3UploadFailedError, ClientError, FileNotFoundError) as e:
|
except (S3UploadFailedError, ClientError, FileNotFoundError) as e:
|
||||||
self.logger.exception(f"Failed to upload the CI logs! Error: {e}")
|
self.logger.exception(f"Failed to upload the CI logs! Error: {e}")
|
||||||
|
|
||||||
def _add_test_result(self, tag:str, test:CITests, status:CITestResult, message:str, start_time:float|int = 0.0) -> None:
|
def _add_test_result(self, tag:str, test:str, status:str, message:str, start_time:float|int = 0.0) -> None:
|
||||||
"""Add a test result to the report
|
"""Add a test result to the report
|
||||||
|
|
||||||
Args:
|
Args:
|
||||||
tag (str): The tag we are testing
|
tag (str): The tag we are testing
|
||||||
test (CITests): The test we are running
|
test (str): The test we are running
|
||||||
status (CITestResult): The status of the test
|
status (str): The status of the test
|
||||||
message (str): The message of the test
|
message (str): The message of the test
|
||||||
start_time (str, optional): The start time of the test. Defaults to 0.0. Used to calculate the runtime of the test.
|
start_time (str, optional): The start time of the test. Defaults to 0.0. Used to calculate the runtime of the test.
|
||||||
"""
|
"""
|
||||||
if not isinstance(test, CITests):
|
if status not in ["PASS","FAIL"]:
|
||||||
raise ValueError("test must be an instance of CITests")
|
raise ValueError("Status must be either PASS or FAIL")
|
||||||
if not isinstance(status, CITestResult):
|
|
||||||
raise ValueError("status must be an instance of CITestResult")
|
|
||||||
if tag not in self.tags:
|
if tag not in self.tags:
|
||||||
raise ValueError("Tag not in the list of tags")
|
raise ValueError("Tag not in the list of tags")
|
||||||
if not start_time:
|
if not start_time:
|
||||||
runtime = "-"
|
runtime = "-"
|
||||||
if isinstance(start_time,(float, int)):
|
if isinstance(start_time,(float, int)):
|
||||||
runtime: str = f"{time.time() - start_time:.2f}s"
|
runtime: str = f"{time.time() - start_time:.2f}s"
|
||||||
self.tag_report_tests[tag]["test"][test.value] = (dict(sorted({
|
self.tag_report_tests[tag]["test"][test] = (dict(sorted({
|
||||||
"status":status.value,
|
"status":status,
|
||||||
"message":message,
|
"message":message,
|
||||||
"runtime": runtime}.items())))
|
"runtime": runtime}.items())))
|
||||||
|
|
||||||
@ -1063,6 +832,7 @@ class CI(SetEnvs):
|
|||||||
return True, ""
|
return True, ""
|
||||||
proto: Literal["https", "http"] = "https" if self.ssl.upper() == "TRUE" else "http"
|
proto: Literal["https", "http"] = "https" if self.ssl.upper() == "TRUE" else "http"
|
||||||
screenshot_timeout = time.time() + self.screenshot_timeout
|
screenshot_timeout = time.time() + self.screenshot_timeout
|
||||||
|
test = "Get screenshot"
|
||||||
start_time = time.time()
|
start_time = time.time()
|
||||||
driver: WebDriver | None = None
|
driver: WebDriver | None = None
|
||||||
browser_logs: str = ""
|
browser_logs: str = ""
|
||||||
@ -1083,7 +853,7 @@ class CI(SetEnvs):
|
|||||||
driver.get_screenshot_as_file(f"{self.outdir}/{tag}.png")
|
driver.get_screenshot_as_file(f"{self.outdir}/{tag}.png")
|
||||||
if not os.path.isfile(f"{self.outdir}/{tag}.png"):
|
if not os.path.isfile(f"{self.outdir}/{tag}.png"):
|
||||||
raise FileNotFoundError(f"Screenshot '{self.outdir}/{tag}.png' not found")
|
raise FileNotFoundError(f"Screenshot '{self.outdir}/{tag}.png' not found")
|
||||||
self._add_test_result(tag, CITests.CAPTURE_SCREENSHOT, CITestResult.PASS, "-", start_time)
|
self._add_test_result(tag, test, "PASS", "-", start_time)
|
||||||
self.logger.success("Screenshot %s: PASSED after %.2f seconds", tag, time.time() - start_time)
|
self.logger.success("Screenshot %s: PASSED after %.2f seconds", tag, time.time() - start_time)
|
||||||
return True, self._get_browser_logs(driver, tag)
|
return True, self._get_browser_logs(driver, tag)
|
||||||
except Exception as error:
|
except Exception as error:
|
||||||
@ -1094,23 +864,23 @@ class CI(SetEnvs):
|
|||||||
raise error
|
raise error
|
||||||
raise TimeoutException("Timeout taking screenshot")
|
raise TimeoutException("Timeout taking screenshot")
|
||||||
except (requests.Timeout, requests.ConnectionError, KeyError) as error:
|
except (requests.Timeout, requests.ConnectionError, KeyError) as error:
|
||||||
self._add_test_result(tag, CITests.CAPTURE_SCREENSHOT, CITestResult.FAIL, f"CONNECTION ERROR: {str(error)}", start_time)
|
self._add_test_result(tag, test, "FAIL", f"CONNECTION ERROR: {str(error)}", start_time)
|
||||||
self.logger.exception("Screenshot %s FAIL CONNECTION ERROR", tag)
|
self.logger.exception("Screenshot %s FAIL CONNECTION ERROR", tag)
|
||||||
self.report_status = CIReportResult.FAIL
|
self.report_status = "FAIL"
|
||||||
if driver:
|
if driver:
|
||||||
browser_logs = self._get_browser_logs(driver, tag)
|
browser_logs = self._get_browser_logs(driver, tag)
|
||||||
return False, browser_logs
|
return False, browser_logs
|
||||||
except TimeoutException as error:
|
except TimeoutException as error:
|
||||||
self._add_test_result(tag, CITests.CAPTURE_SCREENSHOT, CITestResult.FAIL, f"TIMEOUT: {str(error)}", start_time)
|
self._add_test_result(tag, test, "FAIL", f"TIMEOUT: {str(error)}", start_time)
|
||||||
self.logger.exception("Screenshot %s FAIL TIMEOUT", tag)
|
self.logger.exception("Screenshot %s FAIL TIMEOUT", tag)
|
||||||
self.report_status = CIReportResult.FAIL
|
self.report_status = "FAIL"
|
||||||
if driver:
|
if driver:
|
||||||
browser_logs = self._get_browser_logs(driver, tag)
|
browser_logs = self._get_browser_logs(driver, tag)
|
||||||
return False, browser_logs
|
return False, browser_logs
|
||||||
except (WebDriverException, Exception) as error:
|
except (WebDriverException, Exception) as error:
|
||||||
self._add_test_result(tag, CITests.CAPTURE_SCREENSHOT, CITestResult.FAIL, f"UNKNOWN: {str(error)}", start_time)
|
self._add_test_result(tag, test, "FAIL", f"UNKNOWN: {str(error)}", start_time)
|
||||||
self.logger.exception("Screenshot %s FAIL UNKNOWN", tag)
|
self.logger.exception("Screenshot %s FAIL UNKNOWN", tag)
|
||||||
self.report_status = CIReportResult.FAIL
|
self.report_status = "FAIL"
|
||||||
if driver:
|
if driver:
|
||||||
browser_logs = self._get_browser_logs(driver, tag)
|
browser_logs = self._get_browser_logs(driver, tag)
|
||||||
return False, browser_logs
|
return False, browser_logs
|
||||||
|
|||||||
842
package_versions.txt
Normal file → Executable file
842
package_versions.txt
Normal file → Executable file
File diff suppressed because it is too large
Load Diff
@ -1,7 +1,8 @@
|
|||||||
#!/usr/bin/env python3
|
#!/usr/bin/env python3
|
||||||
import os
|
import os
|
||||||
|
import time
|
||||||
from logging import Logger
|
from logging import Logger
|
||||||
from ci.ci import CI, CIError, Platform, CIReportResult
|
from ci.ci import CI, CIError
|
||||||
from ci.logger import configure_logging
|
from ci.logger import configure_logging
|
||||||
|
|
||||||
def run_test() -> None:
|
def run_test() -> None:
|
||||||
@ -9,15 +10,15 @@ def run_test() -> None:
|
|||||||
ci.run(ci.tags)
|
ci.run(ci.tags)
|
||||||
# Don't set the whole report as failed if any of the ARM tag fails.
|
# Don't set the whole report as failed if any of the ARM tag fails.
|
||||||
for tag in ci.report_containers.keys():
|
for tag in ci.report_containers.keys():
|
||||||
if tag.startswith(Platform.AMD64.value) and ci.report_containers[tag]['test_success'] == True:
|
if tag.startswith("amd64") and ci.report_containers[tag]['test_success'] == True:
|
||||||
ci.report_status = CIReportResult.PASS # Override the report_status if an ARM tag failed, but the amd64 tag passed.
|
ci.report_status = 'PASS' # Override the report_status if an ARM tag failed, but the amd64 tag passed.
|
||||||
if ci.report_status == CIReportResult.PASS:
|
if ci.report_status == 'PASS':
|
||||||
logger.success('All tests PASSED after %.2f seconds', ci.total_runtime)
|
logger.success('All tests PASSED after %.2f seconds', ci.total_runtime)
|
||||||
ci.report_render()
|
ci.report_render()
|
||||||
ci.badge_render()
|
ci.badge_render()
|
||||||
ci.json_render()
|
ci.json_render()
|
||||||
ci.report_upload()
|
ci.report_upload()
|
||||||
if ci.report_status == CIReportResult.PASS: # Exit based on test results
|
if ci.report_status == 'PASS': # Exit based on test results
|
||||||
ci.log_upload()
|
ci.log_upload()
|
||||||
return
|
return
|
||||||
logger.error('Tests FAILED')
|
logger.error('Tests FAILED')
|
||||||
|
|||||||
File diff suppressed because it is too large
Load Diff
@ -1,149 +0,0 @@
|
|||||||
NAME VERSION
|
|
||||||
adduser 3.137ubuntu1
|
|
||||||
apt 2.8.3
|
|
||||||
apt-utils 2.8.3
|
|
||||||
base-files 13ubuntu10.3
|
|
||||||
base-passwd 3.6.3build1
|
|
||||||
bash 5.2.21-2ubuntu4
|
|
||||||
bsdutils 1:2.39.3-9ubuntu6.3
|
|
||||||
ca-certificates 20240203
|
|
||||||
catatonit 0.1.7-1
|
|
||||||
coreutils 9.4-3ubuntu6.1
|
|
||||||
cron 3.0pl1-184ubuntu2
|
|
||||||
cron-daemon-common 3.0pl1-184ubuntu2
|
|
||||||
curl 8.5.0-2ubuntu10.6
|
|
||||||
dash 0.5.12-6ubuntu5
|
|
||||||
debconf 1.5.86ubuntu1
|
|
||||||
debianutils 5.17build1
|
|
||||||
diffutils 1:3.10-1build1
|
|
||||||
dirmngr 2.4.4-2ubuntu17.3
|
|
||||||
dpkg 1.22.6ubuntu6.2
|
|
||||||
e2fsprogs 1.47.0-2.4~exp1ubuntu4.1
|
|
||||||
findutils 4.9.0-5build1
|
|
||||||
gcc-14-base 14.2.0-4ubuntu2~24.04
|
|
||||||
gnupg 2.4.4-2ubuntu17.3
|
|
||||||
gnupg-l10n 2.4.4-2ubuntu17.3
|
|
||||||
gnupg-utils 2.4.4-2ubuntu17.3
|
|
||||||
gpg 2.4.4-2ubuntu17.3
|
|
||||||
gpg-agent 2.4.4-2ubuntu17.3
|
|
||||||
gpg-wks-client 2.4.4-2ubuntu17.3
|
|
||||||
gpgconf 2.4.4-2ubuntu17.3
|
|
||||||
gpgsm 2.4.4-2ubuntu17.3
|
|
||||||
gpgv 2.4.4-2ubuntu17.3
|
|
||||||
grep 3.11-4build1
|
|
||||||
gzip 1.12-1ubuntu3.1
|
|
||||||
hostname 3.23+nmu2ubuntu2
|
|
||||||
init-system-helpers 1.66ubuntu1
|
|
||||||
jq 1.7.1-3ubuntu0.24.04.1
|
|
||||||
keyboxd 2.4.4-2ubuntu17.3
|
|
||||||
krb5-locales 1.20.1-6ubuntu2.6
|
|
||||||
libacl1 2.3.2-1build1.1
|
|
||||||
libapt-pkg6.0t64 2.8.3
|
|
||||||
libassuan0 2.5.6-1build1
|
|
||||||
libattr1 1:2.5.2-1build1.1
|
|
||||||
libaudit-common 1:3.1.2-2.1build1.1
|
|
||||||
libaudit1 1:3.1.2-2.1build1.1
|
|
||||||
libblkid1 2.39.3-9ubuntu6.3
|
|
||||||
libbrotli1 1.1.0-2build2
|
|
||||||
libbsd0 0.12.1-1build1.1
|
|
||||||
libbz2-1.0 1.0.8-5.1build0.1
|
|
||||||
libc-bin 2.39-0ubuntu8.6
|
|
||||||
libc6 2.39-0ubuntu8.6
|
|
||||||
libcap-ng0 0.8.4-2build2
|
|
||||||
libcap2 1:2.66-5ubuntu2.2
|
|
||||||
libcom-err2 1.47.0-2.4~exp1ubuntu4.1
|
|
||||||
libcrypt1 1:4.4.36-4build1
|
|
||||||
libcurl4t64 8.5.0-2ubuntu10.6
|
|
||||||
libdb5.3t64 5.3.28+dfsg2-7
|
|
||||||
libdebconfclient0 0.271ubuntu3
|
|
||||||
libext2fs2t64 1.47.0-2.4~exp1ubuntu4.1
|
|
||||||
libffi8 3.4.6-1build1
|
|
||||||
libgcc-s1 14.2.0-4ubuntu2~24.04
|
|
||||||
libgcrypt20 1.10.3-2build1
|
|
||||||
libgmp10 2:6.3.0+dfsg-2ubuntu6.1
|
|
||||||
libgnutls30t64 3.8.3-1.1ubuntu3.4
|
|
||||||
libgpg-error0 1.47-3build2.1
|
|
||||||
libgssapi-krb5-2 1.20.1-6ubuntu2.6
|
|
||||||
libhogweed6t64 3.9.1-2.2build1.1
|
|
||||||
libidn2-0 2.3.7-2build1.1
|
|
||||||
libjq1 1.7.1-3ubuntu0.24.04.1
|
|
||||||
libk5crypto3 1.20.1-6ubuntu2.6
|
|
||||||
libkeyutils1 1.6.3-3build1
|
|
||||||
libkmod2 31+20240202-2ubuntu7.1
|
|
||||||
libkrb5-3 1.20.1-6ubuntu2.6
|
|
||||||
libkrb5support0 1.20.1-6ubuntu2.6
|
|
||||||
libksba8 1.6.6-1build1
|
|
||||||
libldap-common 2.6.7+dfsg-1~exp1ubuntu8.
|
|
||||||
libldap2 2.6.7+dfsg-1~exp1ubuntu8.
|
|
||||||
liblz4-1 1.9.4-1build1.1
|
|
||||||
liblzma5 5.6.1+really5.4.5-1ubuntu
|
|
||||||
libmd0 1.1.0-2build1.1
|
|
||||||
libmount1 2.39.3-9ubuntu6.3
|
|
||||||
libncursesw6 6.4+20240113-1ubuntu2
|
|
||||||
libnettle8t64 3.9.1-2.2build1.1
|
|
||||||
libnghttp2-14 1.59.0-1ubuntu0.2
|
|
||||||
libnpth0t64 1.6-3.1build1
|
|
||||||
libonig5 6.9.9-1build1
|
|
||||||
libp11-kit0 0.25.3-4ubuntu2.1
|
|
||||||
libpam-modules 1.5.3-5ubuntu5.5
|
|
||||||
libpam-modules-bin 1.5.3-5ubuntu5.5
|
|
||||||
libpam-runtime 1.5.3-5ubuntu5.5
|
|
||||||
libpam0g 1.5.3-5ubuntu5.5
|
|
||||||
libpcre2-8-0 10.42-4ubuntu2.1
|
|
||||||
libproc2-0 2:4.0.4-4ubuntu3.2
|
|
||||||
libpsl5t64 0.21.2-1.1build1
|
|
||||||
libreadline8t64 8.2-4build1
|
|
||||||
librtmp1 2.4+20151223.gitfa8646d.1
|
|
||||||
libsasl2-2 2.1.28+dfsg1-5ubuntu3.1
|
|
||||||
libsasl2-modules 2.1.28+dfsg1-5ubuntu3.1
|
|
||||||
libsasl2-modules-db 2.1.28+dfsg1-5ubuntu3.1
|
|
||||||
libseccomp2 2.5.5-1ubuntu3.1
|
|
||||||
libselinux1 3.5-2ubuntu2.1
|
|
||||||
libsemanage-common 3.5-1build5
|
|
||||||
libsemanage2 3.5-1build5
|
|
||||||
libsepol2 3.5-2build1
|
|
||||||
libsmartcols1 2.39.3-9ubuntu6.3
|
|
||||||
libsqlite3-0 3.45.1-1ubuntu2.5
|
|
||||||
libss2 1.47.0-2.4~exp1ubuntu4.1
|
|
||||||
libssh-4 0.10.6-2ubuntu0.1
|
|
||||||
libssl3t64 3.0.13-0ubuntu3.5
|
|
||||||
libstdc++6 14.2.0-4ubuntu2~24.04
|
|
||||||
libsystemd0 255.4-1ubuntu8.10
|
|
||||||
libtasn1-6 4.19.0-3ubuntu0.24.04.1
|
|
||||||
libtinfo6 6.4+20240113-1ubuntu2
|
|
||||||
libudev1 255.4-1ubuntu8.10
|
|
||||||
libunistring5 1.1-2build1.1
|
|
||||||
libuuid1 2.39.3-9ubuntu6.3
|
|
||||||
libxxhash0 0.8.2-2build1
|
|
||||||
libzstd1 1.5.5+dfsg2-2build1.1
|
|
||||||
locales 2.39-0ubuntu8.6
|
|
||||||
login 1:4.13+dfsg1-4ubuntu3.2
|
|
||||||
logsave 1.47.0-2.4~exp1ubuntu4.1
|
|
||||||
mawk 1.3.4.20240123-1build1
|
|
||||||
mount 2.39.3-9ubuntu6.3
|
|
||||||
ncurses-base 6.4+20240113-1ubuntu2
|
|
||||||
ncurses-bin 6.4+20240113-1ubuntu2
|
|
||||||
netcat-openbsd 1.226-1ubuntu2
|
|
||||||
openssl 3.0.13-0ubuntu3.5
|
|
||||||
passwd 1:4.13+dfsg1-4ubuntu3.2
|
|
||||||
perl-base 5.38.2-3.2ubuntu0.2
|
|
||||||
pinentry-curses 1.2.1-3ubuntu5
|
|
||||||
plexmediaserver 1.42.2.10156-f737b826c
|
|
||||||
procps 2:4.0.4-4ubuntu3.2
|
|
||||||
publicsuffix 20231001.0357-0.1
|
|
||||||
readline-common 8.2-4build1
|
|
||||||
sed 4.9-2build1
|
|
||||||
sensible-utils 0.0.22
|
|
||||||
systemd-dev 255.4-1ubuntu8.10
|
|
||||||
systemd-hwe-hwdb 255.1.5
|
|
||||||
systemd-standalone-sysusers 255.4-1ubuntu8.10
|
|
||||||
sysvinit-utils 3.08-6ubuntu3
|
|
||||||
tar 1.35+dfsg-3build1
|
|
||||||
tzdata 2025b-0ubuntu0.24.04.1
|
|
||||||
ubuntu-keyring 2023.11.28.1
|
|
||||||
udev 255.4-1ubuntu8.10
|
|
||||||
unminimize 0.2.1
|
|
||||||
util-linux 2.39.3-9ubuntu6.3
|
|
||||||
wget 1.21.4-1ubuntu4.1
|
|
||||||
zlib1g 1:1.3.dfsg-3.1ubuntu2.1
|
|
||||||
sbom
|
|
||||||
@ -1,5 +1,5 @@
|
|||||||
import os
|
import os
|
||||||
from unittest.mock import Mock, patch
|
from unittest.mock import Mock
|
||||||
import json
|
import json
|
||||||
|
|
||||||
import pytest
|
import pytest
|
||||||
@ -8,7 +8,7 @@ import chromedriver_autoinstaller
|
|||||||
from docker import DockerClient
|
from docker import DockerClient
|
||||||
from moto import mock_aws
|
from moto import mock_aws
|
||||||
|
|
||||||
from ci.ci import CI, SetEnvs, CITestResult, CITests, Platform, BuildCacheTag
|
from ci.ci import CI, SetEnvs
|
||||||
|
|
||||||
os.environ["DRY_RUN"] = "false"
|
os.environ["DRY_RUN"] = "false"
|
||||||
os.environ["IMAGE"] = "linuxserver/test"
|
os.environ["IMAGE"] = "linuxserver/test"
|
||||||
@ -23,26 +23,12 @@ os.environ["SSL"] = "true"
|
|||||||
os.environ["PORT"] = "443"
|
os.environ["PORT"] = "443"
|
||||||
os.environ["WEB_SCREENSHOT"] = "true"
|
os.environ["WEB_SCREENSHOT"] = "true"
|
||||||
os.environ["WEB_AUTH"] = ""
|
os.environ["WEB_AUTH"] = ""
|
||||||
os.environ["COMMIT_SHA"] = "test-commit-sha"
|
|
||||||
os.environ["BUILD_NUMBER"] = "1234"
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def sbom_blob() -> bytes:
|
def sbom_blob() -> bytes:
|
||||||
with open("tests/sbom_blob.txt", "rb") as f:
|
with open("tests/sbom_blob.txt", "rb") as f:
|
||||||
yield f.read()
|
yield f.read()
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def sbom_buildx_blob() -> str:
|
|
||||||
with open("tests/sbom_buildx_blob.txt", "rb") as f:
|
|
||||||
data = f.read()
|
|
||||||
yield data.decode("utf-8").strip()
|
|
||||||
|
|
||||||
@pytest.fixture
|
|
||||||
def sbom_buildx_formatted_blob() -> str:
|
|
||||||
with open("tests/sbom_buildx_formatted_blob.txt", "rb") as f:
|
|
||||||
data = f.read()
|
|
||||||
yield data.decode("utf-8").strip()
|
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def syft_mock_container(sbom_blob:bytes) -> Mock:
|
def syft_mock_container(sbom_blob:bytes) -> Mock:
|
||||||
container = Mock(spec=Container)
|
container = Mock(spec=Container)
|
||||||
@ -52,12 +38,11 @@ def syft_mock_container(sbom_blob:bytes) -> Mock:
|
|||||||
yield container
|
yield container
|
||||||
|
|
||||||
@pytest.fixture
|
@pytest.fixture
|
||||||
def ci(tmpdir, syft_mock_container: Mock, sbom_buildx_blob: str) -> CI:
|
def ci(tmpdir, syft_mock_container: Mock) -> CI:
|
||||||
ci = CI()
|
ci = CI()
|
||||||
ci.client = Mock(DockerClient)
|
ci.client = Mock(DockerClient)
|
||||||
ci.client.containers = Mock()
|
ci.client.containers = Mock()
|
||||||
ci.client.containers.run = Mock(return_value=syft_mock_container)
|
ci.client.containers.run = Mock(return_value=syft_mock_container)
|
||||||
ci.get_sbom_buildx_blob = Mock(return_value=sbom_buildx_blob)
|
|
||||||
ci.outdir = tmpdir
|
ci.outdir = tmpdir
|
||||||
yield ci
|
yield ci
|
||||||
|
|
||||||
@ -123,8 +108,8 @@ def test_convert_env(set_envs: SetEnvs):
|
|||||||
|
|
||||||
def test_add_test_result(ci: CI):
|
def test_add_test_result(ci: CI):
|
||||||
for tag in ci.tags:
|
for tag in ci.tags:
|
||||||
ci._add_test_result(tag=tag, test=CITests.CONTAINER_START, status=CITestResult.PASS, message="-", start_time="")
|
ci._add_test_result(tag=tag, test=f"test-{tag}", status="PASS", message="-", start_time="")
|
||||||
assert ci.tag_report_tests[tag] == {'test': {CITests.CONTAINER_START.value: {"status": "PASS", "message": "-", "runtime": "-"}}}
|
assert ci.tag_report_tests[tag] == {'test': {f"test-{tag}": {"status": "PASS", "message": "-", "runtime": "-"}}}
|
||||||
|
|
||||||
def test_get_build_info(ci: CI, mock_container: Mock):
|
def test_get_build_info(ci: CI, mock_container: Mock):
|
||||||
info: dict[str, str] = ci.get_build_info(mock_container,ci.tags[0])
|
info: dict[str, str] = ci.get_build_info(mock_container,ci.tags[0])
|
||||||
@ -140,20 +125,20 @@ def test_get_build_info(ci: CI, mock_container: Mock):
|
|||||||
assert info == mock_info
|
assert info == mock_info
|
||||||
|
|
||||||
def test_get_platform(ci: CI):
|
def test_get_platform(ci: CI):
|
||||||
assert ci.get_platform(ci.tags[0]) == Platform.AMD64.value
|
assert ci.get_platform(ci.tags[0]) == "amd64"
|
||||||
assert ci.get_platform(ci.tags[1]) == Platform.ARM64.value
|
assert ci.get_platform(ci.tags[1]) == "arm64"
|
||||||
|
|
||||||
def test_watch_container_logs(ci: CI, mock_container: Mock):
|
def test_watch_container_logs(ci: CI, mock_container: Mock):
|
||||||
ci.watch_container_logs(mock_container, ci.tags[0])
|
ci.watch_container_logs(mock_container, ci.tags[0])
|
||||||
assert ci.tag_report_tests[ci.tags[0]]["test"][CITests.CONTAINER_START.value]["status"] == CITestResult.PASS.value
|
assert ci.tag_report_tests[ci.tags[0]]["test"]["Container startup"]["status"] == "PASS"
|
||||||
|
|
||||||
def test_take_screenshot(ci:CI,mock_container: Mock):
|
def test_take_screenshot(ci:CI,mock_container: Mock):
|
||||||
screenshot: bool = ci.take_screenshot(mock_container, ci.tags[0])
|
screenshot: bool = ci.take_screenshot(mock_container, ci.tags[0])
|
||||||
if screenshot:
|
if screenshot:
|
||||||
assert os.path.isfile(os.path.join(ci.outdir, f"{ci.tags[0]}.png")) is True
|
assert os.path.isfile(os.path.join(ci.outdir, f"{ci.tags[0]}.png")) is True
|
||||||
assert ci.tag_report_tests[ci.tags[0]]["test"][CITests.CAPTURE_SCREENSHOT.value]["status"] == CITestResult.PASS.value
|
assert ci.tag_report_tests[ci.tags[0]]["test"]["Get screenshot"]["status"] == "PASS"
|
||||||
else:
|
else:
|
||||||
assert ci.tag_report_tests[ci.tags[0]]["test"][CITests.CAPTURE_SCREENSHOT.value]["status"] == CITestResult.FAIL.value
|
assert ci.tag_report_tests[ci.tags[0]]["test"]["Get screenshot"]["status"] == "FAIL"
|
||||||
|
|
||||||
def test_create_html_ansi_file(ci:CI, log_blob:bytes):
|
def test_create_html_ansi_file(ci:CI, log_blob:bytes):
|
||||||
logs = log_blob.decode("utf-8")
|
logs = log_blob.decode("utf-8")
|
||||||
@ -174,41 +159,10 @@ def test_badge_render(ci:CI):
|
|||||||
ci.badge_render()
|
ci.badge_render()
|
||||||
assert os.path.isfile(os.path.join(ci.outdir,"ci-status.yml")) is True
|
assert os.path.isfile(os.path.join(ci.outdir,"ci-status.yml")) is True
|
||||||
|
|
||||||
def test_get_sbom_syft(ci:CI, syft_mock_container:Mock, sbom_blob:bytes):
|
def test_generate_sbom(ci:CI, syft_mock_container:Mock, sbom_blob:bytes):
|
||||||
sbom = ci.get_sbom_syft(ci.tags[0])
|
sbom = ci.generate_sbom(ci.tags[0])
|
||||||
assert "VERSION" in sbom
|
assert "VERSION" in sbom
|
||||||
|
|
||||||
def test_parse_buildx_sbom(ci:CI, sbom_buildx_blob:str):
|
|
||||||
packages = ci.parse_buildx_sbom(sbom_buildx_blob)
|
|
||||||
assert len(packages) == 148
|
|
||||||
assert packages[0]["name"] == "adduser"
|
|
||||||
assert packages[0]["version"] == "3.137ubuntu1"
|
|
||||||
|
|
||||||
def test_format_package_table(ci:CI, sbom_buildx_blob:str, sbom_buildx_formatted_blob: str):
|
|
||||||
packages = ci.parse_buildx_sbom(sbom_buildx_blob)
|
|
||||||
table = ci.format_package_table(packages)
|
|
||||||
assert "VERSION" in table
|
|
||||||
assert "cron" in table
|
|
||||||
assert "3.0pl1-184ubuntu2" in table
|
|
||||||
|
|
||||||
def test_get_sbom_buildx_blob(ci: CI, sbom_buildx_blob: str) -> None:
|
|
||||||
expected_output = sbom_buildx_blob
|
|
||||||
mock_completed_process = Mock()
|
|
||||||
mock_completed_process.returncode = 0
|
|
||||||
mock_completed_process.stdout = expected_output
|
|
||||||
|
|
||||||
with patch("subprocess.run", return_value=mock_completed_process):
|
|
||||||
sbom = ci.get_sbom_buildx_blob(ci.tags[0])
|
|
||||||
assert sbom.strip() == expected_output.strip()
|
|
||||||
|
|
||||||
def test_make_sbom(ci: CI, sbom_buildx_blob: str, sbom_buildx_formatted_blob: str) -> None:
|
|
||||||
with patch.object(ci, 'get_sbom_buildx_blob', return_value=sbom_buildx_blob):
|
|
||||||
packages = ci.make_sbom(ci.tags[0])
|
|
||||||
assert os.path.isfile(os.path.join(ci.outdir,"amd64-nightly-5.10.1.9109-ls85.sbom.html")) is True
|
|
||||||
assert "VERSION" in packages
|
|
||||||
assert "cron" in packages
|
|
||||||
assert "3.0pl1-184ubuntu2" in packages
|
|
||||||
|
|
||||||
def test_create_s3_client(ci:CI):
|
def test_create_s3_client(ci:CI):
|
||||||
with mock_aws():
|
with mock_aws():
|
||||||
ci.s3_client = ci.create_s3_client()
|
ci.s3_client = ci.create_s3_client()
|
||||||
@ -243,13 +197,3 @@ def test_get_image_name(ci: CI) -> None:
|
|||||||
assert ci.get_image_name() == "linuxserver/lspipepr-plex"
|
assert ci.get_image_name() == "linuxserver/lspipepr-plex"
|
||||||
ci.image = "lsiobase/ubuntu"
|
ci.image = "lsiobase/ubuntu"
|
||||||
assert ci.get_image_name() == "linuxserver/docker-baseimage-ubuntu"
|
assert ci.get_image_name() == "linuxserver/docker-baseimage-ubuntu"
|
||||||
|
|
||||||
def test_get_build_cache_url(ci: CI) -> None:
|
|
||||||
for tag in ci.tags:
|
|
||||||
cache_tag = ci.get_build_cache_platform(tag)
|
|
||||||
expected_url = f"{ci.build_cache_registry}:{cache_tag}-{ci.commit_sha}-{ci.build_number}"
|
|
||||||
assert ci.get_build_cache_url(tag) == expected_url
|
|
||||||
|
|
||||||
def test_get_build_cache_platform(ci: CI) -> None:
|
|
||||||
assert ci.get_build_cache_platform(ci.tags[0]) == BuildCacheTag.AMD64.value
|
|
||||||
assert ci.get_build_cache_platform(ci.tags[1]) == BuildCacheTag.ARM64.value
|
|
||||||
|
|||||||
Loading…
x
Reference in New Issue
Block a user