LAB-linuxserver/docker-ci
1
0
Fork 0
mirror of https://github.com/linuxserver/docker-ci.git synced 2026-02-04 18:50:13 +08:00
Code Issues Packages Projects Releases Wiki Activity

Update readme

Browse Source 
Fix typos
Add new success log level
Fix take_screenshot while loop
Use new loglevel when test passes.
This commit is contained in:
GilbN 2024-05-04 11:41:02 +02:00
parent d015715499
commit bb91f12ed5
5 changed files with 53 additions and 22 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

11
README.md
View File

@ -34,23 +34,24 @@ sudo docker run --rm -i \
-v /var/run/docker.sock:/var/run/docker.sock \
-v /host/path:/ci/output:rw `#Optional, will contain all the files the container creates.` \
-e IMAGE="linuxserver/<dockerimage>" \
-e TAGS="<single tag or array seperated by |>" \
-e TAGS="<single tag or array separated by |>" \
-e META_TAG=<manifest main dockerhub tag> \
-e BASE=<alpine or debian based distro> \
-e SECRET_KEY=<S3 secret> \
-e ACCESS_KEY=<S3 key> \
-e DOCKER_ENV="<optional, Array of env vars seperated by | IE test=test|test2=test2 or single var. Defaults to ''>" \
-e DOCKER_ENV="<optional, Array of env vars separated by | IE test=test|test2=test2 or single var. Defaults to ''>" \
-e WEB_AUTH="<optional, format user:passord. Defaults to 'user:password'>" \
-e WEB_PATH="<optional, format /yourpath>. Defaults to ''." \
-e S3_REGION=<optional, custom S3 Region. Defaults to 'us-east-1'> \
-e S3_BUCKET=<optional, custom S3 Bucket. Defaults to 'ci-tests.linuxserver.io'> \
-e WEB_SCREENSHOT_DELAY=<optional, time in seconds to delay before taking screenshot. Defaults to '30'>
-e WEB_SCREENSHOT_TIMEOUT=<optional, time in seconds before timing out trying to take a screenshot. Defaults to '60'>
-e SBOM_TIMEOUT=<optional, time in seconds before timing out trying to generate a SBOM. Defaults to '900'>
-e WEB_SCREENSHOT=<optional, set to false if not a web app. Defaults to 'false'> \
-e DELAY_START=<optional, time in seconds to delay before taking screenshot. Defaults to '5'> \
-e PORT=<optional, port web application listens on internal docker port. Defaults to '80'> \
-e SSL=<optional , use ssl for the screenshot true/false. Defaults to 'false'> \
-e CI_S6_VERBOSITY=<optional, Updates the S6_VERBOSITY env. Defaults to '2'> \
-e DOCKER_LOGS_DELAY=<optional, How long to wait in seconds while tailing the container logs. Defaults to '300'> \
-e CI_LOG_LEVEL=<optional, Sets the ci logging level. Defaults to 'INFO'> \
-e DOCKER_LOGS_TIMEOUT=<optional, How long to wait in seconds while tailing the container logs before timing out. Defaults to '900'> \
-e DRY_RUN=<optional, Set to 'true' when you don't want to upload files to S3 when testing>
-t lsiodev/ci:latest \
python3 test_build.py

28
ci/ci.py
View File

@ -102,6 +102,8 @@ class SetEnvs():
META_TAG: '{os.environ.get("META_TAG")}'
TAGS: '{os.environ.get("TAGS")}'
S6_VERBOSITY: '{os.environ.get("S6_VERBOSITY")}'
CI_S6_VERBOSITY '{os.environ.get("CI_S6_VERBOSITY")}'
CI_LOG_LEVEL '{os.environ.get("CI_LOG_LEVEL")}'
DOCKER_ENV: '{os.environ.get("DOCKER_ENV")}'
DOCKER_VOLUMES: '{os.environ.get("DOCKER_VOLUMES")}' (Not in use)
DOCKER_PRIVILEGED: '{os.environ.get("DOCKER_PRIVILEGED")}' (Not in use)
@ -119,7 +121,7 @@ class SetEnvs():
S3_REGION: '{os.environ.get("S3_REGION")}'
S3_BUCKET: '{os.environ.get("S3_BUCKET")}'
""")
logger.info(env_data)
self.logger.info(env_data)
def _split_key_value_string(self, kv:str, make_list:bool = False) -> dict[str,str] | list[str]:
"""Split a key value string into a dictionary or list.
@ -165,7 +167,7 @@ class SetEnvs():
volumes (str, optional): A string with key values separated by the pipe symbol. e.g `key1=val1|key2=val2`. Defaults to None.
Raises:
CIError: Raises a CIError Exception if it failes to parse the string
CIError: Raises a CIError Exception if it fails to parse the string
Returns:
list[str]: Returns a list with our keys and values.
@ -184,7 +186,7 @@ class SetEnvs():
"""Make sure all needed ENVs are set
Raises:
CIError: Raises a CIError exception if one of the enviroment values is not set.
CIError: Raises a CIError exception if one of the environment values is not set.
"""
try:
self.image: str = os.environ["IMAGE"]
@ -211,12 +213,12 @@ class CI(SetEnvs):
s3_client (boto3.client): S3 client object
Args:
SetEnvs (Object): Helper class that initializes and checks that all the necessary enviroment variables exists. Object is initialized upon init of CI.
SetEnvs (Object): Helper class that initializes and checks that all the necessary environment variables exists. Object is initialized upon init of CI.
"""
def __init__(self) -> None:
super().__init__() # Init the SetEnvs object.
self.logger = logging.getLogger("LSIO CI")
logging.getLogger("botocore.auth").setLevel(logging.INFO) # Don"t log the S3 authentication steps.
logging.getLogger("botocore.auth").setLevel(logging.INFO) # Don't log the S3 authentication steps.
self.client: DockerClient = docker.from_env()
self.tags = list(self.tags_env.split("|"))
@ -291,7 +293,7 @@ class CI(SetEnvs):
self.take_screenshot(container, tag)
self._endtest(container, tag, build_info, sbom, True)
self.logger.info("Test of %s PASSED after %.2f seconds", tag, time.time() - start_time)
self.logger.success("Test of %s PASSED after %.2f seconds", tag, time.time() - start_time)
return
def _endtest(self, container:Container, tag:str, build_info:dict[str,str], packages:str, test_success: bool) -> None:
@ -412,7 +414,7 @@ class CI(SetEnvs):
if "VERSION" in logblob:
self.logger.info("Get package versions for %s completed", tag)
self._add_test_result(tag, test, "PASS", "-")
self.logger.info("%s package list %s: PASS", test, tag)
self.logger.success("%s package list %s: PASS", test, tag)
self.create_html_ansi_file(str(logblob),tag,"sbom")
try:
syft.remove(force=True)
@ -489,7 +491,7 @@ class CI(SetEnvs):
"maintainer": container.attrs["Config"]["Labels"]["maintainer"],
}
self._add_test_result(tag, test, "PASS", "-")
self.logger.info("Get build info on tag '%s': PASS", tag)
self.logger.success("Get build info on tag '%s': PASS", tag)
except (APIError,KeyError) as error:
self.logger.exception("Get build info on tag '%s': FAIL", tag)
build_info = {"version": "ERROR", "created": "ERROR", "size": "ERROR", "maintainer": "ERROR"}
@ -519,7 +521,7 @@ class CI(SetEnvs):
if "[services.d] done." in logblob or "[ls.io-init] done." in logblob:
self.logger.info("%s completed for %s",test, tag)
self._add_test_result(tag, test, "PASS", "-")
self.logger.info("%s %s: PASS", test, tag)
self.logger.success("%s %s: PASS", test, tag)
return True
time.sleep(1)
except APIError as error:
@ -611,7 +613,7 @@ class CI(SetEnvs):
"""
try:
self.logger.info(f"Creating {tag}.{name}.html")
self.logger.info("Creating %s.%s.html", tag, name)
converter = Ansi2HTMLConverter(title=f"{tag}-{name}")
html_logs: str = converter.convert(blob,full=full)
with open(f"{self.outdir}/{tag}.{name}.html", "w", encoding="utf-8") as file:
@ -683,6 +685,7 @@ class CI(SetEnvs):
screenshot_timeout = time.time() + int(self.screenshot_timeout)
test = "Get screenshot"
try:
self.logger.info("Trying for %s seconds to take a screenshot of %s ",self.screenshot_timeout, tag)
driver: WebDriver = self.setup_driver()
while time.time() < screenshot_timeout:
try:
@ -690,12 +693,13 @@ class CI(SetEnvs):
ip_adr:str = container.attrs.get("NetworkSettings",{}).get("Networks",{}).get("bridge",{}).get("IPAddress","")
endpoint: str = f"{proto}://{self.webauth}@{ip_adr}:{self.port}{self.webpath}"
driver.get(endpoint)
self.logger.info("Trying to take screenshot of %s at %s", tag, endpoint)
self.logger.debug("Trying to take screenshot of %s at %s", tag, endpoint)
driver.get_screenshot_as_file(f"{self.outdir}/{tag}.png")
if not os.path.isfile(f"{self.outdir}/{tag}.png"):
continue
self._add_test_result(tag, test, "PASS", "-")
self.logger.info("Screenshot %s: PASS", tag)
self.logger.success("Screenshot %s: PASS", tag)
return
except Exception as error:
logger.debug("Failed to take screenshot of %s at %s, trying again in 1 second", tag, endpoint)
logger.debug("Error: %s", error)

21
ci/logger.py
View File

@ -19,12 +19,30 @@ else:
logger: Logger = logging.getLogger()
# Add custom log level for success messages
logging.SUCCESS = 25
logging.addLevelName(logging.SUCCESS, "SUCCESS")
def success(self:'Logger', message:str, *args, **kwargs):
"""Log 'message % args' with severity 'SUCCESS'.
To pass exception information, use the keyword argument exc_info with
a true value, e.g.
logger.success("Houston, Tranquility Base Here. The Eagle has Landed.", exc_info=1)
"""
if self.isEnabledFor(logging.SUCCESS):
self._log(logging.SUCCESS, message, args, **kwargs)
logging.Logger.success = success
class ColorPercentStyle(logging.PercentStyle):
"""Custom log formatter that add color to specific log levels."""
grey: str = "38"
yellow: str = "33"
red: str = "31"
cyan: str = "36"
green: str = "32"
def _get_color_fmt(self, color_code, bold=False) -> str:
if bold:
@ -37,7 +55,8 @@ class ColorPercentStyle(logging.PercentStyle):
logging.INFO: self._get_color_fmt(self.cyan),
logging.WARNING: self._get_color_fmt(self.yellow),
logging.ERROR: self._get_color_fmt(self.red),
logging.CRITICAL: self._get_color_fmt(self.red)
logging.CRITICAL: self._get_color_fmt(self.red),
logging.SUCCESS: self._get_color_fmt(self.green)
}
return colors.get(levelno, self._get_color_fmt(self.grey))

9
ci/template.html
View File

@ -131,7 +131,8 @@
.log-debug {color:lightgray}
.log-info {color:lightskyblue}
.log-warning {color:darkorange}
.log-error {color:red}
.log-error {color:red;font-weight: bolder;}
.log-success{color:limegreen;font-weight: bolder;}
}
@media (prefers-color-scheme: light) {
@ -212,7 +213,8 @@
.log-debug {color:#9bb0bf}
.log-info {color:#60707c}
.log-warning {color:darkorange}
.log-error {color:red}
.log-error {color:red;font-weight: bolder;}
.log-success{color:#009879;font-weight: bolder;}
}
body,
@ -662,7 +664,8 @@
pylogs = logs.replace(/\[38;20m/gi,"<span class='log-debug'>"
).replace(/\[33;20m/gi,"<span class='log-warning'>"
).replace(/\[31;20m/gi,"<span class='log-error'>"
).replace(/\[36;20m/gi,"<span class='log-info'>"
).replace(/\[36;20m/gi,"<span class='log-info'>"
).replace(/\[32;20m/gi,"<span class='log-success'>"
).replace(/\[0m/gi,"</span>")
document.getElementById("logs").innerHTML = pylogs
})

6
test_build.py
View File

@ -1,22 +1,26 @@
#!/usr/bin/env python3
import os
import time
from logging import Logger
from ci.ci import CI, CIError
from ci.logger import configure_logging
def run_test() -> None:
"""Run tests on container tags then build and upload reports"""
start_time = time.time()
ci.run(ci.tags)
# Don't set the whole report as failed if any of the ARM tag fails.
for tag in ci.report_containers.keys():
if tag.startswith("amd64") and ci.report_containers[tag]['test_success'] == True:
ci.report_status = 'PASS' # Override the report_status if an ARM tag failed, but the amd64 tag passed.
if ci.report_status == 'PASS':
logger.success('All tests PASSED after %.2f seconds', time.time() - start_time)
ci.report_render()
ci.badge_render()
ci.json_render()
ci.report_upload()
if ci.report_status == 'PASS': # Exit based on test results
logger.info('Tests PASSED')
ci.log_upload()
return
logger.error('Tests FAILED')