Merge pull request #20 from GilbN/patch

Cosmetic and logging updates. Raises exception if test fails.
This commit is contained in:
aptalca 2022-09-19 11:58:48 -04:00 committed by GitHub
commit e2248e89ea
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 34 additions and 448 deletions

View File

@ -59,7 +59,7 @@ class SetEnvs():
env_dict["S6_VERBOSITY"] = self.s6_verbosity
except Exception as error:
self.logger.exception(error)
raise Exception(f"Failed converting DOCKER_ENV: {envs} to dictionary") from error
raise CIError(f"Failed converting DOCKER_ENV: {envs} to dictionary") from error
return env_dict
@ -74,7 +74,7 @@ class SetEnvs():
self.tags_env = os.environ['TAGS']
except KeyError as error:
self.logger.exception("Key %s is not set in ENV!", error)
raise Exception(f'Key {error} is not set in ENV!') from error
raise CIError(f'Key {error} is not set in ENV!') from error
class CI(SetEnvs):
@ -190,7 +190,7 @@ class CI(SetEnvs):
self.tag_report_tests[tag].append(['Container startup', 'PASS', '-'])
self.logger.info('Container startup %s: PASS', tag)
else:
self.logger.warning('Container startup failed for %s', tag)
self.logger.error('Container startup failed for %s', tag)
self.tag_report_tests[tag].append(['Container startup', 'FAIL','INIT NOT FINISHED'])
self.logger.error('Container startup %s: FAIL - INIT NOT FINISHED', tag)
self.report_status = 'FAIL'
@ -274,7 +274,7 @@ class CI(SetEnvs):
except (S3UploadFailedError, ValueError, ClientError) as error:
self.logger.exception('Upload Error: %s',error)
self.log_upload()
raise Exception(f'Upload Error: {error}') from error
raise CIError(f'Upload Error: {error}') from error
# Loop through files in outdir and upload
for filename in os.listdir(self.outdir):
@ -286,7 +286,7 @@ class CI(SetEnvs):
except (S3UploadFailedError, ValueError, ClientError) as error:
self.logger.exception('Upload Error: %s',error)
self.log_upload()
raise Exception(f'Upload Error: {error}') from error
raise CIError(f'Upload Error: {error}') from error
self.logger.info('Report available on https://ci-tests.linuxserver.io/%s/index.html', f'{self.image}/{self.meta_tag}')
@ -305,7 +305,7 @@ class CI(SetEnvs):
self.s3_client.upload_file(file_path, self.bucket, f'{latest_dir}/{object_name}', ExtraArgs=content_type)
def log_upload(self) -> None:
"""Upload debug.log to S3
"""Upload ci.log to S3
Raises:
Exception: S3UploadFailedError
@ -313,7 +313,7 @@ class CI(SetEnvs):
"""
self.logger.info('Uploading logs')
try:
self.upload_file("/debug.log", 'debug.log', {'ContentType': 'text/plain', 'ACL': 'public-read'})
self.upload_file("/ci.log", 'ci.log', {'ContentType': 'text/plain', 'ACL': 'public-read'})
except (S3UploadFailedError, ClientError) as error:
self.logger.exception('Upload Error: %s',error)
@ -407,3 +407,6 @@ class CI(SetEnvs):
driver = webdriver.Chrome(options=chrome_options)
driver.set_page_load_timeout(60)
return driver
class CIError(Exception):
pass

View File

@ -43,22 +43,19 @@ def configure_logging(log_level:str):
# Console logging
ch = logging.StreamHandler()
cf = CustomLogFormatter('%(asctime)-15s | (%(threadName)-9s) %(name)-43s | %(levelname)-8s | (%(module)s.%(funcName)s|line:%(lineno)d) | %(message)s |', '%d/%m/%Y %H:%M:%S')
cf = CustomLogFormatter('%(asctime)-15s | %(threadName)-17s | %(name)-10s | %(levelname)-8s | (%(module)s.%(funcName)s|line:%(lineno)d) | %(message)s |', '%d/%m/%Y %H:%M:%S')
ch.setFormatter(cf)
ch.setLevel(log_level)
logger.addHandler(ch)
# File logging
fh = TimedRotatingFileHandler(os.path.join(os.getcwd(),'debug.log'), when="midnight", interval=1, backupCount=7, delay=True, encoding='utf-8')
f = CustomLogFormatter('%(asctime)-15s | (%(threadName)-9s) %(name)-43s | %(levelname)-8s | (%(module)s.%(funcName)s|line:%(lineno)d) | %(message)s |', '%d/%m/%Y %H:%M:%S')
fh = TimedRotatingFileHandler(os.path.join(os.getcwd(),'ci.log'), when="midnight", interval=1, backupCount=7, delay=True, encoding='utf-8')
f = CustomLogFormatter('%(asctime)-15s | %(threadName)-17s | %(name)-10s | %(levelname)-8s | (%(module)s.%(funcName)s|line:%(lineno)d) | %(message)s |', '%d/%m/%Y %H:%M:%S')
fh.setFormatter(f)
fh.setLevel(log_level)
logger.addHandler(fh)
logging.info('Operating system: %s', platform.platform())
logging.info('Python version: %s', platform.python_version())
if log_level.upper() == "DEBUG":
logging.getLogger("spam").setLevel(logging.DEBUG) # Change external loggers to debug if necessary
logging.debug('Operating system: %s', platform.platform())
logging.debug('Python version: %s', platform.python_version())
else:
logging.getLogger("ham").setLevel(logging.CRITICAL) # Set external loggers to a level if necessary
logging.getLogger("botocore").setLevel(logging.WARNING) # Mute boto3 logging output

View File

@ -1,375 +0,0 @@
#!/usr/bin/env python
import os
import boto3
import time
import sys
import docker
import requests
import anybadge
from requests.adapters import HTTPAdapter
from requests.packages.urllib3.util.retry import Retry
from multiprocessing.pool import Pool
from selenium import webdriver
from selenium.common.exceptions import ErrorInResponseException,TimeoutException
from jinja2 import Template
client = docker.from_env()
session = boto3.session.Session()
reload(sys)
sys.setdefaultencoding('utf8')
# Global Vars
global report_status
global report_tests
global report_containers
report_tests = []
report_containers = []
report_status = 'PASS'
#############
# Functions #
#############
# If the tests cannot even be run just fail the job
def core_fail(message):
print(message)
sys.exit(1)
# Convert env input to dictionary
def convert_env(vars):
global dockerenv
dockerenv = {}
try:
if '|' in vars:
for varpair in vars.split('|'):
var = varpair.split('=')
dockerenv[var[0]] = var[1]
else:
var = vars.split('=')
dockerenv[var[0]] = var[1]
except Exception as error:
core_fail(str(error))
# Update global variables from threaded testing process
def update_globals(data):
global report_status
for (tests,containers,status) in data:
for test in tests:
report_tests.append(test)
for container in containers:
report_containers.append(container)
if status == 'FAIL':
report_status = 'FAIL'
# Set the optional parameters
global webauth
global webpath
global dockerenv
global region
global bucket
global screenshot
global port
global ssl
global testdelay
try:
webauth = os.environ["WEB_AUTH"]
except KeyError:
webauth = 'user:password'
try:
webpath = os.environ["WEB_PATH"]
except KeyError:
webpath = ''
try:
convert_env(os.environ["DOCKER_ENV"])
except KeyError:
dockerenv = {}
try:
region = os.environ["S3_REGION"]
except KeyError:
region = 'us-east-1'
try:
bucket = os.environ["S3_BUCKET"]
except KeyError:
bucket = 'ci-tests.linuxserver.io'
try:
screenshot = os.environ["WEB_SCREENSHOT"]
except KeyError:
screenshot = 'false'
try:
port = os.environ["PORT"]
except KeyError:
port = '80'
try:
ssl = os.environ["SSL"]
except KeyError:
ssl = 'false'
try:
testdelay = os.environ["DELAY_START"]
except KeyError:
testdelay = '5'
# Make sure all needed env variables are set
def check_env():
try:
global image
global tags
global meta_tag
global base
global S3_key
global S3_secret
image = os.environ["IMAGE"]
base = os.environ["BASE"]
S3_key = os.environ["ACCESS_KEY"]
S3_secret = os.environ["SECRET_KEY"]
meta_tag = os.environ["META_TAG"]
tags_env = os.environ["TAGS"]
tags = []
if '|' in tags_env:
for tag in tags_env.split('|'):
tags.append(tag)
else:
tags.append(tags_env)
except KeyError as error:
core_fail(str(error) + ' is not set in ENV')
# Create output path
def create_dir():
global outdir
outdir = os.path.dirname(os.path.realpath(__file__)) + '/output/' + image + '/' + meta_tag + '/'
try:
os.stat(outdir)
except:
os.makedirs(outdir)
# Main container test logic
def container_test(tag):
# Vars for the threaded process
report_tests = []
report_containers = []
report_status = 'PASS'
# End the test with as much info as we have
def endtest(container,report_tests,report_containers,report_status,tag,build_version,packages):
logblob = container.logs().decode("utf-8")
container.remove(force='true')
# Add the info to the report
report_containers.append({
"tag":tag,
"logs":logblob,
"sysinfo":packages,
"build_version":build_version
})
return (report_tests,report_containers,report_status)
# Start the container
print('Starting ' + tag)
container = client.containers.run(image + ':' + tag,
detach=True,
environment=dockerenv)
# Watch the logs for no more than 5 minutes
t_end = time.time() + 60 * 5
logsfound = False
while time.time() < t_end:
try:
logblob = container.logs().decode("utf-8")
if '[services.d] done.' in logblob:
logsfound = True
break
time.sleep(1)
except Exception as error:
print('Startup failed for ' + tag)
report_tests.append(['Startup ' + tag,'FAIL INIT NOT FINISHED'])
report_status = 'FAIL'
(report_tests,report_containers,report_status) = endtest(container,report_tests,report_containers,report_status,tag,'ERROR','ERROR')
return (report_tests,report_containers,report_status)
# Grab build version
try:
build_version = container.attrs["Config"]["Labels"]["build_version"]
report_tests.append(['Get Build Version ' + tag,'PASS'])
except Exception as error:
build_version = 'ERROR'
report_tests.append(['Get Build Version ' + tag,'FAIL'])
report_status = 'FAIL'
(report_tests,report_containers,report_status) = endtest(container,report_tests,report_containers,report_status,tag,build_version,'ERROR')
return (report_tests,report_containers,report_status)
# Check if the startup marker was found in the logs during the 2 minute spinup
if logsfound == True:
print('Startup completed for ' + tag)
report_tests.append(['Startup ' + tag,'PASS'])
elif logsfound == False:
print('Startup failed for ' + tag)
report_tests.append(['Startup ' + tag,'FAIL INIT NOT FINISHED'])
report_status = 'FAIL'
(report_tests,report_containers,report_status) = endtest(container,report_tests,report_containers,report_status,tag,build_version,'ERROR')
return (report_tests,report_containers,report_status)
# Dump package information
print('Dumping package info for ' + tag)
if base == 'alpine':
command = 'apk info -v'
elif base == 'debian' or base == 'ubuntu':
command = 'apt list'
elif base == 'fedora':
command = 'rpm -qa'
elif base == 'arch':
command = 'pacman -Q'
try:
info = container.exec_run(command)
packages = info[1].decode("utf-8")
report_tests.append(['Dump Versions ' + tag,'PASS'])
print('Got Package info for ' + tag)
except Exception as error:
packages = 'ERROR'
print(str(error))
report_tests.append(['Dump Versions ' + tag,'FAIL'])
report_status = 'FAIL'
(report_tests,report_containers,report_status) = endtest(container,report_tests,report_containers,report_status,tag,build_version,packages)
return (report_tests,report_containers,report_status)
# Sleep for the user specified amount of time
time.sleep(int(testdelay))
# Screenshot web interface and check connectivity
if screenshot == 'true':
# Take a screenshot
if ssl == 'true':
proto = 'https://'
else:
proto = 'http://'
container.reload()
ip = container.attrs["NetworkSettings"]["Networks"]["bridge"]["IPAddress"]
endpoint = proto + webauth + '@' + ip + ':' + port + webpath
print('Taking screenshot of ' + tag + ' at ' + endpoint)
testercontainer = client.containers.run('lsiodev/tester:latest',
shm_size='1G',
detach=True,
environment={'URL': endpoint})
time.sleep(30)
testercontainer.reload()
testerip = testercontainer.attrs["NetworkSettings"]["Networks"]["bridge"]["IPAddress"]
testerendpoint = "http://" + testerip + ":3000"
try:
# Selenium webdriver options
chrome_options = webdriver.ChromeOptions()
chrome_options.add_argument('--no-sandbox')
chrome_options.add_argument('--headless')
chrome_options.add_argument('--disable-gpu')
chrome_options.add_argument('--window-size=1920x1080')
driver = webdriver.Chrome(options=chrome_options)
driver.set_page_load_timeout(60)
session = requests.Session()
retries = Retry(total=4, backoff_factor=2, status_forcelist=[ 502, 503, 504 ])
session.mount(proto, HTTPAdapter(max_retries=retries))
session.get(testerendpoint)
driver.get(testerendpoint)
time.sleep(15)
driver.get_screenshot_as_file(outdir + tag + '.png')
report_tests.append(['Screenshot ' + tag,'PASS'])
# Quit selenium webdriver
driver.quit()
except (requests.Timeout, requests.ConnectionError, KeyError) as e:
report_tests.append(['Screenshot ' + tag,'FAIL CONNECTION ERROR'])
except ErrorInResponseException as error:
report_tests.append(['Screenshot ' + tag,'FAIL SERVER ERROR'])
except TimeoutException as error:
report_tests.append(['Screenshot ' + tag,'FAIL TIMEOUT'])
except WebDriverException as error:
report_tests.append(['Screenshot ' + tag,'FAIL UNKNOWN'])
testercontainer.remove(force='true')
# If all info is present end test
(report_tests,report_containers,report_status) = endtest(container,report_tests,report_containers,report_status,tag,build_version,packages)
return (report_tests,report_containers,report_status)
# Render the markdown file for upload
def report_render():
print('Rendering Report')
with open(os.path.dirname(os.path.realpath(__file__)) + '/results.template') as file_:
template = Template(file_.read())
markdown = template.render(
report_tests=report_tests,
report_containers=report_containers,
report_status=report_status,
meta_tag=meta_tag,
image=image,
bucket=bucket,
region=region,
screenshot=screenshot)
with open(outdir + 'report.md', 'w') as f:
f.write(markdown)
# Render the badge file for upload
def badge_render():
try:
badge = anybadge.Badge('CI', report_status, thresholds={'PASS': 'green', 'FAIL': 'red'})
badge.write_badge(outdir + 'badge.svg')
with open(outdir + 'ci-status.yml', 'w') as f:
f.write('CI: "' + report_status + '"')
except Exception as error:
print(error)
# Upload report to S3
def report_upload():
print('Uploading Report')
destination_dir = image + '/' + meta_tag + '/'
latest_dir = image + '/latest/'
s3 = session.client(
's3',
region_name=region,
aws_access_key_id=S3_key,
aws_secret_access_key=S3_secret)
# Index file upload
index_file = os.path.dirname(os.path.realpath(__file__)) + '/index.html'
try:
s3.upload_file(
index_file,
bucket,
destination_dir + 'index.html',
ExtraArgs={'ContentType': "text/html", 'ACL': "public-read"})
s3.upload_file(
index_file,
bucket,
latest_dir + 'index.html',
ExtraArgs={'ContentType': "text/html", 'ACL': "public-read"})
except Exception as error:
core_fail('Upload Error ' + str(error))
# Loop for all others
for filename in os.listdir(outdir):
time.sleep(0.5)
# Set content types for files
if filename.lower().endswith('.svg'):
CT = 'image/svg+xml'
elif filename.lower().endswith('.png'):
CT = 'image/png'
elif filename.lower().endswith('.md'):
CT = 'text/markdown'
elif filename.lower().endswith('.yml'):
CT = 'text/yaml'
try:
s3.upload_file(
outdir + filename,
bucket,
destination_dir + filename,
ExtraArgs={'ContentType': CT,'ACL': "public-read",'CacheControl': 'no-cache'})
s3.upload_file(
outdir + filename,
bucket,
latest_dir + filename,
ExtraArgs={'ContentType': CT,'ACL': "public-read",'CacheControl': 'no-cache'})
except Exception as error:
core_fail('Upload Error ' + str(error))
##################
# Test Run Logic #
##################
check_env()
create_dir()
# Run through all the tags
pool=Pool(processes=3)
r = pool.map_async(container_test, tags, callback=update_globals)
r.wait()
report_render()
badge_render()
report_upload()
# Exit based on test results
if report_status == 'PASS':
print('Tests Passed exiting 0')
sys.exit(0)
elif report_status == 'FAIL':
print('Tests Failed exiting 1')
sys.exit(1)

View File

@ -1,42 +0,0 @@
# Test Results {{ image }}:{{ meta_tag }}
## Cumulative: {{ report_status }}
| Test | Result |
| ----------------------- | --- |{% for test in report_tests %}
| {{ test[0] }} | {{ test[1] }} |{% endfor %}
<main>
{% for container in report_containers %}
<section markdown="1">
## {{ image }}:{{ container["tag"] }}
{% if screenshot == 'true' %}
[![{{ container["tag"] }}]({{ container["tag"] }}.png =600x*)]({{ container["tag"] }}.png)
{% endif %}
### Build Version: {{ container["build_version"] }}
### Logs
<details><summary>Expand</summary>
<p>
```
{{ container["logs"] }}
```
</p>
</details>
### Package info
<details><summary>Expand</summary>
<p>
```
{{ container["sysinfo"] }}
```
</p>
</details>
</section>
{% endfor %}
</main>

View File

@ -243,6 +243,8 @@
.section-header {
border-radius: 10px 10px 0 0;
overflow-wrap: break-word;
}
.section-header-h2 {
@ -306,7 +308,7 @@
main {
display: flex;
justify-content: center;
align-items: flex-start;
align-items: stretch;
flex-wrap: wrap;
max-width: 100%;
}
@ -337,7 +339,6 @@
}
.table-container {
padding-top: 1rem;
overflow: auto;
}
.styled-table {
@ -385,7 +386,7 @@
cursor: pointer;
}
#debug_logs {
#logs {
overflow: auto;
}
@ -474,7 +475,7 @@
</details>
{% if 'arm32' in container["tag"] and container["dotnet"] == true %}
<div class="dotnet-notice">
<p class="dotnet-heading">Warning:<span class="dotnet-note">.NET application. Service might not start on ARM32 with QEMU</span></p>
<p class="dotnet-heading">Warning:<span class="dotnet-note">May be a .NET app. Service might not start on ARM32 with QEMU</span></p>
</div>
{% endif %}
<div class="table-container">
@ -506,18 +507,18 @@
</main>
</div>
<section class="debug-section">
<h3><a href="debug.log">Python debug logs</a></h3>
<h3><a href="ci.log">Python logs</a></h3>
<details>
<summary>Expand</summary>
<pre id="debug_logs"></pre>
<pre id="logs"></pre>
</details>
</section>
</div>
<script type="text/javascript" charset="utf-8">
fetch("debug.log")
fetch("ci.log")
.then(response => response.text())
.then(logs => {
document.getElementById("debug_logs").innerText = logs
document.getElementById("logs").innerText = logs
})
</script>
</body>

View File

@ -1,7 +1,7 @@
#!/usr/bin/env python3
import os
from ci.ci import CI
from ci.ci import CI, CIError
from ci.logger import configure_logging
def run_test():
@ -16,15 +16,17 @@ def run_test():
return
logger.error('Tests FAILED')
ci.log_upload()
raise CIError('CI Tests did not PASS!')
if __name__ == '__main__':
log_level = os.environ.get("CI_LOG_LEVEL","DEBUG")
configure_logging(log_level)
import logging
logger = logging.getLogger(__name__)
ci = CI()
try:
log_level = os.environ.get("CI_LOG_LEVEL","INFO")
configure_logging(log_level)
import logging
logger = logging.getLogger(__name__)
ci = CI()
run_test()
except Exception as err:
logger.exception("%s\nI Can't Believe You've Done This",err)
logger.exception(err)
raise CIError("I Can't Believe You've Done This!") from err