adds debug logging, pep8 formatting, upload logs to s3

This commit is contained in:
Marius 2022-06-02 22:18:09 +02:00
parent 05ff8ade1f
commit 7186a6bb29
3 changed files with 203 additions and 149 deletions

279
ci/ci.py
View File

@ -2,7 +2,7 @@
import os
import time
import sys
import logging
import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
@ -14,127 +14,133 @@ from selenium import webdriver
from selenium.common.exceptions import TimeoutException, WebDriverException
from jinja2 import Template
logging.basicConfig(format='%(asctime)s : %(levelname)s : %(module)s : %(funcName)s : %(message)s',
level=logging.DEBUG, handlers=[logging.StreamHandler(), logging.FileHandler("debug.log", mode='w')])
class CI():
"""What's up doc"""
'''What's up doc'''
def __init__(self):
self.logger = logging.getLogger(__name__)
self.client = docker.from_env()
self.session = boto3.Session()
self.report_tests = []
self.report_containers = []
self.report_status = "PASS"
self.report_status = 'PASS'
self.dockerenv = {}
self.error_message = ""
self.error_message = ''
# Set the optional parameters
self.webauth = os.environ.get("WEB_AUTH","user:password")
self.webpath = os.environ.get("WEB_PATH","")
self.region = os.environ.get("S3_REGION","us-east-1")
self.bucket = os.environ.get("S3_BUCKET","ci-tests.linuxserver.io")
self.screenshot = os.environ.get("WEB_SCREENSHOT","false")
self.port = os.environ.get("PORT","80")
self.ssl = os.environ.get("SSL","false")
self.testdelay = os.environ.get("DELAY_START","5")
self.webauth = os.environ.get('WEB_AUTH', 'user:password')
self.webpath = os.environ.get('WEB_PATH', '')
self.region = os.environ.get('S3_REGION', 'us-east-1')
self.bucket = os.environ.get('S3_BUCKET', 'ci-tests.linuxserver.io')
self.screenshot = os.environ.get('WEB_SCREENSHOT', 'false')
self.port = os.environ.get('PORT', '80')
self.ssl = os.environ.get('SSL', 'false')
self.testdelay = os.environ.get('DELAY_START', '5')
self.check_env()
self.outdir = os.path.dirname(os.path.realpath(__file__)) + '/output/' + self.image + '/' + self.meta_tag + '/'
os.makedirs(self.outdir,exist_ok=True)
self.outdir = f'{os.path.dirname(os.path.realpath(__file__))}/output/{self.image}/{self.meta_tag}'
os.makedirs(self.outdir, exist_ok=True)
def convert_env(self,envs):
"""Convert env input to dictionary"""
def convert_env(self, envs):
'''Convert env input to dictionary'''
try:
if "|" in envs:
for varpair in envs.split("|"):
var = varpair.split("=")
if '|' in envs:
for varpair in envs.split('|'):
var = varpair.split('=')
self.dockerenv[var[0]] = var[1]
else:
var = envs.split("=")
var = envs.split('=')
self.dockerenv[var[0]] = var[1]
except Exception as error:
self.error_message = error
raise Exception(self.error_message) from error
def check_env(self):
"""Make sure all needed env variables are set"""
'''Make sure all needed env variables are set'''
try:
self.image = os.environ["IMAGE"]
self.base = os.environ["BASE"]
self.s3_key = os.environ["ACCESS_KEY"]
self.s3_secret = os.environ["SECRET_KEY"]
self.meta_tag = os.environ["META_TAG"]
self.tags_env = os.environ["TAGS"]
self.image = os.environ['IMAGE']
self.base = os.environ['BASE']
self.s3_key = os.environ['ACCESS_KEY']
self.s3_secret = os.environ['SECRET_KEY']
self.meta_tag = os.environ['META_TAG']
self.tags_env = os.environ['TAGS']
self.tags = []
if "|" in self.tags_env:
for tag in self.tags_env.split("|"):
if '|' in self.tags_env:
for tag in self.tags_env.split('|'):
self.tags.append(tag)
else:
self.tags.append(self.tags_env)
except KeyError as error:
self.error_message = f"Key {error} is not set in ENV!"
self.error_message = f'Key {error} is not set in ENV!'
raise Exception(self.error_message) from error
def container_test(self, tag):
"""Main container test logic"""
def _endtest(self:CI, container, tag, build_version, packages):
"""End the test with as much info as we have"""
logblob = container.logs().decode("utf-8")
'''Main container test logic'''
def _endtest(self: CI, container, tag, build_version, packages):
'''End the test with as much info as we have'''
logblob = container.logs().decode('utf-8')
container.remove(force='true')
# Add the info to the report
self.report_containers.append({
"tag":tag,
"logs":logblob,
"sysinfo":packages,
"build_version":build_version
'tag': tag,
'logs': logblob,
'sysinfo': packages,
'build_version': build_version
})
return (self.report_tests,self.report_containers,self.report_status)
return (self.report_tests, self.report_containers, self.report_status)
# Start the container
print('Starting ' + tag)
container = self.client.containers.run(self.image + ':' + tag,
detach=True,
environment=self.dockerenv)
self.logger.info(f'Starting {tag}')
container = self.client.containers.run(f'{self.image}:{tag}',
detach=True,
environment=self.dockerenv)
# Watch the logs for no more than 5 minutes
logsfound = False
t_end = time.time() + 60 * 5
while time.time() < t_end:
try:
logblob = container.logs().decode("utf-8")
logblob = container.logs().decode('utf-8')
if '[services.d] done.' in logblob or '[ls.io-init] done.' in logblob:
logsfound = True
break
time.sleep(1)
except Exception:
print(f'Startup failed for {tag}')
self.report_tests.append([f'Startup {tag}','FAIL INIT NOT FINISHED'])
self.logger.exception(f'Startup failed for {tag}')
self.report_tests.append(
[f'Startup {tag}', 'FAIL INIT NOT FINISHED'])
self.report_status = 'FAIL'
_endtest(self,container,tag,'ERROR','ERROR')
return (self.report_tests,self.report_containers,self.report_status)
_endtest(self, container, tag, 'ERROR', 'ERROR')
return (self.report_tests, self.report_containers, self.report_status)
# Grab build version
try:
build_version = container.attrs["Config"]["Labels"]["build_version"]
self.report_tests.append([f'Get Build Version {tag}','PASS'])
build_version = container.attrs['Config']['Labels']['build_version']
self.report_tests.append([f'Get Build Version {tag}', 'PASS'])
except Exception:
build_version = 'ERROR'
self.report_tests.append([f'Get Build Version {tag}','FAIL'])
self.report_tests.append([f'Get Build Version {tag}', 'FAIL'])
self.report_status = 'FAIL'
_endtest(self,container,tag,build_version,'ERROR')
return (self.report_tests,self.report_containers,self.report_status)
_endtest(self, container, tag, build_version, 'ERROR')
return (self.report_tests, self.report_containers, self.report_status)
# Check if the startup marker was found in the logs during the 2 minute spinup
if logsfound is True:
print(f'Startup completed for {tag}')
self.report_tests.append([f'Startup {tag}','PASS'])
self.logger.info(f'Startup completed for {tag}')
self.report_tests.append([f'Startup {tag}', 'PASS'])
elif logsfound is False:
print('Startup failed for ' + tag)
self.report_tests.append([f'Startup {tag}','FAIL INIT NOT FINISHED'])
self.logger.warning(f'Startup failed for {tag}')
self.report_tests.append(
[f'Startup {tag}', 'FAIL INIT NOT FINISHED'])
self.report_status = 'FAIL'
_endtest(self,container,tag,build_version,'ERROR')
return (self.report_tests,self.report_containers,self.report_status)
_endtest(self, container, tag, build_version, 'ERROR')
return (self.report_tests, self.report_containers, self.report_status)
# Dump package information
print('Dumping package info for ' + tag)
self.logger.info(f'Dumping package info for {tag}')
if self.base == 'alpine':
command = 'apk info -v'
elif self.base in ('debian', 'ubuntu'):
@ -145,30 +151,29 @@ class CI():
command = 'pacman -Q'
try:
info = container.exec_run(command)
packages = info[1].decode("utf-8")
self.report_tests.append([f'Dump Versions {tag}','PASS'])
print(f'Got Package info for {tag}')
packages = info[1].decode('utf-8')
self.report_tests.append([f'Dump Versions {tag}', 'PASS'])
self.logger.info(f'Got Package info for {tag}')
except Exception as error:
packages = 'ERROR'
print(str(error))
self.report_tests.append([f'Dump Versions {tag}','FAIL'])
self.logger.exception(str(error))
self.report_tests.append([f'Dump Versions {tag}', 'FAIL'])
self.report_status = 'FAIL'
_endtest(self,container,tag,build_version,packages)
return (self.report_tests,self.report_containers,self.report_status)
_endtest(self, container, tag, build_version, packages)
return (self.report_tests, self.report_containers, self.report_status)
# Sleep for the user specified amount of time
time.sleep(int(self.testdelay))
# Screenshot web interface and check connectivity
if self.screenshot == 'true':
self.take_screenshot(container,tag)
self.take_screenshot(container, tag)
# If all info is present end test
_endtest(self,container,tag,build_version,packages)
return (self.report_tests,self.report_containers,self.report_status)
_endtest(self, container, tag, build_version, packages)
return (self.report_tests, self.report_containers, self.report_status)
def report_render(self):
"""Render the markdown file for upload"""
print('Rendering Report')
with open(os.path.dirname(os.path.realpath(__file__)) + '/results.template',encoding="utf-8") as file_:
'''Render the markdown file for upload'''
self.logger.info('Rendering Report')
with open(f'{os.path.dirname(os.path.realpath(__file__))}/results.template', encoding='utf-8') as file_:
template = Template(file_.read())
markdown = template.render(
report_tests=self.report_tests,
@ -179,46 +184,46 @@ class CI():
bucket=self.bucket,
region=self.region,
screenshot=self.screenshot)
with open(self.outdir + 'report.md', 'w', encoding="utf-8") as file:
with open(f'{self.outdir}/report.md', 'w', encoding='utf-8') as file:
file.write(markdown)
def badge_render(self):
"""Render the badge file for upload"""
'''Render the badge file for upload'''
try:
badge = anybadge.Badge('CI', self.report_status, thresholds={'PASS': 'green', 'FAIL': 'red'})
badge.write_badge(self.outdir + 'badge.svg')
with open(self.outdir + 'ci-status.yml', 'w', encoding="utf-8") as file:
file.write('CI: "' + self.report_status + '"')
badge = anybadge.Badge('CI', self.report_status, thresholds={
'PASS': 'green', 'FAIL': 'red'})
badge.write_badge(f'{self.outdir}/badge.svg')
with open(f'{self.outdir}/ci-status.yml', 'w', encoding='utf-8') as file:
file.write(f'CI: "{self.report_status}"')
except Exception as error:
print(error)
self.logger.exception(error)
def report_upload(self):
"""Upload report to S3"""
print('Uploading Report')
destination_dir = self.image + '/' + self.meta_tag + '/'
latest_dir = self.image + '/latest/'
'''Upload report to S3'''
self.logger.info('Uploading Report')
destination_dir = f'{self.image}/{self.meta_tag}'
latest_dir = f'{self.image}/latest'
s3_instance = self.session.client(
's3',
region_name=self.region,
aws_access_key_id=self.s3_key,
aws_secret_access_key=self.s3_secret)
# Index file upload
index_file = os.path.dirname(os.path.realpath(__file__)) + '/index.html'
index_file = f'{os.path.dirname(os.path.realpath(__file__))}/index.html'
try:
s3_instance.upload_file(
index_file,
self.bucket,
destination_dir + 'index.html',
ExtraArgs={'ContentType': "text/html", 'ACL': "public-read"})
f'{destination_dir}/index.html',
ExtraArgs={'ContentType': 'text/html', 'ACL': 'public-read'})
s3_instance.upload_file(
index_file,
self.bucket,
latest_dir + 'index.html',
ExtraArgs={'ContentType': "text/html", 'ACL': "public-read"})
f'{latest_dir}/index.html',
ExtraArgs={'ContentType': 'text/html', 'ACL': 'public-read'})
except Exception as error:
self.error_message = f'Upload Error: {error}'
self.logger.exception(f'Upload Error: {error}')
raise Exception(self.error_message) from error
# Loop for all others
for filename in os.listdir(self.outdir):
@ -232,36 +237,66 @@ class CI():
ctype = 'text/markdown'
elif filename.lower().endswith('.yml'):
ctype = 'text/yaml'
else:
ctype = 'text/plain'
try:
s3_instance.upload_file(
self.outdir + filename,
f'{self.outdir}/{filename}',
self.bucket,
destination_dir + filename,
ExtraArgs={'ContentType': ctype,'ACL': "public-read",'CacheControl': 'no-cache'})
f'{destination_dir}/{filename}',
ExtraArgs={'ContentType': ctype, 'ACL': 'public-read', 'CacheControl': 'no-cache'})
s3_instance.upload_file(
self.outdir + filename,
f'{self.outdir}/{filename}',
self.bucket,
latest_dir + filename,
ExtraArgs={'ContentType': ctype,'ACL': "public-read",'CacheControl': 'no-cache'})
f'{latest_dir}/{filename}',
ExtraArgs={'ContentType': ctype, 'ACL': 'public-read', 'CacheControl': 'no-cache'})
except Exception as error:
self.error_message = f'Upload Error: {error}'
self.logger.exception(f'Upload Error: {error}')
raise Exception(self.error_message) from error
def take_screenshot(self,container,tag):
"""Take a screenshot and save it to self.outdir"""
def log_upload(self):
'''Upload debug log to S3'''
self.logger.info('Uploading logs')
destination_dir = f'{self.image}/{self.meta_tag}'
latest_dir = f'{self.image}/latest'
s3_instance = self.session.client(
's3',
region_name=self.region,
aws_access_key_id=self.s3_key,
aws_secret_access_key=self.s3_secret)
# Log file upload
try:
s3_instance.upload_file(
'/debug.log',
self.bucket,
f'{destination_dir}/debug.log',
ExtraArgs={'ContentType': 'text/plain', 'ACL': 'public-read'})
s3_instance.upload_file(
'/debug.log',
self.bucket,
f'{latest_dir}/debug.log',
ExtraArgs={'ContentType': 'text/plain', 'ACL': 'public-read'})
except Exception as error:
self.error_message = f'Upload Error: {error}'
self.logger.exception(f'Upload Error: {error}')
raise Exception(self.error_message) from error
def take_screenshot(self, container, tag):
'''Take a screenshot and save it to self.outdir'''
proto = 'https' if self.ssl == 'true' else 'http'
container.reload()
ip = container.attrs["NetworkSettings"]["Networks"]["bridge"]["IPAddress"]
endpoint = f"{proto}://{self.webauth}@{ip}:{self.port}{self.webpath}"
print(f"Taking screenshot of {tag} at {endpoint}")
ip = container.attrs['NetworkSettings']['Networks']['bridge']['IPAddress']
endpoint = f'{proto}://{self.webauth}@{ip}:{self.port}{self.webpath}'
self.logger.info(f'Taking screenshot of {tag} at {endpoint}')
testercontainer = self.client.containers.run('lsiodev/tester:latest',
shm_size='1G',
detach=True,
environment={'URL': endpoint})
shm_size='1G',
detach=True,
environment={'URL': endpoint})
time.sleep(30)
testercontainer.reload()
testerip = testercontainer.attrs["NetworkSettings"]["Networks"]["bridge"]["IPAddress"]
testerendpoint = "http://" + testerip + ":3000"
testerip = testercontainer.attrs['NetworkSettings']['Networks']['bridge']['IPAddress']
testerendpoint = 'http://' + testerip + ':3000'
try:
# Selenium webdriver options
chrome_options = webdriver.ChromeOptions()
@ -272,21 +307,25 @@ class CI():
driver = webdriver.Chrome(options=chrome_options)
driver.set_page_load_timeout(60)
session = requests.Session()
retries = Retry(total=4, backoff_factor=2, status_forcelist=[ 502, 503, 504 ])
retries = Retry(total=4, backoff_factor=2,
status_forcelist=[502, 503, 504])
session.mount(proto, HTTPAdapter(max_retries=retries))
session.get(testerendpoint)
driver.get(testerendpoint)
time.sleep(15)
driver.get_screenshot_as_file(self.outdir + tag + '.png')
self.report_tests.append(['Screenshot ' + tag,'PASS'])
driver.get_screenshot_as_file(f'{self.outdir}/{tag}.png')
self.report_tests.append([f'Screenshot {tag}', 'PASS'])
# Quit selenium webdriver
driver.quit()
except (requests.Timeout, requests.ConnectionError, KeyError):
self.report_tests.append(['Screenshot ' + tag,'FAIL CONNECTION ERROR'])
#except ErrorInResponseException as error:
# self.report_tests.append(['Screenshot ' + tag,'FAIL SERVER ERROR'])
self.report_tests.append(
[f'Screenshot {tag}', 'FAIL CONNECTION ERROR'])
self.logger.exception(f'Screenshot {tag} FAIL CONNECTION ERROR')
except TimeoutException:
self.report_tests.append(['Screenshot ' + tag,'FAIL TIMEOUT'])
self.report_tests.append([f'Screenshot {tag}', 'FAIL TIMEOUT'])
self.logger.exception(f'Screenshot {tag} FAIL TIMEOUT')
except WebDriverException as error:
self.report_tests.append(['Screenshot ' + tag,f'FAIL UNKNOWN: {error}'])
self.report_tests.append(
[f'Screenshot {tag}', f'FAIL UNKNOWN: {error}'])
self.logger.exception(f'Screenshot {tag} FAIL UNKNOWN: {error}')
testercontainer.remove(force='true')

View File

@ -158,14 +158,27 @@
<h1>Linux<span>Server</span>.io</h1>
</header>
<div id="results"></div>
<section style="max-width: 80vw !important;">
<h3>Debug logs</h3>
<details>
<summary>Expand</summary>
<pre id="debug_logs"></pre>
</details>
</section>
</div>
<script src="https://code.jquery.com/jquery-3.3.1.min.js" type="text/javascript" charset="utf-8"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/showdown/1.8.6/showdown.min.js" type="text/javascript" charset="utf-8"></script>
<script src="https://cdnjs.cloudflare.com/ajax/libs/showdown/1.8.6/showdown.min.js" integrity="sha512-IWAOP5QX4d1WA8ZSKXJtMWdAW8fOifwB0LyZFKaq2WcvsWF/Xmg8aQwxMOUpCjSaYY/V+NASlReVD6uCSDDvHg==" crossorigin="anonymous"></script>
<script type="text/javascript" charset="utf-8">
$.get('report.md').done(function (data) {
fetch('report.md')
.then(response => response.text())
.then(report => {
var converter = new showdown.Converter({parseImgDimensions: true,tables: true,ghCodeBlocks: true});
$('#results').append(converter.makeHtml(data))
});
document.getElementById("results").innerHTML = converter.makeHtml(report)
})
fetch("debug.log")
.then(response => response.text())
.then(logs => {
document.getElementById("debug_logs").innerText = logs
})
</script>
</body>
</html>
</html>

View File

@ -1,26 +1,28 @@
#!/usr/bin/env python3
from multiprocessing.pool import Pool
import sys
from ci.ci import CI
ci = CI()
print(vars(ci))
# Run through all the tags
#pool=Pool(processes=3)
#r = pool.map_async(ci.container_test, ci.tags)
#r.wait()
for tag in ci.tags:
ci.container_test(tag)
ci.report_render()
ci.badge_render()
ci.report_upload()
# Exit based on test results
if ci.report_status == 'PASS':
print('Tests Passed exiting 0')
print(ci.report_tests)
sys.exit(0)
elif ci.report_status == 'FAIL':
print('Tests Failed exiting 1')
sys.exit(1)
def run_test(ci:CI):
logger = ci.logger
for tag in ci.tags: # Run through all the tags
ci.container_test(tag)
ci.report_render()
ci.badge_render()
ci.report_upload()
# Exit based on test results
if ci.report_status == 'PASS':
logger.info('Tests Passed exiting')
logger.info(ci.report_tests)
ci.log_upload()
return
elif ci.report_status == 'FAIL':
logger.error('Tests Failed exiting')
ci.log_upload()
return
if __name__ == '__main__':
ci = CI()
logger = ci.logger
try:
run_test(ci)
except Exception:
logger.exception("I Can't Believe You've Done This")