mirror of https://github.com/vitalif/openscad
476 lines
14 KiB
Python
Executable File
476 lines
14 KiB
Python
Executable File
#!/usr/bin/python
|
|
|
|
# test_pretty_print by don bright 2012. Copyright assigned to Marius Kintel and
|
|
# Clifford Wolf 2012. Released under the GPL 2, or later, as described in
|
|
# the file named 'COPYING' in OpenSCAD's project root.
|
|
|
|
#
|
|
# This program 'pretty prints' the ctest output, including
|
|
# - log files from builddir/Testing/Temporary/
|
|
# - .png and .txt files from testname-output/*
|
|
#
|
|
# The result is a single html report file with images data-uri encoded
|
|
# into the file. It can be uploaded as a single static file to a web server
|
|
# or the 'test_upload.py' script can be used.
|
|
|
|
|
|
# Design philosophy
|
|
#
|
|
# 1. parse the data (images, logs) into easy-to-use data structures
|
|
# 2. wikifiy the data
|
|
# 3. save the wikified data to disk
|
|
# 4. generate html, including base64 encoding of images
|
|
# 5. save html file
|
|
# 6. upload html to public site and share with others
|
|
|
|
# todo
|
|
#
|
|
# 1. why is hash differing
|
|
|
|
import string
|
|
import sys
|
|
import re
|
|
import os
|
|
import hashlib
|
|
import subprocess
|
|
import time
|
|
import platform
|
|
try:
|
|
from urllib.error import URLError
|
|
from urllib.request import urlopen
|
|
from urllib.parse import urlencode
|
|
except:
|
|
from urllib2 import URLError
|
|
from urllib2 import urlopen
|
|
from urllib import urlencode
|
|
|
|
def tryread(filename):
|
|
data = None
|
|
try:
|
|
f = open(filename,'rb')
|
|
data = f.read()
|
|
f.close()
|
|
except Exception as e:
|
|
print 'couldn\'t open ',filename
|
|
print type(e), e
|
|
return data
|
|
|
|
def trysave(filename, data):
|
|
dir = os.path.dirname(filename)
|
|
try:
|
|
if not os.path.isdir(dir):
|
|
if not dir == '':
|
|
debug( 'creating' + dir)
|
|
os.mkdir(dir)
|
|
f=open(filename,'wb')
|
|
f.write(data)
|
|
f.close()
|
|
except Exception as e:
|
|
print 'problem writing to',filename
|
|
print type(e), e
|
|
return None
|
|
return True
|
|
|
|
def ezsearch(pattern, str):
|
|
x = re.search(pattern,str,re.DOTALL|re.MULTILINE)
|
|
if x and len(x.groups())>0: return x.group(1).strip()
|
|
return ''
|
|
|
|
def read_gitinfo():
|
|
# won't work if run from outside of branch.
|
|
try:
|
|
data = subprocess.Popen(['git', 'remote', '-v'], stdout=subprocess.PIPE).stdout.read()
|
|
origin = ezsearch('^origin *?(.*?)\(fetch.*?$', data)
|
|
upstream = ezsearch('^upstream *?(.*?)\(fetch.*?$', data)
|
|
data = subprocess.Popen(['git', 'branch'], stdout=subprocess.PIPE).stdout.read()
|
|
branch = ezsearch('^\*(.*?)$', data)
|
|
out = 'Git branch: ' + branch + ' from origin ' + origin + '\n'
|
|
out += 'Git upstream: ' + upstream + '\n'
|
|
except:
|
|
out = 'Problem running git'
|
|
return out
|
|
|
|
def read_sysinfo(filename):
|
|
data = tryread(filename)
|
|
if not data:
|
|
sinfo = platform.sys.platform
|
|
sinfo += '\nsystem cannot create offscreen GL framebuffer object'
|
|
sinfo += '\nsystem cannot create GL based images'
|
|
sysid = platform.sys.platform+'_no_GL_renderer'
|
|
return sinfo, sysid
|
|
|
|
machine = ezsearch('Machine:(.*?)\n',data)
|
|
machine = machine.replace(' ','-').replace('/','-')
|
|
|
|
osinfo = ezsearch('OS info:(.*?)\n',data)
|
|
osplain = osinfo.split(' ')[0].strip().replace('/','-')
|
|
if 'windows' in osinfo.lower():
|
|
osplain = 'win'
|
|
|
|
renderer = ezsearch('GL Renderer:(.*?)\n',data)
|
|
tmp = renderer.split(' ')
|
|
tmp = string.join(tmp[0:3],'-')
|
|
tmp = tmp.split('/')[0]
|
|
renderer = tmp
|
|
|
|
data += read_gitinfo()
|
|
|
|
data += 'Image comparison: ImageMagick'
|
|
|
|
data = data.strip()
|
|
|
|
# create 4 letter hash and stick on end of sysid
|
|
nondate_data = re.sub("\n.*?ompile date.*?\n", "\n", data).strip()
|
|
hexhash = hashlib.md5(nondate_data).hexdigest()[-4:].upper()
|
|
hash_ = ''.join(chr(ord(c) + 97 - 48) for c in hexhash)
|
|
|
|
sysid = '_'.join([osplain, machine, renderer, hash_])
|
|
sysid = sysid.replace('(', '_').replace(')', '_')
|
|
sysid = sysid.lower()
|
|
|
|
return data, sysid
|
|
|
|
class Test:
|
|
def __init__(self, fullname, subpr, passed, output, type, actualfile,
|
|
expectedfile, scadfile, log):
|
|
self.fullname, self.time = fullname, time
|
|
self.passed, self.output = passed, output
|
|
self.type, self.actualfile = type, actualfile
|
|
self.expectedfile, self.scadfile = expectedfile, scadfile
|
|
self.fulltestlog = log
|
|
|
|
def __str__(self):
|
|
x = 'fullname: ' + self.fullname
|
|
x+= '\nactualfile: ' + self.actualfile
|
|
x+= '\nexpectedfile: ' + self.expectedfile
|
|
x+= '\ntesttime: ' + self.time
|
|
x+= '\ntesttype: ' + self.type
|
|
x+= '\npassed: ' + str(self.passed)
|
|
x+= '\nscadfile: ' + self.scadfile
|
|
x+= '\noutput bytes: ' + str(len(self.output))
|
|
x+= '\ntestlog bytes: ' + str(len(self.fulltestlog))
|
|
x+= '\n'
|
|
return x
|
|
|
|
def parsetest(teststring):
|
|
patterns = ["Test:(.*?)\n", # fullname
|
|
"Test time =(.*?) sec\n",
|
|
"Test time.*?Test (Passed)", # pass/fail
|
|
"Output:(.*?)<end of output>",
|
|
'Command:.*?-s" "(.*?)"', # type
|
|
"^ actual .*?:(.*?)\n",
|
|
"^ expected .*?:(.*?)\n",
|
|
'Command:.*?(testdata.*?)"' # scadfile
|
|
]
|
|
hits = map( lambda pattern: ezsearch(pattern, teststring), patterns)
|
|
test = Test(hits[0], hits[1], hits[2]=='Passed', hits[3], hits[4], hits[5],
|
|
hits[6], hits[7], teststring)
|
|
if len(test.actualfile) > 0:
|
|
test.actualfile_data = tryread(test.actualfile)
|
|
if len(test.expectedfile) > 0:
|
|
test.expectedfile_data = tryread(test.expectedfile)
|
|
return test
|
|
|
|
def parselog(data):
|
|
startdate = ezsearch('Start testing: (.*?)\n', data)
|
|
enddate = ezsearch('End testing: (.*?)\n', data)
|
|
pattern = '([0-9]*/[0-9]* Testing:.*?time elapsed.*?\n)'
|
|
test_chunks = re.findall(pattern,data, re.S)
|
|
tests = map( parsetest, test_chunks )
|
|
tests = sorted(tests, key = lambda t: t.passed)
|
|
return startdate, tests, enddate
|
|
|
|
def load_makefiles(builddir):
|
|
filelist = []
|
|
for root, dirs, files in os.walk(builddir):
|
|
for fname in files: filelist += [ os.path.join(root, fname) ]
|
|
files = [file for file in filelist if 'build.make' in os.path.basename(file)
|
|
or 'flags.make' in os.path.basename(file)]
|
|
files = [file for file in files if 'esting' not in file and 'emporary' not in file]
|
|
result = {}
|
|
for fname in files:
|
|
result[fname.replace(builddir, '')] = tryread(fname)
|
|
return result
|
|
|
|
|
|
def png_encode64(fname, width=250, data=None):
|
|
# en.wikipedia.org/wiki/Data_URI_scheme
|
|
data = data or tryread(fname) or ''
|
|
data_uri = data.encode('base64').replace('\n', '')
|
|
tag = '''<img src="data:image/png;base64,%s" width="%s" %s/>'''
|
|
if data == '':
|
|
alt = 'alt="error: no image generated" '
|
|
else:
|
|
alt = 'alt="openscad_test_image" '
|
|
tag %= (data_uri, width, alt)
|
|
return tag
|
|
|
|
|
|
def findlogfile(builddir):
|
|
logpath = os.path.join(builddir, 'Testing', 'Temporary')
|
|
logfilename = os.path.join(logpath, 'LastTest.log.tmp')
|
|
if not os.path.isfile(logfilename):
|
|
logfilename = os.path.join(logpath, 'LastTest.log')
|
|
if not os.path.isfile(logfilename):
|
|
print 'can\'t find and/or open logfile', logfilename
|
|
sys.exit()
|
|
return logfilename
|
|
|
|
# --- Templating ---
|
|
|
|
class Templates(object):
|
|
html_template = '''<html>
|
|
<head><title>Test run for {sysid}</title>
|
|
{style}
|
|
</head>
|
|
<body>
|
|
<h1>{project_name} test run report</h1>
|
|
<p>
|
|
<b>Sysid</b>: {sysid}
|
|
</p>
|
|
<p>
|
|
<b>Result summary</b>: {numpassed} / {numtests} tests passed ({percent}%)
|
|
</p>
|
|
|
|
<h2>System info</h2>
|
|
<pre>{sysinfo}</pre>
|
|
|
|
<p>start time: {startdate}</p>
|
|
<p>end time: {enddate}</p>
|
|
|
|
<h2>Image tests</h2>
|
|
{image_tests}
|
|
|
|
<h2>Text tests</h2>
|
|
{text_tests}
|
|
|
|
<h2>build.make and flags.make</h2>
|
|
{makefiles}
|
|
</body></html>'''
|
|
|
|
style = '''
|
|
<style>
|
|
body {
|
|
color: black;
|
|
}
|
|
|
|
table {
|
|
border-collapse: collapse;
|
|
}
|
|
|
|
table td, th {
|
|
border: 2px solid gray;
|
|
}
|
|
|
|
.text-name {
|
|
border: 2px solid black;
|
|
padding: 0.14em;
|
|
}
|
|
</style>'''
|
|
|
|
image_template = '''<table>
|
|
<tbody>
|
|
<tr><td colspan="2">{test_name}</td></tr>
|
|
<tr><td> Expected image </td><td> Actual image </td></tr>
|
|
<tr><td> {expected} </td><td> {actual} </td></tr>
|
|
</tbody>
|
|
</table>
|
|
|
|
<pre>
|
|
{test_log}
|
|
</pre>
|
|
'''
|
|
|
|
text_template = '''
|
|
<span class="text-name">{test_name}</span>
|
|
|
|
<pre>
|
|
{test_log}
|
|
</pre>
|
|
'''
|
|
|
|
makefile_template = '''
|
|
<h4>{name}</h4>
|
|
<pre>
|
|
{text}
|
|
</pre>
|
|
'''
|
|
|
|
def __init__(self, **defaults):
|
|
self.filled = {}
|
|
self.defaults = defaults
|
|
|
|
def fill(self, template, *args, **kwargs):
|
|
kwds = self.defaults.copy()
|
|
kwds.update(kwargs)
|
|
return getattr(self, template).format(*args, **kwds)
|
|
|
|
def add(self, template, var, *args, **kwargs):
|
|
self.filled[var] = self.filled.get(var, '') + self.fill(template, *args, **kwargs)
|
|
return self.filled[var]
|
|
|
|
def get(self, var):
|
|
return self.filled.get(var, '')
|
|
|
|
|
|
def to_html(project_name, startdate, tests, enddate, sysinfo, sysid, makefiles):
|
|
passed_tests = [test for test in tests if test.passed]
|
|
failed_tests = [test for test in tests if not test.passed]
|
|
|
|
report_tests = failed_tests
|
|
if include_passed:
|
|
report_tests = tests
|
|
|
|
percent = '%.0f' % (100.0 * len(passed_tests) / len(tests)) if tests else 'n/a'
|
|
|
|
templates = Templates()
|
|
for test in report_tests:
|
|
if test.type in ('txt', 'ast', 'csg', 'term', 'echo'):
|
|
templates.add('text_template', 'text_tests',
|
|
test_name=test.fullname,
|
|
test_log=test.fulltestlog)
|
|
elif test.type == 'png':
|
|
# FIXME: Handle missing test.actualfile or test.expectedfile
|
|
actual_img = png_encode64(test.actualfile,
|
|
data=vars(test).get('actualfile_data'))
|
|
expected_img = png_encode64(test.expectedfile,
|
|
data=vars(test).get('expectedfile_data'))
|
|
templates.add('image_template', 'image_tests',
|
|
test_name=test.fullname,
|
|
test_log=test.fulltestlog,
|
|
actual=actual_img,
|
|
expected=expected_img)
|
|
else:
|
|
raise TypeError('Unknown test type %r' % test.type)
|
|
|
|
for mf in sorted(makefiles.keys()):
|
|
mfname = mf.strip().lstrip(os.path.sep)
|
|
text = open(os.path.join(builddir, mfname)).read()
|
|
templates.add('makefile_template', 'makefiles', name=mfname, text=text)
|
|
|
|
text_tests = templates.get('text_tests')
|
|
image_tests = templates.get('image_tests')
|
|
makefiles_str = templates.get('makefiles')
|
|
|
|
return templates.fill('html_template', style=Templates.style,
|
|
sysid=sysid, sysinfo=sysinfo,
|
|
startdate=startdate, enddate=enddate,
|
|
project_name=project_name,
|
|
numtests=len(tests),
|
|
numpassed=len(passed_tests),
|
|
percent=percent, image_tests=image_tests,
|
|
text_tests=text_tests, makefiles=makefiles_str)
|
|
|
|
# --- End Templating ---
|
|
|
|
# --- Web Upload ---
|
|
|
|
def postify(data):
|
|
return urlencode(data).encode()
|
|
|
|
def create_page():
|
|
data = {
|
|
'action': 'create',
|
|
'type': 'html'
|
|
}
|
|
try:
|
|
response = urlopen('http://www.dinkypage.com', data=postify(data))
|
|
except:
|
|
return None
|
|
return response.geturl()
|
|
|
|
def upload_html(page_url, title, html):
|
|
data = {
|
|
'mode': 'editor',
|
|
'title': title,
|
|
'html': html,
|
|
'ajax': '1'
|
|
}
|
|
try:
|
|
response = urlopen(page_url, data=postify(data))
|
|
except URLError, e:
|
|
print 'Upload error: ' + str(e)
|
|
return False
|
|
return 'success' in response.read().decode()
|
|
|
|
# --- End Web Upload ---
|
|
|
|
def debug(x):
|
|
if debug_test_pp:
|
|
print 'test_pretty_print: ' + x
|
|
|
|
debug_test_pp = False
|
|
include_passed = False
|
|
builddir = os.getcwd()
|
|
|
|
def main():
|
|
global builddir, debug_test_pp
|
|
global maxretry, dry, include_passed
|
|
project_name = 'OpenSCAD'
|
|
|
|
if bool(os.getenv("TEST_GENERATE")):
|
|
sys.exit(0)
|
|
|
|
# --- Command Line Parsing ---
|
|
|
|
if '--debug' in ' '.join(sys.argv):
|
|
debug_test_pp = True
|
|
maxretry = 10
|
|
|
|
if '--include-passed' in sys.argv:
|
|
include_passed = True
|
|
|
|
dry = False
|
|
debug('running test_pretty_print')
|
|
if '--dryrun' in sys.argv:
|
|
dry = True
|
|
|
|
suffix = ezsearch('--suffix=(.*?) ', ' '.join(sys.argv) + ' ')
|
|
builddir = ezsearch('--builddir=(.*?) ', ' '.join(sys.argv) + ' ')
|
|
if not builddir:
|
|
builddir = os.getcwd()
|
|
debug('build dir set to ' + builddir)
|
|
|
|
upload = False
|
|
if '--upload' in sys.argv:
|
|
upload = True
|
|
debug('will upload test report')
|
|
|
|
# Workaround for old cmake's not being able to pass parameters
|
|
# to CTEST_CUSTOM_POST_TEST
|
|
if bool(os.getenv("OPENSCAD_UPLOAD_TESTS")):
|
|
upload = True
|
|
|
|
# --- End Command Line Parsing ---
|
|
|
|
sysinfo, sysid = read_sysinfo(os.path.join(builddir, 'sysinfo.txt'))
|
|
makefiles = load_makefiles(builddir)
|
|
logfilename = findlogfile(builddir)
|
|
testlog = tryread(logfilename)
|
|
startdate, tests, enddate = parselog(testlog)
|
|
if debug_test_pp:
|
|
print 'found sysinfo.txt,',
|
|
print 'found', len(makefiles),'makefiles,',
|
|
print 'found', len(tests),'test results'
|
|
|
|
|
|
html = to_html(project_name, startdate, tests, enddate, sysinfo, sysid, makefiles)
|
|
html_basename = sysid + '_report.html'
|
|
html_filename = os.path.join(builddir, 'Testing', 'Temporary', html_basename)
|
|
debug('saving ' + html_filename + ' ' + str(len(html)) + ' bytes')
|
|
trysave(html_filename, html)
|
|
print "report saved:\n", html_filename.replace(os.getcwd()+os.path.sep,'')
|
|
|
|
if upload:
|
|
page_url = create_page()
|
|
if upload_html(page_url, title='OpenSCAD test results', html=html):
|
|
share_url = page_url.partition('?')[0]
|
|
print 'html report uploaded at', share_url
|
|
else:
|
|
print 'could not upload html report'
|
|
|
|
debug('test_pretty_print complete')
|
|
|
|
if __name__=='__main__':
|
|
main()
|