update run_tests.py to support parallel jobs
This commit is contained in:
Arvid Norberg
@@ -42,10 +42,11 @@ from datetime import datetime
|
|||||||
import json
|
import json
|
||||||
import sys
|
import sys
|
||||||
import yaml
|
import yaml
|
||||||
|
from multiprocessing import Pool
|
||||||
|
|
||||||
# the .regression.yml configuration file format looks like this (it's yaml):
|
# the .regression.yml configuration file format looks like this (it's yaml):
|
||||||
|
|
||||||
# test-dirs:
|
# test_dirs:
|
||||||
# - <path-to-test-folder>
|
# - <path-to-test-folder>
|
||||||
# - ...
|
# - ...
|
||||||
#
|
#
|
||||||
@@ -54,106 +55,39 @@ import yaml
|
|||||||
# - ...
|
# - ...
|
||||||
#
|
#
|
||||||
|
|
||||||
toolsets = sys.argv[1:]
|
def svn_info():
|
||||||
|
# figure out which revision this is
|
||||||
|
p = subprocess.Popen(['svn', 'info'], stdout=subprocess.PIPE)
|
||||||
|
|
||||||
try:
|
revision = -1
|
||||||
cfg = open('.regression.yml', 'r')
|
author = ''
|
||||||
except:
|
|
||||||
print '.regressions.yml not found in current directory'
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
cfg = yaml.load(cfg.read())
|
for l in p.stdout:
|
||||||
|
|
||||||
test_dirs = []
|
|
||||||
configs = []
|
|
||||||
options = ['boost=source']
|
|
||||||
if 'test_dirs' in cfg:
|
|
||||||
for d in cfg['test_dirs']:
|
|
||||||
test_dirs.append(d)
|
|
||||||
else:
|
|
||||||
print 'no test directory specified by .regressions.yml'
|
|
||||||
sys.exit(1)
|
|
||||||
|
|
||||||
configs = []
|
|
||||||
if 'features' in cfg:
|
|
||||||
for d in cfg['features']:
|
|
||||||
configs.append(d.split(' '))
|
|
||||||
else:
|
|
||||||
configs = [['']]
|
|
||||||
|
|
||||||
architecture = platform.machine()
|
|
||||||
build_platform = platform.system() + '-' + platform.release()
|
|
||||||
|
|
||||||
fail_color = '\033[31;1m'
|
|
||||||
pass_color = '\033[32;1m'
|
|
||||||
end_seq = '\033[0m'
|
|
||||||
|
|
||||||
if platform.system() == 'Windows':
|
|
||||||
fail_color == ''
|
|
||||||
pass_color == ''
|
|
||||||
end_seq = ''
|
|
||||||
|
|
||||||
# figure out which revision this is
|
|
||||||
p = subprocess.Popen(['svn', 'info'], stdout=subprocess.PIPE)
|
|
||||||
|
|
||||||
revision = -1
|
|
||||||
author = ''
|
|
||||||
timestamp = datetime.now()
|
|
||||||
|
|
||||||
for l in p.stdout:
|
|
||||||
if 'Last Changed Rev' in l:
|
if 'Last Changed Rev' in l:
|
||||||
revision = int(l.split(':')[1].strip())
|
revision = int(l.split(':')[1].strip())
|
||||||
if 'Last Changed Author' in l:
|
if 'Last Changed Author' in l:
|
||||||
author = l.split(':')[1].strip()
|
author = l.split(':')[1].strip()
|
||||||
|
|
||||||
if revision == -1:
|
if revision == -1:
|
||||||
print 'Failed to extract subversion revision'
|
print 'Failed to extract subversion revision'
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
if author == '':
|
if author == '':
|
||||||
print 'Failed to extract subversion author'
|
print 'Failed to extract subversion author'
|
||||||
sys.exit(1)
|
sys.exit(1)
|
||||||
|
|
||||||
print '%d - %s - %s' % (revision, author, timestamp)
|
return (revision, author)
|
||||||
|
|
||||||
print 'toolsets: ', toolsets
|
def run_tests(toolset, tests, features, options, test_dir):
|
||||||
print 'configs: ', configs
|
xml_file = 'bjam_build.%d.xml' % random.randint(0, 100000)
|
||||||
|
|
||||||
xml_file = 'bjam_build.%d.xml' % random.randint(0, 100000)
|
|
||||||
|
|
||||||
rev_dir = os.path.join(os.getcwd(), 'regression_tests')
|
|
||||||
try: os.mkdir(rev_dir)
|
|
||||||
except: pass
|
|
||||||
rev_dir = os.path.join(rev_dir, '%d' % revision)
|
|
||||||
try: os.mkdir(rev_dir)
|
|
||||||
except: pass
|
|
||||||
|
|
||||||
for test_dir in test_dirs:
|
|
||||||
print 'running tests from %s' % test_dir
|
|
||||||
os.chdir(test_dir)
|
|
||||||
|
|
||||||
# figure out which tests are exported by this Jamfile
|
|
||||||
p = subprocess.Popen(['bjam', '--dump-tests', 'non-existing-target'], stdout=subprocess.PIPE)
|
|
||||||
|
|
||||||
tests = []
|
|
||||||
|
|
||||||
for l in p.stdout:
|
|
||||||
if not 'boost-test(RUN)' in l: continue
|
|
||||||
test_name = os.path.split(l.split(' ')[1][1:-1])[1]
|
|
||||||
tests.append(test_name)
|
|
||||||
print 'found %d tests' % len(tests)
|
|
||||||
|
|
||||||
for toolset in toolsets:
|
|
||||||
print 'toolset %s' % toolset
|
|
||||||
results = {}
|
results = {}
|
||||||
toolset_found = False
|
toolset_found = False
|
||||||
# TODO: run tests in parallel
|
|
||||||
|
os.chdir(test_dir)
|
||||||
|
|
||||||
for t in tests:
|
for t in tests:
|
||||||
print t
|
p = subprocess.Popen(['bjam', '--out-xml=%s' % xml_file, toolset, t] + options + features.split(' '), stdout=subprocess.PIPE)
|
||||||
for features in configs:
|
|
||||||
print 'running %s [%s] [%s]' % (t, toolset, ' '.join(features)),
|
|
||||||
sys.stdout.flush()
|
|
||||||
p = subprocess.Popen(['bjam', '--out-xml=%s' % xml_file, toolset, t] + options + features, stdout=subprocess.PIPE)
|
|
||||||
output = ''
|
output = ''
|
||||||
warnings = 0
|
warnings = 0
|
||||||
for l in p.stdout:
|
for l in p.stdout:
|
||||||
@@ -192,13 +126,138 @@ for test_dir in test_dirs:
|
|||||||
except: pass
|
except: pass
|
||||||
|
|
||||||
r = { 'status': p.returncode, 'output': output, 'warnings': warnings, 'command': command }
|
r = { 'status': p.returncode, 'output': output, 'warnings': warnings, 'command': command }
|
||||||
results[t + '|' + '|'.join(features)] = r
|
results[t + '|' + features] = r
|
||||||
|
|
||||||
|
fail_color = '\033[31;1m'
|
||||||
|
pass_color = '\033[32;1m'
|
||||||
|
end_seq = '\033[0m'
|
||||||
|
|
||||||
|
if platform.system() == 'Windows':
|
||||||
|
fail_color == ''
|
||||||
|
pass_color == ''
|
||||||
|
end_seq = ''
|
||||||
|
|
||||||
|
print '%s [%s] [%s]' % (t, toolset, features),
|
||||||
if p.returncode == 0: print pass_color + 'PASSED' + end_seq
|
if p.returncode == 0: print pass_color + 'PASSED' + end_seq
|
||||||
else: print fail_color + 'FAILED' + end_seq
|
else: print fail_color + 'FAILED' + end_seq
|
||||||
|
|
||||||
|
return (toolset, results)
|
||||||
|
|
||||||
|
def print_usage():
|
||||||
|
print '''usage: run_tests.py [options] bjam-toolset [bjam-toolset...]
|
||||||
|
options:
|
||||||
|
-j<n> use n parallel processes
|
||||||
|
-h prints this message and exits
|
||||||
|
'''
|
||||||
|
|
||||||
|
def main():
|
||||||
|
|
||||||
|
toolsets = []
|
||||||
|
|
||||||
|
num_processes = 2
|
||||||
|
|
||||||
|
for arg in sys.argv[1:]:
|
||||||
|
if arg[0] == '-':
|
||||||
|
if arg[1] == 'j':
|
||||||
|
num_processes = int(arg[2:])
|
||||||
|
elif arg[1] == 'h':
|
||||||
|
print_usage()
|
||||||
|
sys.exit(1)
|
||||||
|
else:
|
||||||
|
print 'unknown option: %s' % arg
|
||||||
|
print_usage()
|
||||||
|
sys.exit(1)
|
||||||
|
else:
|
||||||
|
toolsets.append(arg)
|
||||||
|
|
||||||
|
if toolsets == []:
|
||||||
|
print_usage()
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
try:
|
||||||
|
cfg = open('.regression.yml', 'r')
|
||||||
|
except:
|
||||||
|
print '.regressions.yml not found in current directory'
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
cfg = yaml.load(cfg.read())
|
||||||
|
|
||||||
|
test_dirs = []
|
||||||
|
configs = []
|
||||||
|
options = ['boost=source']
|
||||||
|
if 'test_dirs' in cfg:
|
||||||
|
for d in cfg['test_dirs']:
|
||||||
|
test_dirs.append(d)
|
||||||
|
else:
|
||||||
|
print 'no test directory specified by .regressions.yml'
|
||||||
|
sys.exit(1)
|
||||||
|
|
||||||
|
configs = []
|
||||||
|
if 'features' in cfg:
|
||||||
|
for d in cfg['features']:
|
||||||
|
configs.append(d)
|
||||||
|
else:
|
||||||
|
configs = [['']]
|
||||||
|
|
||||||
|
architecture = platform.machine()
|
||||||
|
build_platform = platform.system() + '-' + platform.release()
|
||||||
|
|
||||||
|
revision, author = svn_info()
|
||||||
|
|
||||||
|
timestamp = datetime.now()
|
||||||
|
|
||||||
|
tester_pool = Pool(processes=4)
|
||||||
|
|
||||||
|
print '%d - %s - %s' % (revision, author, timestamp)
|
||||||
|
|
||||||
|
print 'toolsets: ', toolsets
|
||||||
|
print 'configs: ', configs
|
||||||
|
|
||||||
|
rev_dir = os.path.join(os.getcwd(), 'regression_tests')
|
||||||
|
try: os.mkdir(rev_dir)
|
||||||
|
except: pass
|
||||||
|
rev_dir = os.path.join(rev_dir, '%d' % revision)
|
||||||
|
try: os.mkdir(rev_dir)
|
||||||
|
except: pass
|
||||||
|
|
||||||
|
for test_dir in test_dirs:
|
||||||
|
print 'running tests from "%s"' % test_dir
|
||||||
|
os.chdir(test_dir)
|
||||||
|
test_dir = os.getcwd()
|
||||||
|
|
||||||
|
# figure out which tests are exported by this Jamfile
|
||||||
|
p = subprocess.Popen(['bjam', '--dump-tests', 'non-existing-target'], stdout=subprocess.PIPE)
|
||||||
|
|
||||||
|
tests = []
|
||||||
|
|
||||||
|
for l in p.stdout:
|
||||||
|
if not 'boost-test(RUN)' in l: continue
|
||||||
|
test_name = os.path.split(l.split(' ')[1][1:-1])[1]
|
||||||
|
tests.append(test_name)
|
||||||
|
print 'found %d tests' % len(tests)
|
||||||
|
|
||||||
|
for toolset in toolsets:
|
||||||
|
print 'toolset %s' % toolset
|
||||||
|
results = {}
|
||||||
|
toolset_found = False
|
||||||
|
|
||||||
|
futures = []
|
||||||
|
for features in configs:
|
||||||
|
futures.append(tester_pool.apply_async(run_tests, [toolset, tests, features, options, test_dir]))
|
||||||
|
|
||||||
|
for future in futures:
|
||||||
|
(toolset, r) = future.get()
|
||||||
|
results.update(r)
|
||||||
|
|
||||||
|
# for features in configs:
|
||||||
|
# (toolset, r) = run_tests(toolset, tests, features, options, test_dir)
|
||||||
|
# results.update(r)
|
||||||
|
|
||||||
# each file contains a full set of tests for one speific toolset and platform
|
# each file contains a full set of tests for one speific toolset and platform
|
||||||
f = open(os.path.join(rev_dir, build_platform + '#' + toolset + '.json'), 'w+')
|
f = open(os.path.join(rev_dir, build_platform + '#' + toolset + '.json'), 'w+')
|
||||||
print >>f, json.dumps(results)
|
print >>f, json.dumps(results)
|
||||||
f.close()
|
f.close()
|
||||||
|
|
||||||
|
if __name__ == "__main__":
|
||||||
|
main()
|
||||||
|
|
||||||
|
|||||||
Reference in New Issue
Block a user