Mercurial > hg > truffle
changeset 4215:a2caa019ba3a
Fix mx : commands' scripts mx_init hook should be called before parsing command line arguments.
Fix mx : call the mx_post_parse_cmd_line hook from commands' scripts
OutputParser : cosmetic changes to logged output, return the retcode along yith the parsed output
Add a new Test class representing a sanity check and/or a benchmark
Port dacapo command to use this class, begning work on benchmarks
| author | Gilles Duboscq <gilles.m.duboscq@gmail.com> |
|---|---|
| date | Wed, 04 Jan 2012 13:52:46 +0100 |
| parents | e4cfa571d8c4 |
| children | a13d61d3910e |
| files | mx/commands.py mx/outputparser.py mx/sanitycheck.py mxtool/mx.py |
| diffstat | 4 files changed, 141 insertions(+), 102 deletions(-) [+] |
line wrap: on
line diff
--- a/mx/commands.py Tue Jan 03 18:47:27 2012 -0800 +++ b/mx/commands.py Wed Jan 04 13:52:46 2012 +0100 @@ -27,9 +27,10 @@ # ---------------------------------------------------------------------------------------------------- import os, sys, shutil, StringIO, zipfile, tempfile, re, time, datetime, platform, subprocess -from os.path import join, exists, dirname, isdir, isfile, isabs, basename +from os.path import join, exists, dirname, isdir, isabs, basename from argparse import ArgumentParser, REMAINDER import mx +import sanitycheck _graal_home = dirname(dirname(__file__)) _vmSourcesAvailable = exists(join(_graal_home, 'make')) and exists(join(_graal_home, 'src')) @@ -140,64 +141,50 @@ DaCapo options are distinguised from VM options by a '@' prefix. For example, '@--iterations @5' will pass '--iterations 5' to the DaCapo harness.""" - - benchmarks = [ - 'avrora', - 'batik', - 'eclipse', - 'fop', - 'h2', - 'jython', - 'luindex', - 'lusearch', - 'pmd', - 'sunflow', - 'tomcat', - 'tradebeans', - 'tradesoap', - 'xalan' - ] + + numTests = {} - dacapo = mx.get_env('DACAPO_CP') - if dacapo is None: - dacapo = _graal_home + r'/lib/dacapo-9.12-bach.jar' + if len(args) > 0: + level = getattr(sanitycheck.SanityCheckLevel, args[0], None) + if level is not None: + del args[0] + for (bench, ns) in sanitycheck.dacapoSanityWarmup.items(): + if ns[level] > 0: + numTests[bench] = ns[level] + else: + while len(args) != 0 and args[0][0] not in ['-', '@']: + n = 1 + if args[0].isdigit(): + n = int(args[0]) + assert len(args) > 1 and args[1][0] not in ['-', '@'] and not args[1].isdigit() + bm = args[1] + del args[0] + else: + bm = args[0] + + del args[0] + if bm not in sanitycheck.dacapoSanityWarmup.keys(): + mx.abort('unknown benchmark: ' + bm + '\nselect one of: ' + str(sanitycheck.dacapoSanityWarmup.keys())) + numTests[bm] = n - if not isfile(dacapo) or not dacapo.endswith('.jar'): - mx.abort('Specified DaCapo jar file does not exist or is not a jar file: ' + dacapo) - - vmOpts = ['-Xms1g', '-Xmx2g', '-cp', dacapo] - - selected = [] - while len(args) != 0 and args[0][0] not in ['-', '@']: - bm = args[0] - del args[0] - if bm not in benchmarks: - mx.abort('unknown benchmark: ' + bm + '\nselect one of: ' + str(benchmarks)) - selected.append(bm) - - if len(selected) != 0: - benchmarks = selected + if len(numTests) is 0: + for bench in sanitycheck.dacapoSanityWarmup.keys(): + numTests[bench] = 1 # Extract DaCapo options dacapoArgs = [(arg[1:]) for arg in args if arg.startswith('@')] # The remainder are VM options - vmOpts += [arg for arg in args if not arg.startswith('@')] - - dacapoSuccess = re.compile(r"^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====$") - passed = [] - - for bm in benchmarks: - def errFilter(line): - if dacapoSuccess.match(line): - passed.append(bm) - sys.stderr.write(line) - vm(vmOpts + ['Harness'] + dacapoArgs + [bm], err=errFilter) - - failed = list(set(benchmarks).difference(set(passed))) + vmOpts = [arg for arg in args if not arg.startswith('@')] + + failed = [] + print str(numTests) + for (test, n) in numTests.items(): + if not sanitycheck.getDacapo(test, n, dacapoArgs).test('-graal', opts=vmOpts): + failed.append(test) if len(failed) != 0: - mx.abort('Benchmark failures: ' + str(failed)) + mx.abort('DaCapo failures: ' + str(failed)) def _jdk(build='product', create=False): """ @@ -246,7 +233,7 @@ mx.abort('Unknown build type: ' + build) # run a command in the windows SDK Debug Shell -def runInDebugShell(cmd, workingDir, logFile=None, findInOutput=None, respondTo={}): +def _runInDebugShell(cmd, workingDir, logFile=None, findInOutput=None, respondTo={}): newLine = os.linesep STARTTOKEN = 'RUNINDEBUGSHELL_STARTSEQUENCE' ENDTOKEN = 'RUNINDEBUGSHELL_ENDSEQUENCE' @@ -310,16 +297,16 @@ if platform.system() == 'Windows': compilelogfile = _graal_home + '/graalCompile.log' - runInDebugShell('msbuild ' + _graal_home + r'\build\vs-amd64\jvm.vcproj /p:Configuration=compiler1_product /target:clean', _graal_home) + _runInDebugShell('msbuild ' + _graal_home + r'\build\vs-amd64\jvm.vcproj /p:Configuration=compiler1_product /target:clean', _graal_home) winCompileCmd = r'set HotSpotMksHome=' + _mksHome + r'& set OUT_DIR=' + jdk + r'& set JAVA_HOME=' + jdk + r'& set path=%JAVA_HOME%\bin;%path%;%HotSpotMksHome%& cd /D "' +_graal_home + r'\make\windows"& call create.bat ' + _graal_home + '' print(winCompileCmd) winCompileSuccess = re.compile(r"^Writing \.vcxproj file:") - if not runInDebugShell(winCompileCmd, _graal_home, compilelogfile, winCompileSuccess): + if not _runInDebugShell(winCompileCmd, _graal_home, compilelogfile, winCompileSuccess): mx.log('Error executing create command') return winBuildCmd = 'msbuild ' + _graal_home + r'\build\vs-amd64\jvm.vcxproj /p:Configuration=compiler1_product /p:Platform=x64' winBuildSuccess = re.compile('Build succeeded.') - if not runInDebugShell(winBuildCmd, _graal_home, compilelogfile, winBuildSuccess): + if not _runInDebugShell(winBuildCmd, _graal_home, compilelogfile, winBuildSuccess): mx.log('Error building project') return else: @@ -581,6 +568,11 @@ duration = datetime.timedelta(seconds=time.time() - start) mx.log(time.strftime('%d %b %Y %H:%M:%S - Gate done (duration - ' + str(duration) + ')')) + +def bench(args): + + for test in sanitycheck.getDacapos(level=sanitycheck.SanityCheckLevel.Fast): + print test.bench('-graal') def mx_init(): _vmbuild = 'product' @@ -591,6 +583,7 @@ 'dacapo': [dacapo, '[benchmark] [VM options|DaCapo options]'], 'example': [example, '[-v] example names...'], 'gate' : [gate, ''], + 'bench' : [bench, ''], 'unittest' : [unittest, '[filters...]'], 'vm': [vm, '[-options] class [args...]'], 'ideinit': [ideinit, ''] @@ -617,7 +610,7 @@ major = int(parts[1]) if not major >= 7: mx.abort('Requires Java version 1.7 or greater, got version ' + version) - + if (_vmSourcesAvailable): if hasattr(opts, 'vmbuild'): global _vmbuild
--- a/mx/outputparser.py Tue Jan 03 18:47:27 2012 -0800 +++ b/mx/outputparser.py Wed Jan 04 13:52:46 2012 +0100 @@ -10,26 +10,25 @@ def addMatcher(self, matcher): self.matchers.append(matcher) - def parse(self, cmd, cwd=None): + def parse(self, vm, cmd, cwd=None): ret = [{}] def parseLine(line): - line = line.strip() anyMatch = False for matcher in self.matchers: - parsed = matcher.parse(line) + parsed = matcher.parse(line.strip()) if parsed: anyMatch = True if matcher.startNewLine and len(ret[0]) > 0: ret.append({}) ret[len(ret)-1].update(parsed) if anyMatch : - mx.log(line) + mx.log('>' + line.rstrip()) else : - mx.log('# ' + line) + mx.log( line.rstrip()) - commands.vm(cmd, nonZeroIsFatal=self.nonZeroIsFatal, out=parseLine, err=parseLine, cwd=cwd) - return ret + retcode = commands.vm(cmd, nonZeroIsFatal=self.nonZeroIsFatal, out=parseLine, err=parseLine, cwd=cwd) + return {'parsed' : ret, 'retcode' : retcode} class Matcher:
--- a/mx/sanitycheck.py Tue Jan 03 18:47:27 2012 -0800 +++ b/mx/sanitycheck.py Wed Jan 04 13:52:46 2012 +0100 @@ -2,14 +2,13 @@ import re import mx import os +import commands from os.path import isfile -dacapoVMOpts = ['-Xms1g', '-Xmx2g'] - dacapoSanityWarmup = { 'avrora': [0, 0, 3, 6], 'batik': [0 , 0, 5, 5], - 'eclipse': [1 , 4, 5, 10], + 'eclipse': [2 , 4, 5, 10], 'fop': [4 , 8, 10, 20], 'h2': [0 , 0, 5, 5], 'jython': [0 , 0, 5, 10], @@ -19,49 +18,94 @@ 'sunflow': [0 , 0, 5, 10], 'tomcat': [0 , 0, 5, 10], 'tradebeans': [0 , 0, 5, 10], - 'tradesoap': [0 , 4, 5, 10], + 'tradesoap': [2 , 4, 5, 10], 'xalan': [0 , 0, 5, 10], } -def getDacapoCmd(bench, vmOpts=dacapoVMOpts,n=5): - dacapo = mx.check_get_env('DACAPO_CP') - if not isfile(dacapo) or not dacapo.endswith('.jar'): - mx.abort('Specified DaCapo jar file does not exist or is not a jar file: ' + dacapo) - return vmOpts + ['-cp', dacapo, 'Harness', '-n', str(n), bench] - class SanityCheckLevel: Fast, Gate, Normal, Extensive = range(4) -def getSanityChecks(level=SanityCheckLevel.Normal): - checks = [] +def getDacapos(level=SanityCheckLevel.Normal, dacapoArgs=None): + checks = {} - dacapoSuccess = re.compile(r"^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====$") for (bench, ns) in dacapoSanityWarmup.items(): if ns[level] > 0: - checks.append({'cmd' : getDacapoCmd(bench, vmOpts=dacapoVMOpts + ['-esa'], n=ns[level]), 'success' : dacapoSuccess}) - - bootstrapSuccess = re.compile(r"in [0-9]+ ms$"); - checks.append({'cmd' : ['-esa', '-version'], 'success' : bootstrapSuccess}) + checks[bench] = getDacapo(bench, ns[level], dacapoArgs) return checks -def runSanityCheck(cmd, successRE, cwd=None): - parser = OutputParser(nonZeroIsFatal=False) - jvmError = re.compile(r"\b(?P<jvmerror>([A-Z]:|/).*[/\\]hs_err_pid[0-9]+\.log)\b") - parser.addMatcher(Matcher(successRE, {'const:passed' : 'const:1'})) - parser.addMatcher(Matcher(jvmError, {'const:jvmError' : 'jvmerror'})) +def getDacapo(name, n, dacapoArgs=None): + dacapo = mx.get_env('DACAPO_CP') + if dacapo is None: + dacapo = commands._graal_home + r'/lib/dacapo-9.12-bach.jar' + + if not isfile(dacapo) or not dacapo.endswith('.jar'): + mx.abort('Specified DaCapo jar file does not exist or is not a jar file: ' + dacapo) + + dacapoSuccess = re.compile(r"^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====$") + dacapoFail = re.compile(r"^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) FAILED (warmup|) =====$") + dacapoTime = re.compile(r"===== DaCapo 9\.12 (?P<benchmark>[a-zA-Z0-9_]+) PASSED in (?P<time>[0-9]+) msec =====") - result = parser.parse(cmd, cwd) - assert len(result) == 1, 'Sanity check matchers should not return more than one line' - parsed = result[0] + dacapoMatcher = Matcher(dacapoTime, {'const:name' : 'benchmark', 'const:score' : 'time'}) + + return Test("DaCapo-" + name, "DaCapo", ['-jar', dacapo, name, '-n', str(n), ] + dacapoArgs, [dacapoSuccess], [dacapoFail], dacapoMatcher, ['-Xms1g', '-Xmx2g', '-XX:MaxPermSize=256m']) + +class Test: + def __init__(self, name, group, cmd, succesREs=None, failureREs=None, scoreMatchers=None, vmOpts=None): + self.name = name + self.group = group + self.succesREs = succesREs + self.failureREs = failureREs + self.scoreMatchers = scoreMatchers + self.vmOpts = vmOpts + self.cmd = cmd - if parsed.has_key('jvmError'): - mx.log('JVM Error : dumping error log...') - f = open(parsed['jvmError'], 'rb'); - for line in iter(f.readline(), ''): - mx.log(line) - f.close() - os.unlink(parsed['jvmError']) - return False - return parsed.has_key('passed') - \ No newline at end of file + def test(self, vm, cwd=None, opts=None): + parser = OutputParser(nonZeroIsFatal = False) + jvmError = re.compile(r"(?P<jvmerror>([A-Z]:|/).*[/\\]hs_err_pid[0-9]+\.log)") + parser.addMatcher(Matcher(jvmError, {'const:jvmError' : 'jvmerror'})) + + for succesRE in self.succesREs: + parser.addMatcher(Matcher(succesRE, {'const:passed' : 'const:1'})) + for failureRE in self.failureREs: + parser.addMatcher(Matcher(failureRE, {'const:failed' : 'const:1'})) + + result = parser.parse(vm, self.vmOpts + opts + self.cmd, cwd) + + parsedLines = result['parsed'] + assert len(parsedLines) == 1, 'Test matchers should not return more than one line' + + parsed = parsedLines[0] + + if parsed.has_key('jvmError'): + mx.log('/!\\JVM Error : dumping error log...') + f = open(parsed['jvmError'], 'rb'); + for line in iter(f.readline, ''): + mx.log(line.rstrip()) + f.close() + os.unlink(parsed['jvmError']) + return False + + if parsed.has_key('failed') and parsed['failed'] is 1: + return False + + return result['retcode'] is 0 and parsed.has_key('passed') and parsed['passed'] is '1' + + def bench(self, vm, cwd=None, opts=None): + parser = OutputParser(nonZeroIsFatal = False) + + for scoreMatcher in self.scoreMatchers: + parser.addMatcher(scoreMatcher) + + result = parser.parse(self.vmOpts + opts + self.cmd, vm, cwd) + + parsed = result['parsed'] + + ret = {} + + for line in parsed: + assert line.has_key('name') and line.has_key('score') + ret[line['name']] = line['score'] + + return ret + \ No newline at end of file
--- a/mxtool/mx.py Tue Jan 03 18:47:27 2012 -0800 +++ b/mxtool/mx.py Wed Jan 04 13:52:46 2012 +0100 @@ -231,7 +231,10 @@ self.includes = [] self.commands = None self.primary = primary - self._load_env(join(dir, 'mx')) + mxDir = join(dir, 'mx') + self._load_env(mxDir) + if primary: + self._load_commands(mxDir) def _load_projects(self, mxDir): libsMap = dict() @@ -308,6 +311,8 @@ if not hasattr(mod, 'mx_init'): abort(commands + ' must define an mx_init(env) function') + if hasattr(mod, 'mx_post_parse_cmd_line'): + self.mx_post_parse_cmd_line = mod.mx_post_parse_cmd_line mod.mx_init() self.commands = mod @@ -333,10 +338,8 @@ mxDir = join(self.dir, 'mx') self._load_includes(mxDir) self._load_projects(mxDir) - if self.primary: - self._load_commands(mxDir) - if commands is not None and hasattr(commands, 'mx_post_parse_cmd_line'): - commands.mx_post_parse_cmd_line(opts) + if self.mx_post_parse_cmd_line is not None: + self.mx_post_parse_cmd_line(opts) for p in self.projects: existing = _projects.get(p.name) if existing is not None: