Mercurial > hg > truffle

comparison mx/sanitycheck.py @ 7564:c420a487b10f

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
changed convention for specifying constants versus named groups in a template for a value to extract from the output of a benchmark execution
author Doug Simon <doug.simon@oracle.com>
date Wed, 30 Jan 2013 11:33:31 +0100
parents 3aab15f42934
children 9a521597686b a8bc60aeacb8
comparison
equal deleted inserted replaced
7563:3aab15f42934 7564:c420a487b10f
21 # or visit www.oracle.com if you need additional information or have any 21 # or visit www.oracle.com if you need additional information or have any
22 # questions. 22 # questions.
23 # 23 #
24 # ---------------------------------------------------------------------------------------------------- 24 # ----------------------------------------------------------------------------------------------------
25 25
26 from outputparser import OutputParser, Matcher 26 from outputparser import OutputParser, ValuesMatcher
27 import re, mx, commands, os, sys, StringIO, subprocess 27 import re, mx, commands, os, sys, StringIO, subprocess
28 from os.path import isfile, join, exists 28 from os.path import isfile, join, exists
29 29
30 dacapoSanityWarmup = { 30 dacapoSanityWarmup = {
31 'avrora': [0, 0, 3, 6, 13], 31 'avrora': [0, 0, 3, 6, 13],
102 mx.abort('Please set the SPECJBB2005 environment variable to a SPECjbb2005 directory') 102 mx.abort('Please set the SPECJBB2005 environment variable to a SPECjbb2005 directory')
103 103
104 score = re.compile(r"^Valid run, Score is (?P<score>[0-9]+)$", re.MULTILINE) 104 score = re.compile(r"^Valid run, Score is (?P<score>[0-9]+)$", re.MULTILINE)
105 error = re.compile(r"VALIDATION ERROR") 105 error = re.compile(r"VALIDATION ERROR")
106 success = re.compile(r"^Valid run, Score is [0-9]+$", re.MULTILINE) 106 success = re.compile(r"^Valid run, Score is [0-9]+$", re.MULTILINE)
107 matcher = Matcher(score, {'const:group' : "const:SPECjbb2005", 'const:name' : 'const:score', 'const:score' : 'score'}) 107 matcher = ValuesMatcher(score, {'group' : 'SPECjbb2005', 'name' : 'score', 'score' : '<score>'})
108 classpath = ['jbb.jar', 'check.jar'] 108 classpath = ['jbb.jar', 'check.jar']
109 return Test("SPECjbb2005", ['spec.jbb.JBBmain', '-propfile', 'SPECjbb.props'] + benchArgs, [success], [error], [matcher], vmOpts=['-Xms3g', '-XX:+UseSerialGC', '-XX:-UseCompressedOops', '-cp', os.pathsep.join(classpath)], defaultCwd=specjbb2005) 109 return Test("SPECjbb2005", ['spec.jbb.JBBmain', '-propfile', 'SPECjbb.props'] + benchArgs, [success], [error], [matcher], vmOpts=['-Xms3g', '-XX:+UseSerialGC', '-XX:-UseCompressedOops', '-cp', os.pathsep.join(classpath)], defaultCwd=specjbb2005)
110 110
111 def getSPECjvm2008(benchArgs = [], skipCheck=False, skipKitValidation=False, warmupTime=None, iterationTime=None): 111 def getSPECjvm2008(benchArgs = [], skipCheck=False, skipKitValidation=False, warmupTime=None, iterationTime=None):
112 112
116 116
117 score = re.compile(r"^(Score on|Noncompliant) (?P<benchmark>[a-zA-Z0-9\._]+)( result)?: (?P<score>[0-9]+((,|\.)[0-9]+)?)( SPECjvm2008 Base)? ops/m$", re.MULTILINE) 117 score = re.compile(r"^(Score on|Noncompliant) (?P<benchmark>[a-zA-Z0-9\._]+)( result)?: (?P<score>[0-9]+((,|\.)[0-9]+)?)( SPECjvm2008 Base)? ops/m$", re.MULTILINE)
118 error = re.compile(r"^Errors in benchmark: ", re.MULTILINE) 118 error = re.compile(r"^Errors in benchmark: ", re.MULTILINE)
119 # The ' ops/m' at the end of the success string is important : it's how you can tell valid and invalid runs apart 119 # The ' ops/m' at the end of the success string is important : it's how you can tell valid and invalid runs apart
120 success = re.compile(r"^(Noncompliant c|C)omposite result: [0-9]+((,|\.)[0-9]+)?( SPECjvm2008 (Base|Peak))? ops/m$", re.MULTILINE) 120 success = re.compile(r"^(Noncompliant c|C)omposite result: [0-9]+((,|\.)[0-9]+)?( SPECjvm2008 (Base|Peak))? ops/m$", re.MULTILINE)
121 matcher = Matcher(score, {'const:group' : "const:SPECjvm2008", 'const:name' : 'benchmark', 'const:score' : 'score'}) 121 matcher = ValuesMatcher(score, {'group' : 'SPECjvm2008', 'name' : '<benchmark>', 'score' : '<score>'})
122 122
123 opts = [] 123 opts = []
124 if warmupTime is not None: 124 if warmupTime is not None:
125 opts += ['-wt', str(warmupTime)] 125 opts += ['-wt', str(warmupTime)]
126 if iterationTime is not None: 126 if iterationTime is not None:
157 dacapoSuccess = re.compile(r"^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====$", re.MULTILINE) 157 dacapoSuccess = re.compile(r"^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====$", re.MULTILINE)
158 dacapoFail = re.compile(r"^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) FAILED (warmup|) =====$", re.MULTILINE) 158 dacapoFail = re.compile(r"^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) FAILED (warmup|) =====$", re.MULTILINE)
159 dacapoTime = re.compile(r"===== DaCapo 9\.12 (?P<benchmark>[a-zA-Z0-9_]+) PASSED in (?P<time>[0-9]+) msec =====") 159 dacapoTime = re.compile(r"===== DaCapo 9\.12 (?P<benchmark>[a-zA-Z0-9_]+) PASSED in (?P<time>[0-9]+) msec =====")
160 dacapoTime1 = re.compile(r"===== DaCapo 9\.12 (?P<benchmark>[a-zA-Z0-9_]+) completed warmup 1 in (?P<time>[0-9]+) msec =====") 160 dacapoTime1 = re.compile(r"===== DaCapo 9\.12 (?P<benchmark>[a-zA-Z0-9_]+) completed warmup 1 in (?P<time>[0-9]+) msec =====")
161 161
162 dacapoMatcher = Matcher(dacapoTime, {'const:group' : "const:DaCapo", 'const:name' : 'benchmark', 'const:score' : 'time'}) 162 dacapoMatcher = ValuesMatcher(dacapoTime, {'group' : 'DaCapo', 'name' : '<benchmark>', 'score' : '<time>'})
163 dacapoMatcher1 = Matcher(dacapoTime1, {'const:group' : "const:DaCapo-1stRun", 'const:name' : 'benchmark', 'const:score' : 'time'}) 163 dacapoMatcher1 = ValuesMatcher(dacapoTime1, {'group' : 'DaCapo-1stRun', 'name' : '<benchmark>', 'score' : '<time>'})
164 164
165 return Test("DaCapo-" + name, ['-jar', dacapo, name, '-n', str(n), ] + dacapoArgs, [dacapoSuccess], [dacapoFail], [dacapoMatcher, dacapoMatcher1], ['-Xms2g', '-XX:+UseSerialGC', '-XX:-UseCompressedOops']) 165 return Test("DaCapo-" + name, ['-jar', dacapo, name, '-n', str(n), ] + dacapoArgs, [dacapoSuccess], [dacapoFail], [dacapoMatcher, dacapoMatcher1], ['-Xms2g', '-XX:+UseSerialGC', '-XX:-UseCompressedOops'])
166 166
167 def getScalaDacapos(level=SanityCheckLevel.Normal, gateBuildLevel=None, dacapoArgs=[]): 167 def getScalaDacapos(level=SanityCheckLevel.Normal, gateBuildLevel=None, dacapoArgs=[]):
168 checks = [] 168 checks = []
188 188
189 dacapoSuccess = re.compile(r"^===== DaCapo 0\.1\.0(-SNAPSHOT)? ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====$", re.MULTILINE) 189 dacapoSuccess = re.compile(r"^===== DaCapo 0\.1\.0(-SNAPSHOT)? ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====$", re.MULTILINE)
190 dacapoFail = re.compile(r"^===== DaCapo 0\.1\.0(-SNAPSHOT)? ([a-zA-Z0-9_]+) FAILED (warmup|) =====$", re.MULTILINE) 190 dacapoFail = re.compile(r"^===== DaCapo 0\.1\.0(-SNAPSHOT)? ([a-zA-Z0-9_]+) FAILED (warmup|) =====$", re.MULTILINE)
191 dacapoTime = re.compile(r"===== DaCapo 0\.1\.0(-SNAPSHOT)? (?P<benchmark>[a-zA-Z0-9_]+) PASSED in (?P<time>[0-9]+) msec =====") 191 dacapoTime = re.compile(r"===== DaCapo 0\.1\.0(-SNAPSHOT)? (?P<benchmark>[a-zA-Z0-9_]+) PASSED in (?P<time>[0-9]+) msec =====")
192 192
193 dacapoMatcher = Matcher(dacapoTime, {'const:group' : "const:Scala-DaCapo", 'const:name' : 'benchmark', 'const:score' : 'time'}) 193 dacapoMatcher = ValuesMatcher(dacapoTime, {'group' : "Scala-DaCapo", 'name' : '<benchmark>', 'score' : '<time>'})
194 194
195 return Test("Scala-DaCapo-" + name, ['-jar', dacapo, name, '-n', str(n), ] + dacapoArgs, [dacapoSuccess], [dacapoFail], [dacapoMatcher], ['-Xms2g', '-XX:+UseSerialGC', '-XX:-UseCompressedOops']) 195 return Test("Scala-DaCapo-" + name, ['-jar', dacapo, name, '-n', str(n), ] + dacapoArgs, [dacapoSuccess], [dacapoFail], [dacapoMatcher], ['-Xms2g', '-XX:+UseSerialGC', '-XX:-UseCompressedOops'])
196 196
197 def getBootstraps(): 197 def getBootstraps():
198 time = re.compile(r"Bootstrapping Graal\.+ in (?P<time>[0-9]+) ms") 198 time = re.compile(r"Bootstrapping Graal\.+ in (?P<time>[0-9]+) ms")
199 scoreMatcher = Matcher(time, {'const:group' : 'const:Bootstrap', 'const:name' : 'const:BootstrapTime', 'const:score' : 'time'}) 199 scoreMatcher = ValuesMatcher(time, {'group' : 'Bootstrap', 'name' : 'BootstrapTime', 'score' : '<time>'})
200 scoreMatcherBig = Matcher(time, {'const:group' : 'const:Bootstrap-bigHeap', 'const:name' : 'const:BootstrapTime', 'const:score' : 'time'}) 200 scoreMatcherBig = ValuesMatcher(time, {'group' : 'Bootstrap-bigHeap', 'name' : 'BootstrapTime', 'score' : '<time>'})
201 201
202 tests = [] 202 tests = []
203 tests.append(Test("Bootstrap", ['-version'], successREs=[time], scoreMatchers=[scoreMatcher], ignoredVMs=['client', 'server'], benchmarkCompilationRate=False)) 203 tests.append(Test("Bootstrap", ['-version'], successREs=[time], scoreMatchers=[scoreMatcher], ignoredVMs=['client', 'server'], benchmarkCompilationRate=False))
204 tests.append(Test("Bootstrap-bigHeap", ['-version'], successREs=[time], scoreMatchers=[scoreMatcherBig], vmOpts=['-Xms2g'], ignoredVMs=['client', 'server'], benchmarkCompilationRate=False)) 204 tests.append(Test("Bootstrap-bigHeap", ['-version'], successREs=[time], scoreMatchers=[scoreMatcherBig], vmOpts=['-Xms2g'], ignoredVMs=['client', 'server'], benchmarkCompilationRate=False))
205 return tests 205 return tests
240 return True 240 return True
241 if cwd is None: 241 if cwd is None:
242 cwd = self.defaultCwd 242 cwd = self.defaultCwd
243 parser = OutputParser() 243 parser = OutputParser()
244 jvmError = re.compile(r"(?P<jvmerror>([A-Z]:|/).*[/\\]hs_err_pid[0-9]+\.log)") 244 jvmError = re.compile(r"(?P<jvmerror>([A-Z]:|/).*[/\\]hs_err_pid[0-9]+\.log)")
245 parser.addMatcher(Matcher(jvmError, {'const:jvmError' : 'jvmerror'})) 245 parser.addMatcher(ValuesMatcher(jvmError, {'jvmError' : '<jvmerror>'}))
246 246
247 for successRE in self.successREs: 247 for successRE in self.successREs:
248 parser.addMatcher(Matcher(successRE, {'const:passed' : 'const:1'})) 248 parser.addMatcher(ValuesMatcher(successRE, {'passed' : '1'}))
249 for failureRE in self.failureREs: 249 for failureRE in self.failureREs:
250 parser.addMatcher(Matcher(failureRE, {'const:failed' : 'const:1'})) 250 parser.addMatcher(ValuesMatcher(failureRE, {'failed' : '1'}))
251 251
252 tee = Tee() 252 tee = Tee()
253 retcode = commands.vm(self.vmOpts + opts + self.cmd, vm, nonZeroIsFatal=False, out=tee.eat, err=subprocess.STDOUT, cwd=cwd, vmbuild=vmbuild) 253 retcode = commands.vm(self.vmOpts + opts + self.cmd, vm, nonZeroIsFatal=False, out=tee.eat, err=subprocess.STDOUT, cwd=cwd, vmbuild=vmbuild)
254 output = tee.output.getvalue() 254 output = tee.output.getvalue()
255 valueMaps = parser.parse(output) 255 valueMaps = parser.parse(output)
283 if cwd is None: 283 if cwd is None:
284 cwd = self.defaultCwd 284 cwd = self.defaultCwd
285 parser = OutputParser() 285 parser = OutputParser()
286 286
287 for successRE in self.successREs: 287 for successRE in self.successREs:
288 parser.addMatcher(Matcher(successRE, {'const:passed' : 'const:1'})) 288 parser.addMatcher(ValuesMatcher(successRE, {'passed' : '1'}))
289 for failureRE in self.failureREs: 289 for failureRE in self.failureREs:
290 parser.addMatcher(Matcher(failureRE, {'const:failed' : 'const:1'})) 290 parser.addMatcher(ValuesMatcher(failureRE, {'failed' : '1'}))
291 for scoreMatcher in self.scoreMatchers: 291 for scoreMatcher in self.scoreMatchers:
292 parser.addMatcher(scoreMatcher) 292 parser.addMatcher(scoreMatcher)
293 293
294 if self.benchmarkCompilationRate: 294 if self.benchmarkCompilationRate:
295 opts.append('-Dgraal.benchmark.compilation=true') 295 opts.append('-Dgraal.benchmark.compilation=true')
296 bps = re.compile(r"ParsedBytecodesPerSecond@final: (?P<rate>[0-9]+)") 296 bps = re.compile(r"ParsedBytecodesPerSecond@final: (?P<rate>[0-9]+)")
297 ibps = re.compile(r"InlinedBytecodesPerSecond@final: (?P<rate>[0-9]+)") 297 ibps = re.compile(r"InlinedBytecodesPerSecond@final: (?P<rate>[0-9]+)")
298 parser.addMatcher(Matcher(bps, {'const:group' : 'const:ParsedBytecodesPerSecond', 'const:name' : 'const:' + self.name, 'const:score' : 'rate'})) 298 parser.addMatcher(ValuesMatcher(bps, {'group' : 'ParsedBytecodesPerSecond', 'name' : self.name, 'score' : '<rate>'}))
299 parser.addMatcher(Matcher(ibps, {'const:group' : 'const:InlinedBytecodesPerSecond', 'const:name' : 'const:' + self.name, 'const:score' : 'rate'})) 299 parser.addMatcher(ValuesMatcher(ibps, {'group' : 'InlinedBytecodesPerSecond', 'name' : self.name, 'score' : '<rate>'}))
300 300
301 outputfile = self.name + '.output' 301 outputfile = self.name + '.output'
302 if _debugBenchParser and exists(outputfile): 302 if _debugBenchParser and exists(outputfile):
303 with open(outputfile) as fp: 303 with open(outputfile) as fp:
304 output = fp.read() 304 output = fp.read()