Mercurial > hg > truffle
comparison mx/sanitycheck.py @ 7564:c420a487b10f
changed convention for specifying constants versus named groups in a template for a value to extract from the output of a benchmark execution
author | Doug Simon <doug.simon@oracle.com> |
---|---|
date | Wed, 30 Jan 2013 11:33:31 +0100 |
parents | 3aab15f42934 |
children | 9a521597686b a8bc60aeacb8 |
comparison
equal
deleted
inserted
replaced
7563:3aab15f42934 | 7564:c420a487b10f |
---|---|
21 # or visit www.oracle.com if you need additional information or have any | 21 # or visit www.oracle.com if you need additional information or have any |
22 # questions. | 22 # questions. |
23 # | 23 # |
24 # ---------------------------------------------------------------------------------------------------- | 24 # ---------------------------------------------------------------------------------------------------- |
25 | 25 |
26 from outputparser import OutputParser, Matcher | 26 from outputparser import OutputParser, ValuesMatcher |
27 import re, mx, commands, os, sys, StringIO, subprocess | 27 import re, mx, commands, os, sys, StringIO, subprocess |
28 from os.path import isfile, join, exists | 28 from os.path import isfile, join, exists |
29 | 29 |
30 dacapoSanityWarmup = { | 30 dacapoSanityWarmup = { |
31 'avrora': [0, 0, 3, 6, 13], | 31 'avrora': [0, 0, 3, 6, 13], |
102 mx.abort('Please set the SPECJBB2005 environment variable to a SPECjbb2005 directory') | 102 mx.abort('Please set the SPECJBB2005 environment variable to a SPECjbb2005 directory') |
103 | 103 |
104 score = re.compile(r"^Valid run, Score is (?P<score>[0-9]+)$", re.MULTILINE) | 104 score = re.compile(r"^Valid run, Score is (?P<score>[0-9]+)$", re.MULTILINE) |
105 error = re.compile(r"VALIDATION ERROR") | 105 error = re.compile(r"VALIDATION ERROR") |
106 success = re.compile(r"^Valid run, Score is [0-9]+$", re.MULTILINE) | 106 success = re.compile(r"^Valid run, Score is [0-9]+$", re.MULTILINE) |
107 matcher = Matcher(score, {'const:group' : "const:SPECjbb2005", 'const:name' : 'const:score', 'const:score' : 'score'}) | 107 matcher = ValuesMatcher(score, {'group' : 'SPECjbb2005', 'name' : 'score', 'score' : '<score>'}) |
108 classpath = ['jbb.jar', 'check.jar'] | 108 classpath = ['jbb.jar', 'check.jar'] |
109 return Test("SPECjbb2005", ['spec.jbb.JBBmain', '-propfile', 'SPECjbb.props'] + benchArgs, [success], [error], [matcher], vmOpts=['-Xms3g', '-XX:+UseSerialGC', '-XX:-UseCompressedOops', '-cp', os.pathsep.join(classpath)], defaultCwd=specjbb2005) | 109 return Test("SPECjbb2005", ['spec.jbb.JBBmain', '-propfile', 'SPECjbb.props'] + benchArgs, [success], [error], [matcher], vmOpts=['-Xms3g', '-XX:+UseSerialGC', '-XX:-UseCompressedOops', '-cp', os.pathsep.join(classpath)], defaultCwd=specjbb2005) |
110 | 110 |
111 def getSPECjvm2008(benchArgs = [], skipCheck=False, skipKitValidation=False, warmupTime=None, iterationTime=None): | 111 def getSPECjvm2008(benchArgs = [], skipCheck=False, skipKitValidation=False, warmupTime=None, iterationTime=None): |
112 | 112 |
116 | 116 |
117 score = re.compile(r"^(Score on|Noncompliant) (?P<benchmark>[a-zA-Z0-9\._]+)( result)?: (?P<score>[0-9]+((,|\.)[0-9]+)?)( SPECjvm2008 Base)? ops/m$", re.MULTILINE) | 117 score = re.compile(r"^(Score on|Noncompliant) (?P<benchmark>[a-zA-Z0-9\._]+)( result)?: (?P<score>[0-9]+((,|\.)[0-9]+)?)( SPECjvm2008 Base)? ops/m$", re.MULTILINE) |
118 error = re.compile(r"^Errors in benchmark: ", re.MULTILINE) | 118 error = re.compile(r"^Errors in benchmark: ", re.MULTILINE) |
119 # The ' ops/m' at the end of the success string is important : it's how you can tell valid and invalid runs apart | 119 # The ' ops/m' at the end of the success string is important : it's how you can tell valid and invalid runs apart |
120 success = re.compile(r"^(Noncompliant c|C)omposite result: [0-9]+((,|\.)[0-9]+)?( SPECjvm2008 (Base|Peak))? ops/m$", re.MULTILINE) | 120 success = re.compile(r"^(Noncompliant c|C)omposite result: [0-9]+((,|\.)[0-9]+)?( SPECjvm2008 (Base|Peak))? ops/m$", re.MULTILINE) |
121 matcher = Matcher(score, {'const:group' : "const:SPECjvm2008", 'const:name' : 'benchmark', 'const:score' : 'score'}) | 121 matcher = ValuesMatcher(score, {'group' : 'SPECjvm2008', 'name' : '<benchmark>', 'score' : '<score>'}) |
122 | 122 |
123 opts = [] | 123 opts = [] |
124 if warmupTime is not None: | 124 if warmupTime is not None: |
125 opts += ['-wt', str(warmupTime)] | 125 opts += ['-wt', str(warmupTime)] |
126 if iterationTime is not None: | 126 if iterationTime is not None: |
157 dacapoSuccess = re.compile(r"^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====$", re.MULTILINE) | 157 dacapoSuccess = re.compile(r"^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====$", re.MULTILINE) |
158 dacapoFail = re.compile(r"^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) FAILED (warmup|) =====$", re.MULTILINE) | 158 dacapoFail = re.compile(r"^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) FAILED (warmup|) =====$", re.MULTILINE) |
159 dacapoTime = re.compile(r"===== DaCapo 9\.12 (?P<benchmark>[a-zA-Z0-9_]+) PASSED in (?P<time>[0-9]+) msec =====") | 159 dacapoTime = re.compile(r"===== DaCapo 9\.12 (?P<benchmark>[a-zA-Z0-9_]+) PASSED in (?P<time>[0-9]+) msec =====") |
160 dacapoTime1 = re.compile(r"===== DaCapo 9\.12 (?P<benchmark>[a-zA-Z0-9_]+) completed warmup 1 in (?P<time>[0-9]+) msec =====") | 160 dacapoTime1 = re.compile(r"===== DaCapo 9\.12 (?P<benchmark>[a-zA-Z0-9_]+) completed warmup 1 in (?P<time>[0-9]+) msec =====") |
161 | 161 |
162 dacapoMatcher = Matcher(dacapoTime, {'const:group' : "const:DaCapo", 'const:name' : 'benchmark', 'const:score' : 'time'}) | 162 dacapoMatcher = ValuesMatcher(dacapoTime, {'group' : 'DaCapo', 'name' : '<benchmark>', 'score' : '<time>'}) |
163 dacapoMatcher1 = Matcher(dacapoTime1, {'const:group' : "const:DaCapo-1stRun", 'const:name' : 'benchmark', 'const:score' : 'time'}) | 163 dacapoMatcher1 = ValuesMatcher(dacapoTime1, {'group' : 'DaCapo-1stRun', 'name' : '<benchmark>', 'score' : '<time>'}) |
164 | 164 |
165 return Test("DaCapo-" + name, ['-jar', dacapo, name, '-n', str(n), ] + dacapoArgs, [dacapoSuccess], [dacapoFail], [dacapoMatcher, dacapoMatcher1], ['-Xms2g', '-XX:+UseSerialGC', '-XX:-UseCompressedOops']) | 165 return Test("DaCapo-" + name, ['-jar', dacapo, name, '-n', str(n), ] + dacapoArgs, [dacapoSuccess], [dacapoFail], [dacapoMatcher, dacapoMatcher1], ['-Xms2g', '-XX:+UseSerialGC', '-XX:-UseCompressedOops']) |
166 | 166 |
167 def getScalaDacapos(level=SanityCheckLevel.Normal, gateBuildLevel=None, dacapoArgs=[]): | 167 def getScalaDacapos(level=SanityCheckLevel.Normal, gateBuildLevel=None, dacapoArgs=[]): |
168 checks = [] | 168 checks = [] |
188 | 188 |
189 dacapoSuccess = re.compile(r"^===== DaCapo 0\.1\.0(-SNAPSHOT)? ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====$", re.MULTILINE) | 189 dacapoSuccess = re.compile(r"^===== DaCapo 0\.1\.0(-SNAPSHOT)? ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====$", re.MULTILINE) |
190 dacapoFail = re.compile(r"^===== DaCapo 0\.1\.0(-SNAPSHOT)? ([a-zA-Z0-9_]+) FAILED (warmup|) =====$", re.MULTILINE) | 190 dacapoFail = re.compile(r"^===== DaCapo 0\.1\.0(-SNAPSHOT)? ([a-zA-Z0-9_]+) FAILED (warmup|) =====$", re.MULTILINE) |
191 dacapoTime = re.compile(r"===== DaCapo 0\.1\.0(-SNAPSHOT)? (?P<benchmark>[a-zA-Z0-9_]+) PASSED in (?P<time>[0-9]+) msec =====") | 191 dacapoTime = re.compile(r"===== DaCapo 0\.1\.0(-SNAPSHOT)? (?P<benchmark>[a-zA-Z0-9_]+) PASSED in (?P<time>[0-9]+) msec =====") |
192 | 192 |
193 dacapoMatcher = Matcher(dacapoTime, {'const:group' : "const:Scala-DaCapo", 'const:name' : 'benchmark', 'const:score' : 'time'}) | 193 dacapoMatcher = ValuesMatcher(dacapoTime, {'group' : "Scala-DaCapo", 'name' : '<benchmark>', 'score' : '<time>'}) |
194 | 194 |
195 return Test("Scala-DaCapo-" + name, ['-jar', dacapo, name, '-n', str(n), ] + dacapoArgs, [dacapoSuccess], [dacapoFail], [dacapoMatcher], ['-Xms2g', '-XX:+UseSerialGC', '-XX:-UseCompressedOops']) | 195 return Test("Scala-DaCapo-" + name, ['-jar', dacapo, name, '-n', str(n), ] + dacapoArgs, [dacapoSuccess], [dacapoFail], [dacapoMatcher], ['-Xms2g', '-XX:+UseSerialGC', '-XX:-UseCompressedOops']) |
196 | 196 |
197 def getBootstraps(): | 197 def getBootstraps(): |
198 time = re.compile(r"Bootstrapping Graal\.+ in (?P<time>[0-9]+) ms") | 198 time = re.compile(r"Bootstrapping Graal\.+ in (?P<time>[0-9]+) ms") |
199 scoreMatcher = Matcher(time, {'const:group' : 'const:Bootstrap', 'const:name' : 'const:BootstrapTime', 'const:score' : 'time'}) | 199 scoreMatcher = ValuesMatcher(time, {'group' : 'Bootstrap', 'name' : 'BootstrapTime', 'score' : '<time>'}) |
200 scoreMatcherBig = Matcher(time, {'const:group' : 'const:Bootstrap-bigHeap', 'const:name' : 'const:BootstrapTime', 'const:score' : 'time'}) | 200 scoreMatcherBig = ValuesMatcher(time, {'group' : 'Bootstrap-bigHeap', 'name' : 'BootstrapTime', 'score' : '<time>'}) |
201 | 201 |
202 tests = [] | 202 tests = [] |
203 tests.append(Test("Bootstrap", ['-version'], successREs=[time], scoreMatchers=[scoreMatcher], ignoredVMs=['client', 'server'], benchmarkCompilationRate=False)) | 203 tests.append(Test("Bootstrap", ['-version'], successREs=[time], scoreMatchers=[scoreMatcher], ignoredVMs=['client', 'server'], benchmarkCompilationRate=False)) |
204 tests.append(Test("Bootstrap-bigHeap", ['-version'], successREs=[time], scoreMatchers=[scoreMatcherBig], vmOpts=['-Xms2g'], ignoredVMs=['client', 'server'], benchmarkCompilationRate=False)) | 204 tests.append(Test("Bootstrap-bigHeap", ['-version'], successREs=[time], scoreMatchers=[scoreMatcherBig], vmOpts=['-Xms2g'], ignoredVMs=['client', 'server'], benchmarkCompilationRate=False)) |
205 return tests | 205 return tests |
240 return True | 240 return True |
241 if cwd is None: | 241 if cwd is None: |
242 cwd = self.defaultCwd | 242 cwd = self.defaultCwd |
243 parser = OutputParser() | 243 parser = OutputParser() |
244 jvmError = re.compile(r"(?P<jvmerror>([A-Z]:|/).*[/\\]hs_err_pid[0-9]+\.log)") | 244 jvmError = re.compile(r"(?P<jvmerror>([A-Z]:|/).*[/\\]hs_err_pid[0-9]+\.log)") |
245 parser.addMatcher(Matcher(jvmError, {'const:jvmError' : 'jvmerror'})) | 245 parser.addMatcher(ValuesMatcher(jvmError, {'jvmError' : '<jvmerror>'})) |
246 | 246 |
247 for successRE in self.successREs: | 247 for successRE in self.successREs: |
248 parser.addMatcher(Matcher(successRE, {'const:passed' : 'const:1'})) | 248 parser.addMatcher(ValuesMatcher(successRE, {'passed' : '1'})) |
249 for failureRE in self.failureREs: | 249 for failureRE in self.failureREs: |
250 parser.addMatcher(Matcher(failureRE, {'const:failed' : 'const:1'})) | 250 parser.addMatcher(ValuesMatcher(failureRE, {'failed' : '1'})) |
251 | 251 |
252 tee = Tee() | 252 tee = Tee() |
253 retcode = commands.vm(self.vmOpts + opts + self.cmd, vm, nonZeroIsFatal=False, out=tee.eat, err=subprocess.STDOUT, cwd=cwd, vmbuild=vmbuild) | 253 retcode = commands.vm(self.vmOpts + opts + self.cmd, vm, nonZeroIsFatal=False, out=tee.eat, err=subprocess.STDOUT, cwd=cwd, vmbuild=vmbuild) |
254 output = tee.output.getvalue() | 254 output = tee.output.getvalue() |
255 valueMaps = parser.parse(output) | 255 valueMaps = parser.parse(output) |
283 if cwd is None: | 283 if cwd is None: |
284 cwd = self.defaultCwd | 284 cwd = self.defaultCwd |
285 parser = OutputParser() | 285 parser = OutputParser() |
286 | 286 |
287 for successRE in self.successREs: | 287 for successRE in self.successREs: |
288 parser.addMatcher(Matcher(successRE, {'const:passed' : 'const:1'})) | 288 parser.addMatcher(ValuesMatcher(successRE, {'passed' : '1'})) |
289 for failureRE in self.failureREs: | 289 for failureRE in self.failureREs: |
290 parser.addMatcher(Matcher(failureRE, {'const:failed' : 'const:1'})) | 290 parser.addMatcher(ValuesMatcher(failureRE, {'failed' : '1'})) |
291 for scoreMatcher in self.scoreMatchers: | 291 for scoreMatcher in self.scoreMatchers: |
292 parser.addMatcher(scoreMatcher) | 292 parser.addMatcher(scoreMatcher) |
293 | 293 |
294 if self.benchmarkCompilationRate: | 294 if self.benchmarkCompilationRate: |
295 opts.append('-Dgraal.benchmark.compilation=true') | 295 opts.append('-Dgraal.benchmark.compilation=true') |
296 bps = re.compile(r"ParsedBytecodesPerSecond@final: (?P<rate>[0-9]+)") | 296 bps = re.compile(r"ParsedBytecodesPerSecond@final: (?P<rate>[0-9]+)") |
297 ibps = re.compile(r"InlinedBytecodesPerSecond@final: (?P<rate>[0-9]+)") | 297 ibps = re.compile(r"InlinedBytecodesPerSecond@final: (?P<rate>[0-9]+)") |
298 parser.addMatcher(Matcher(bps, {'const:group' : 'const:ParsedBytecodesPerSecond', 'const:name' : 'const:' + self.name, 'const:score' : 'rate'})) | 298 parser.addMatcher(ValuesMatcher(bps, {'group' : 'ParsedBytecodesPerSecond', 'name' : self.name, 'score' : '<rate>'})) |
299 parser.addMatcher(Matcher(ibps, {'const:group' : 'const:InlinedBytecodesPerSecond', 'const:name' : 'const:' + self.name, 'const:score' : 'rate'})) | 299 parser.addMatcher(ValuesMatcher(ibps, {'group' : 'InlinedBytecodesPerSecond', 'name' : self.name, 'score' : '<rate>'})) |
300 | 300 |
301 outputfile = self.name + '.output' | 301 outputfile = self.name + '.output' |
302 if _debugBenchParser and exists(outputfile): | 302 if _debugBenchParser and exists(outputfile): |
303 with open(outputfile) as fp: | 303 with open(outputfile) as fp: |
304 output = fp.read() | 304 output = fp.read() |