Mercurial > hg > truffle

comparison mx/sanitycheck.py @ 5195:8125c1f6472b

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Add DaCapo 1st run as a benchmark group
author Gilles Duboscq <duboscq@ssw.jku.at>
date Thu, 05 Apr 2012 17:11:41 +0200
parents eeb2ca1bf3a9
children beb13d0326ba
comparison
equal deleted inserted replaced
5192:20993edddd73 5195:8125c1f6472b
104 104
105 score = re.compile(r"^(Score on|Noncompliant) (?P<benchmark>[a-zA-Z0-9\._]+)( result)?: (?P<score>[0-9]+(,|\.)[0-9]+)( SPECjvm2008 Base)? ops/m$") 105 score = re.compile(r"^(Score on|Noncompliant) (?P<benchmark>[a-zA-Z0-9\._]+)( result)?: (?P<score>[0-9]+(,|\.)[0-9]+)( SPECjvm2008 Base)? ops/m$")
106 error = re.compile(r"^Errors in benchmark: ") 106 error = re.compile(r"^Errors in benchmark: ")
107 # The ' ops/m' at the end of the success string is important : it's how you can tell valid and invalid runs apart 107 # The ' ops/m' at the end of the success string is important : it's how you can tell valid and invalid runs apart
108 success = re.compile(r"^(Noncompliant c|C)omposite result: [0-9]+(,|\.)[0-9]+( SPECjvm2008 (Base|Peak))? ops/m$") 108 success = re.compile(r"^(Noncompliant c|C)omposite result: [0-9]+(,|\.)[0-9]+( SPECjvm2008 (Base|Peak))? ops/m$")
109 matcher = Matcher(score, {'const:name' : 'benchmark', 'const:score' : 'score'}, startNewLine=True) 109 matcher = Matcher(score, {'const:group' : "const:SPECjvm2008", 'const:name' : 'benchmark', 'const:score' : 'score'}, startNewLine=True)
110 110
111 opts = [] 111 opts = []
112 if warmupTime is not None: 112 if warmupTime is not None:
113 opts += ['-wt', str(warmupTime)] 113 opts += ['-wt', str(warmupTime)]
114 if iterationTime is not None: 114 if iterationTime is not None:
115 opts += ['-it', str(iterationTime)] 115 opts += ['-it', str(iterationTime)]
116 if skipKitValidation: 116 if skipKitValidation:
117 opts += ['-ikv'] 117 opts += ['-ikv']
118 118
119 return Test("SPECjvm2008", "SPECjvm2008", ['-jar', 'SPECjvm2008.jar'] + opts + benchArgs, [success], [error], [matcher], vmOpts=['-Xms3g', '-XX:+UseSerialGC'], defaultCwd=specjvm2008) 119 return Test("SPECjvm2008", ['-jar', 'SPECjvm2008.jar'] + opts + benchArgs, [success], [error], [matcher], vmOpts=['-Xms3g', '-XX:+UseSerialGC'], defaultCwd=specjvm2008)
120 120
121 def getDacapos(level=SanityCheckLevel.Normal, gateBuildLevel=None, dacapoArgs=[]): 121 def getDacapos(level=SanityCheckLevel.Normal, gateBuildLevel=None, dacapoArgs=[]):
122 checks = [] 122 checks = []
123 123
124 for (bench, ns) in dacapoSanityWarmup.items(): 124 for (bench, ns) in dacapoSanityWarmup.items():
141 mx.abort('Specified DaCapo jar file does not exist or is not a jar file: ' + dacapo) 141 mx.abort('Specified DaCapo jar file does not exist or is not a jar file: ' + dacapo)
142 142
143 dacapoSuccess = re.compile(r"^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====$") 143 dacapoSuccess = re.compile(r"^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====$")
144 dacapoFail = re.compile(r"^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) FAILED (warmup|) =====$") 144 dacapoFail = re.compile(r"^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) FAILED (warmup|) =====$")
145 dacapoTime = re.compile(r"===== DaCapo 9\.12 (?P<benchmark>[a-zA-Z0-9_]+) PASSED in (?P<time>[0-9]+) msec =====") 145 dacapoTime = re.compile(r"===== DaCapo 9\.12 (?P<benchmark>[a-zA-Z0-9_]+) PASSED in (?P<time>[0-9]+) msec =====")
146 146 dacapoTime1 = re.compile(r"===== DaCapo 9\.12 (?P<benchmark>[a-zA-Z0-9_]+) completed warmup 1 in (?P<time>[0-9]+) msec =====")
147 dacapoMatcher = Matcher(dacapoTime, {'const:name' : 'benchmark', 'const:score' : 'time'}) 147
148 148 dacapoMatcher = Matcher(dacapoTime, {'const:group' : "const:DaCapo", 'const:name' : 'benchmark', 'const:score' : 'time'})
149 return Test("DaCapo-" + name, "DaCapo", ['-jar', dacapo, name, '-n', str(n), ] + dacapoArgs, [dacapoSuccess], [dacapoFail], [dacapoMatcher], ['-Xms2g', '-XX:MaxPermSize=256m', '-XX:+UseSerialGC']) 149 dacapoMatcher1 = Matcher(dacapoTime1, {'const:group' : "const:DaCapo-1stRun", 'const:name' : 'benchmark', 'const:score' : 'time'})
150
151 return Test("DaCapo-" + name, ['-jar', dacapo, name, '-n', str(n), ] + dacapoArgs, [dacapoSuccess], [dacapoFail], [dacapoMatcher, dacapoMatcher1], ['-Xms2g', '-XX:MaxPermSize=256m', '-XX:+UseSerialGC'])
150 152
151 def getScalaDacapos(level=SanityCheckLevel.Normal, gateBuildLevel=None, dacapoArgs=[]): 153 def getScalaDacapos(level=SanityCheckLevel.Normal, gateBuildLevel=None, dacapoArgs=[]):
152 checks = [] 154 checks = []
153 155
154 for (bench, ns) in dacapoScalaSanityWarmup.items(): 156 for (bench, ns) in dacapoScalaSanityWarmup.items():
172 174
173 dacapoSuccess = re.compile(r"^===== DaCapo 0\.1\.0(-SNAPSHOT)? ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====$") 175 dacapoSuccess = re.compile(r"^===== DaCapo 0\.1\.0(-SNAPSHOT)? ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====$")
174 dacapoFail = re.compile(r"^===== DaCapo 0\.1\.0(-SNAPSHOT)? ([a-zA-Z0-9_]+) FAILED (warmup|) =====$") 176 dacapoFail = re.compile(r"^===== DaCapo 0\.1\.0(-SNAPSHOT)? ([a-zA-Z0-9_]+) FAILED (warmup|) =====$")
175 dacapoTime = re.compile(r"===== DaCapo 0\.1\.0(-SNAPSHOT)? (?P<benchmark>[a-zA-Z0-9_]+) PASSED in (?P<time>[0-9]+) msec =====") 177 dacapoTime = re.compile(r"===== DaCapo 0\.1\.0(-SNAPSHOT)? (?P<benchmark>[a-zA-Z0-9_]+) PASSED in (?P<time>[0-9]+) msec =====")
176 178
177 dacapoMatcher = Matcher(dacapoTime, {'const:name' : 'benchmark', 'const:score' : 'time'}) 179 dacapoMatcher = Matcher(dacapoTime, {'const:group' : "const:Scala-DaCapo", 'const:name' : 'benchmark', 'const:score' : 'time'})
178 180
179 return Test("Scala-DaCapo-" + name, "Scala-DaCapo", ['-jar', dacapo, name, '-n', str(n), ] + dacapoArgs, [dacapoSuccess], [dacapoFail], [dacapoMatcher], ['-Xms2g', '-XX:MaxPermSize=256m', '-XX:+UseSerialGC']) 181 return Test("Scala-DaCapo-" + name, ['-jar', dacapo, name, '-n', str(n), ] + dacapoArgs, [dacapoSuccess], [dacapoFail], [dacapoMatcher], ['-Xms2g', '-XX:MaxPermSize=256m', '-XX:+UseSerialGC'])
180 182
181 def getBootstraps(): 183 def getBootstraps():
182 time = re.compile(r"Bootstrapping Graal\.+ in (?P<time>[0-9]+) ms") 184 time = re.compile(r"Bootstrapping Graal\.+ in (?P<time>[0-9]+) ms")
183 scoreMatcher = Matcher(time, {'const:name' : 'const:BootstrapTime', 'const:score' : 'time'}) 185 scoreMatcher = Matcher(time, {'const:group' : 'const:Bootstrap', 'const:name' : 'const:BootstrapTime', 'const:score' : 'time'})
186 scoreMatcherBig = Matcher(time, {'const:group' : 'const:Bootstrap-bigHeap', 'const:name' : 'const:BootstrapTime', 'const:score' : 'time'})
187
184 tests = [] 188 tests = []
185 tests.append(Test("Bootstrap", "Bootstrap", ['-version'], successREs=[time], scoreMatchers=[scoreMatcher], ingoreVms=['client', 'server'])) 189 tests.append(Test("Bootstrap", ['-version'], successREs=[time], scoreMatchers=[scoreMatcher], ingoreVms=['client', 'server']))
186 tests.append(Test("Bootstrap-bigHeap", "Bootstrap-bigHeap", ['-version'], successREs=[time], scoreMatchers=[scoreMatcher], vmOpts=['-Xms2g'], ingoreVms=['client', 'server'])) 190 tests.append(Test("Bootstrap-bigHeap", ['-version'], successREs=[time], scoreMatchers=[scoreMatcherBig], vmOpts=['-Xms2g'], ingoreVms=['client', 'server']))
187 return tests 191 return tests
188 192
189 """ 193 """
190 Encapsulates a single program that is a sanity test and/or a benchmark. 194 Encapsulates a single program that is a sanity test and/or a benchmark.
191 """ 195 """
192 class Test: 196 class Test:
193 def __init__(self, name, group, cmd, successREs=[], failureREs=[], scoreMatchers=[], vmOpts=[], defaultCwd=None, ingoreVms=[]): 197 def __init__(self, name, cmd, successREs=[], failureREs=[], scoreMatchers=[], vmOpts=[], defaultCwd=None, ingoreVms=[]):
194 self.name = name 198 self.name = name
195 self.group = group
196 self.successREs = successREs 199 self.successREs = successREs
197 self.failureREs = failureREs + [re.compile(r"Exception occured in scope: ")] 200 self.failureREs = failureREs + [re.compile(r"Exception occured in scope: ")]
198 self.scoreMatchers = scoreMatchers 201 self.scoreMatchers = scoreMatchers
199 self.vmOpts = vmOpts 202 self.vmOpts = vmOpts
200 self.cmd = cmd 203 self.cmd = cmd
272 ret = {} 275 ret = {}
273 276
274 passed = False; 277 passed = False;
275 278
276 for line in parsed: 279 for line in parsed:
277 assert (line.has_key('name') and line.has_key('score')) or line.has_key('passed') 280 assert (line.has_key('name') and line.has_key('score') and line.has_key('group')) or line.has_key('passed') or line.has_key('failed')
278 if line.has_key('failed') and line['failed'] is '1': 281 if line.has_key('failed') and line['failed'] is '1':
279 mx.abort("Benchmark failed") 282 mx.abort("Benchmark failed")
280 if line.has_key('passed') and line['passed'] is '1': 283 if line.has_key('passed') and line['passed'] is '1':
281 passed = True 284 passed = True
282 ret[line['name']] = line['score'] 285 if line.has_key('name') and line.has_key('score') and line.has_key('group'):
286 if not ret.has_key(line['group']):
287 ret[line['group']] = {};
288 ret[line['group']][line['name']] = line['score']
283 289
284 if not passed: 290 if not passed:
285 mx.abort("Benchmark failed (not passed)") 291 mx.abort("Benchmark failed (not passed)")
286 292
287 return ret 293 return ret