Mercurial > hg > truffle

comparison mx/mx_truffle.py @ 21957:04fd8d2361df

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Additional simplification of the mx_truffle.py
author Jaroslav Tulach <jaroslav.tulach@oracle.com>
date Wed, 17 Jun 2015 11:43:13 +0200
parents 4e3e8d522120
children ef8c90391f1e
comparison
equal deleted inserted replaced
21956:4e3e8d522120 21957:04fd8d2361df
30 from os.path import join, exists, dirname, basename 30 from os.path import join, exists, dirname, basename
31 from argparse import ArgumentParser, RawDescriptionHelpFormatter, REMAINDER 31 from argparse import ArgumentParser, RawDescriptionHelpFormatter, REMAINDER
32 from outputparser import OutputParser, ValuesMatcher 32 from outputparser import OutputParser, ValuesMatcher
33 import mx 33 import mx
34 import xml.dom.minidom 34 import xml.dom.minidom
35 import sanitycheck
36 import itertools 35 import itertools
37 import json, textwrap 36 import json, textwrap
38 import fnmatch 37 import fnmatch
39 import mx_graal_makefile
40 38
41 # This works because when mx loads this file, it makes sure __file__ gets an absolute path 39 # This works because when mx loads this file, it makes sure __file__ gets an absolute path
42 _graal_home = dirname(dirname(__file__)) 40 _graal_home = dirname(dirname(__file__))
43 41
44 """ Used to distinguish an exported GraalVM (see 'mx export'). """ 42 """ Used to distinguish an exported GraalVM (see 'mx export'). """
45 _vmSourcesAvailable = True 43 _vmSourcesAvailable = True
46
47 """ The VMs that can be built and run along with an optional description. Only VMs with a
48 description are listed in the dialogue for setting the default VM (see _get_vm()). """
49 _vmChoices = {
50 'jvmci' : 'Normal compilation is performed with a tiered system (C1 + Graal), Truffle compilation is performed with Graal.',
51 'server' : 'Normal compilation is performed with a tiered system (C1 + C2), Truffle compilation is performed with Graal. Use this for optimal Truffle performance.',
52 'client' : None, # normal compilation with client compiler, explicit compilation (e.g., by Truffle) with Graal
53 'server-nojvmci' : None, # all compilation with tiered system (i.e., client + server), JVMCI omitted
54 'client-nojvmci' : None, # all compilation with client compiler, JVMCI omitted
55 'original' : None, # default VM copied from bootstrap JDK
56 'graal' : None, # alias for jvmci
57 'server-nograal' : None, # alias for server-nojvmci
58 'client-nograal' : None, # alias for client-nojvmci
59 }
60 44
61 """ The VM that will be run by the 'vm' command and built by default by the 'build' command. 45 """ The VM that will be run by the 'vm' command and built by default by the 'build' command.
62 This can be set via the global '--vm' option or the DEFAULT_VM environment variable. 46 This can be set via the global '--vm' option or the DEFAULT_VM environment variable.
63 It can also be temporarily set by using of a VM context manager object in a 'with' statement. """ 47 It can also be temporarily set by using of a VM context manager object in a 'with' statement. """
64 _vm = None 48 _vm = None
65
66 """ The VM builds that will be run by the 'vm' command - default is first in list """
67 _vmbuildChoices = ['product', 'fastdebug', 'debug', 'optimized']
68
69 """ The VM build that will be run by the 'vm' command.
70 This can be set via the global '--vmbuild' option.
71 It can also be temporarily set by using of a VM context manager object in a 'with' statement. """
72 _vmbuild = _vmbuildChoices[0]
73 49
74 """ Prefix for running the VM. """ 50 """ Prefix for running the VM. """
75 _vm_prefix = None 51 _vm_prefix = None
76 52
77 _make_eclipse_launch = False 53 _make_eclipse_launch = False
122 with open(envPath) as fp: 98 with open(envPath) as fp:
123 if 'DEFAULT_VM=' + vm in fp.read(): 99 if 'DEFAULT_VM=' + vm in fp.read():
124 mx.log('Please update the DEFAULT_VM value in ' + envPath + ' to replace "graal" with "jvmci"') 100 mx.log('Please update the DEFAULT_VM value in ' + envPath + ' to replace "graal" with "jvmci"')
125 vm = vm.replace('graal', 'jvmci') 101 vm = vm.replace('graal', 'jvmci')
126 if vm is None: 102 if vm is None:
127 if not mx.is_interactive(): 103 mx.abort('Need to specify VM with --vm option or DEFAULT_VM environment variable')
128 mx.abort('Need to specify VM with --vm option or DEFAULT_VM environment variable')
129 mx.log('Please select the VM to be executed from the following: ')
130 items = [k for k in _vmChoices.keys() if _vmChoices[k] is not None]
131 descriptions = [_vmChoices[k] for k in _vmChoices.keys() if _vmChoices[k] is not None]
132 vm = mx.select_items(items, descriptions, allowMultiple=False)
133 if mx.ask_yes_no('Persist this choice by adding "DEFAULT_VM=' + vm + '" to ' + envPath, 'y'):
134 with open(envPath, 'a') as fp:
135 print >> fp, 'DEFAULT_VM=' + vm
136 _vm = vm 104 _vm = vm
137 return vm 105 return vm
138
139 """
140 A context manager that can be used with the 'with' statement to set the VM
141 used by all VM executions within the scope of the 'with' statement. For example:
142
143 with VM('server'):
144 dacapo(['pmd'])
145 """
146 class VM:
147 def __init__(self, vm=None, build=None):
148 assert vm is None or vm in _vmChoices.keys()
149 assert build is None or build in _vmbuildChoices
150 self.vm = vm if vm else _vm
151 self.build = build if build else _vmbuild
152 self.previousVm = _vm
153 self.previousBuild = _vmbuild
154
155 def __enter__(self):
156 global _vm, _vmbuild
157 _vm = self.vm
158 _vmbuild = self.build
159
160 def __exit__(self, exc_type, exc_value, traceback):
161 global _vm, _vmbuild
162 _vm = self.previousVm
163 _vmbuild = self.previousBuild
164 106
165 def chmodRecursive(dirname, chmodFlagsDir): 107 def chmodRecursive(dirname, chmodFlagsDir):
166 if mx.get_os() == 'windows': 108 if mx.get_os() == 'windows':
167 return 109 return
168 110
234 176
235 def _genFileName(archivtype, middle): 177 def _genFileName(archivtype, middle):
236 idPrefix = infos['revision'] + '_' 178 idPrefix = infos['revision'] + '_'
237 idSuffix = '.tar.gz' 179 idSuffix = '.tar.gz'
238 return join(_graal_home, "graalvm_" + archivtype + "_" + idPrefix + middle + idSuffix) 180 return join(_graal_home, "graalvm_" + archivtype + "_" + idPrefix + middle + idSuffix)
239
240 def _genFileArchPlatformName(archivtype, middle):
241 return _genFileName(archivtype, infos['platform'] + '_' + infos['architecture'] + '_' + middle)
242
243
244 # archive different build types of hotspot
245 for vmBuild in _vmbuildChoices:
246 jdkpath = join(_jdksDir(), vmBuild)
247 if not exists(jdkpath):
248 mx.logv("skipping " + vmBuild)
249 continue
250
251 tarName = _genFileArchPlatformName('basejdk', vmBuild)
252 mx.logv("creating basejdk " + tarName)
253 vmSet = set()
254 with tarfile.open(tarName, 'w:gz') as tar:
255 for root, _, files in os.walk(jdkpath):
256 if basename(root) in _vmChoices.keys():
257 # TODO: add some assert to check path assumption
258 vmSet.add(root)
259 continue
260
261 for f in files:
262 name = join(root, f)
263 # print name
264 tar.add(name, name)
265
266 n = _writeJson("basejdk-" + vmBuild, {'vmbuild' : vmBuild})
267 tar.add(n, n)
268
269 # create a separate archive for each VM
270 for vm in vmSet:
271 bVm = basename(vm)
272 vmTarName = _genFileArchPlatformName('vm', vmBuild + '_' + bVm)
273 mx.logv("creating vm " + vmTarName)
274
275 debugFiles = set()
276 with tarfile.open(vmTarName, 'w:gz') as tar:
277 for root, _, files in os.walk(vm):
278 for f in files:
279 # TODO: mac, windows, solaris?
280 if any(map(f.endswith, [".debuginfo"])):
281 debugFiles.add(f)
282 else:
283 name = join(root, f)
284 # print name
285 tar.add(name, name)
286
287 n = _writeJson("vm-" + vmBuild + "-" + bVm, {'vmbuild' : vmBuild, 'vm' : bVm})
288 tar.add(n, n)
289
290 if len(debugFiles) > 0:
291 debugTarName = _genFileArchPlatformName('debugfilesvm', vmBuild + '_' + bVm)
292 mx.logv("creating debugfilesvm " + debugTarName)
293 with tarfile.open(debugTarName, 'w:gz') as tar:
294 for f in debugFiles:
295 name = join(root, f)
296 # print name
297 tar.add(name, name)
298
299 n = _writeJson("debugfilesvm-" + vmBuild + "-" + bVm, {'vmbuild' : vmBuild, 'vm' : bVm})
300 tar.add(n, n)
301 181
302 # graal directory 182 # graal directory
303 graalDirTarName = _genFileName('classfiles', 'javac') 183 graalDirTarName = _genFileName('classfiles', 'javac')
304 mx.logv("creating graal " + graalDirTarName) 184 mx.logv("creating graal " + graalDirTarName)
305 with tarfile.open(graalDirTarName, 'w:gz') as tar: 185 with tarfile.open(graalDirTarName, 'w:gz') as tar:
339 failed.append(bm) 219 failed.append(bm)
340 220
341 if len(failed) != 0: 221 if len(failed) != 0:
342 mx.abort('Benchmark failures: ' + str(failed)) 222 mx.abort('Benchmark failures: ' + str(failed))
343 223
344 def dacapo(args):
345 """run one or more DaCapo benchmarks"""
346
347 def launcher(bm, harnessArgs, extraVmOpts):
348 return sanitycheck.getDacapo(bm, harnessArgs).test(_get_vm(), extraVmOpts=extraVmOpts)
349
350 _run_benchmark(args, sanitycheck.dacapoSanityWarmup.keys(), launcher)
351
352 def scaladacapo(args):
353 """run one or more Scala DaCapo benchmarks"""
354
355 def launcher(bm, harnessArgs, extraVmOpts):
356 return sanitycheck.getScalaDacapo(bm, harnessArgs).test(_get_vm(), extraVmOpts=extraVmOpts)
357
358 _run_benchmark(args, sanitycheck.dacapoScalaSanityWarmup.keys(), launcher)
359
360 def _vmLibDirInJdk(jdk): 224 def _vmLibDirInJdk(jdk):
361 """ 225 """
362 Get the directory within a JDK where the server and client 226 Get the directory within a JDK where the server and client
363 subdirectories are located. 227 subdirectories are located.
364 """ 228 """
394 258
395 def _handle_missing_VM(bld, vm=None): 259 def _handle_missing_VM(bld, vm=None):
396 if not vm: 260 if not vm:
397 vm = _get_vm() 261 vm = _get_vm()
398 mx.log('The ' + bld + ' ' + vm + ' VM has not been created') 262 mx.log('The ' + bld + ' ' + vm + ' VM has not been created')
399 if mx.is_interactive(): 263 mx.abort('You need to run "mx --javahome ' + vm + ' use the selected VM')
400 if mx.ask_yes_no('Build it now', 'y'):
401 with VM(vm, bld):
402 build([])
403 return
404 mx.abort('You need to run "mx --vm ' + vm + ' --vmbuild ' + bld + ' build" to build the selected VM')
405 264
406 def _jdk(build=None, vmToCheck=None, create=False, installJars=True): 265 def _jdk(build=None, vmToCheck=None, create=False, installJars=True):
407 """ 266 """
408 Get the JDK into which Graal is installed, creating it first if necessary. 267 Get the JDK into which Graal is installed, creating it first if necessary.
409 """ 268 """
410 if not build:
411 build = _vmbuild if _vmSourcesAvailable else 'product'
412 jdk = join(_jdksDir(), build) 269 jdk = join(_jdksDir(), build)
413 if create: 270 if create:
414 srcJdk = mx.java().jdk 271 srcJdk = mx.java().jdk
415 if not exists(jdk): 272 if not exists(jdk):
416 mx.log('Creating ' + jdk + ' from ' + srcJdk) 273 mx.log('Creating ' + jdk + ' from ' + srcJdk)
849 706
850 if os.environ.get('BUILDING_FROM_IDE', None) == 'true': 707 if os.environ.get('BUILDING_FROM_IDE', None) == 'true':
851 build = os.environ.get('IDE_BUILD_TARGET', None) 708 build = os.environ.get('IDE_BUILD_TARGET', None)
852 if build is None or len(build) == 0: 709 if build is None or len(build) == 0:
853 return 710 return
854 if build not in _vmbuildChoices:
855 mx.abort('VM build "' + build + '" specified by IDE_BUILD_TARGET environment variable is unknown (must be one of ' +
856 str(_vmbuildChoices) + ')')
857 711
858 if vm is None: 712 if vm is None:
859 vm = _get_vm() 713 vm = _get_vm()
860 714
861 def vmg(args): 715 def vmg(args):
1157 def shortunittest(args): 1011 def shortunittest(args):
1158 """alias for 'unittest --whitelist test/whitelist_shortunittest.txt'{0}""" 1012 """alias for 'unittest --whitelist test/whitelist_shortunittest.txt'{0}"""
1159 1013
1160 args = ['--whitelist', 'test/whitelist_shortunittest.txt'] + args 1014 args = ['--whitelist', 'test/whitelist_shortunittest.txt'] + args
1161 unittest(args) 1015 unittest(args)
1162
1163 def microbench(args):
1164 """run JMH microbenchmark projects"""
1165 vmArgs, jmhArgs = _extract_VM_args(args, useDoubleDash=True)
1166
1167 # look for -f in JMH arguments
1168 containsF = False
1169 forking = True
1170 for i in range(len(jmhArgs)):
1171 arg = jmhArgs[i]
1172 if arg.startswith('-f'):
1173 containsF = True
1174 if arg == '-f' and (i+1) < len(jmhArgs):
1175 arg += jmhArgs[i+1]
1176 try:
1177 if int(arg[2:]) == 0:
1178 forking = False
1179 except ValueError:
1180 pass
1181
1182 # default to -f1 if not specified otherwise
1183 if not containsF:
1184 jmhArgs += ['-f1']
1185
1186 # find all projects with a direct JMH dependency
1187 jmhProjects = []
1188 for p in mx.projects():
1189 if 'JMH' in p.deps:
1190 jmhProjects.append(p.name)
1191 cp = mx.classpath(jmhProjects)
1192
1193 # execute JMH runner
1194 args = ['-cp', cp]
1195 if not forking:
1196 args += vmArgs
1197 args += ['org.openjdk.jmh.Main']
1198 if forking:
1199 (_, _, jvm, _, _) = _parseVmArgs(vmArgs)
1200 args += ['--jvmArgsPrepend', ' '.join(['-' + jvm] + vmArgs)]
1201 vm(args + jmhArgs)
1202
1203 def buildvms(args):
1204 """build one or more VMs in various configurations"""
1205
1206 vmsDefault = ','.join(_vmChoices.keys())
1207 vmbuildsDefault = ','.join(_vmbuildChoices)
1208
1209 parser = ArgumentParser(prog='mx buildvms')
1210 parser.add_argument('--vms', help='a comma separated list of VMs to build (default: ' + vmsDefault + ')', metavar='<args>', default=vmsDefault)
1211 parser.add_argument('--builds', help='a comma separated list of build types (default: ' + vmbuildsDefault + ')', metavar='<args>', default=vmbuildsDefault)
1212 parser.add_argument('--check-distributions', action='store_true', dest='check_distributions', help='check built distributions for overlap')
1213 parser.add_argument('-n', '--no-check', action='store_true', help='omit running "java -version" after each build')
1214 parser.add_argument('-c', '--console', action='store_true', help='send build output to console instead of log file')
1215
1216 args = parser.parse_args(args)
1217 vms = args.vms.split(',')
1218 builds = args.builds.split(',')
1219
1220 allStart = time.time()
1221 check_dists_args = ['--check-distributions'] if args.check_distributions else []
1222 for v in vms:
1223 if not isVMSupported(v):
1224 mx.log('The ' + v + ' VM is not supported on this platform - skipping')
1225 continue
1226
1227 for vmbuild in builds:
1228 if v == 'original' and vmbuild != 'product':
1229 continue
1230 if not args.console:
1231 logFile = join(v + '-' + vmbuild + '.log')
1232 log = open(join(_graal_home, logFile), 'wb')
1233 start = time.time()
1234 mx.log('BEGIN: ' + v + '-' + vmbuild + '\t(see: ' + logFile + ')')
1235 verbose = ['-v'] if mx._opts.verbose else []
1236 # Run as subprocess so that output can be directed to a file
1237 cmd = [sys.executable, '-u', join('mxtool', 'mx.py')] + verbose + ['--vm', v, '--vmbuild', vmbuild, 'build'] + check_dists_args
1238 mx.logv("executing command: " + str(cmd))
1239 subprocess.check_call(cmd, cwd=_graal_home, stdout=log, stderr=subprocess.STDOUT)
1240 duration = datetime.timedelta(seconds=time.time() - start)
1241 mx.log('END: ' + v + '-' + vmbuild + '\t[' + str(duration) + ']')
1242 else:
1243 with VM(v, vmbuild):
1244 build(check_dists_args)
1245 if not args.no_check:
1246 vmargs = ['-version']
1247 if v == 'jvmci':
1248 vmargs.insert(0, '-XX:-BootstrapJVMCI')
1249 vm(vmargs, vm=v, vmbuild=vmbuild)
1250 allDuration = datetime.timedelta(seconds=time.time() - allStart)
1251 mx.log('TOTAL TIME: ' + '[' + str(allDuration) + ']')
1252 1016
1253 class Task: 1017 class Task:
1254 # None or a list of strings. If not None, only tasks whose title 1018 # None or a list of strings. If not None, only tasks whose title
1255 # matches at least one of the substrings in this list will return 1019 # matches at least one of the substrings in this list will return
1256 # a non-None value from __enter__. The body of a 'with Task(...) as t' 1020 # a non-None value from __enter__. The body of a 'with Task(...) as t'
1291 self.duration = datetime.timedelta(seconds=self.end - self.start) 1055 self.duration = datetime.timedelta(seconds=self.end - self.start)
1292 mx.log(time.strftime('gate: %d %b %Y %H:%M:%S: ABORT: ') + self.title + ' [' + str(self.duration) + ']') 1056 mx.log(time.strftime('gate: %d %b %Y %H:%M:%S: ABORT: ') + self.title + ' [' + str(self.duration) + ']')
1293 mx.abort(codeOrMessage) 1057 mx.abort(codeOrMessage)
1294 return self 1058 return self
1295 1059
1296 def ctw(args):
1297 """run CompileTheWorld"""
1298
1299 defaultCtwopts = '-Inline'
1300
1301 parser = ArgumentParser(prog='mx ctw')
1302 parser.add_argument('--ctwopts', action='store', help='space separated JVMCI options used for CTW compilations (default: --ctwopts="' + defaultCtwopts + '")', default=defaultCtwopts, metavar='<options>')
1303 parser.add_argument('--jar', action='store', help='jar of classes to compiled instead of rt.jar', metavar='<path>')
1304
1305 args, vmargs = parser.parse_known_args(args)
1306
1307 if args.ctwopts:
1308 vmargs.append('-G:CompileTheWorldConfig=' + args.ctwopts)
1309
1310 if args.jar:
1311 jar = os.path.abspath(args.jar)
1312 else:
1313 jar = join(_jdk(installJars=False), 'jre', 'lib', 'rt.jar')
1314 vmargs.append('-G:CompileTheWorldExcludeMethodFilter=sun.awt.X11.*.*')
1315
1316 vmargs += ['-XX:+CompileTheWorld']
1317 vm_ = _get_vm()
1318 if isJVMCIEnabled(vm_):
1319 if vm_ == 'jvmci':
1320 vmargs += ['-XX:+BootstrapJVMCI']
1321 vmargs += ['-G:CompileTheWorldClasspath=' + jar]
1322 else:
1323 vmargs += ['-Xbootclasspath/p:' + jar]
1324
1325 # suppress menubar and dock when running on Mac; exclude x11 classes as they may cause vm crashes (on Solaris)
1326 vmargs = ['-Djava.awt.headless=true'] + vmargs
1327
1328 vm(vmargs)
1329
1330 def _basic_gate_body(args, tasks): 1060 def _basic_gate_body(args, tasks):
1331 # Run unit tests on server-hosted-jvmci 1061 # Run unit tests on server-hosted-jvmci
1332 with VM('server', 'product'): 1062 with Task('UnitTests:hosted-product', tasks) as t:
1333 with Task('UnitTests:hosted-product', tasks) as t: 1063 if t: unittest(['--enable-timing', '--verbose', '--fail-fast'])
1334 if t: unittest(['--enable-timing', '--verbose', '--fail-fast'])
1335 1064
1336 1065
1337 def gate(args, gate_body=_basic_gate_body): 1066 def gate(args, gate_body=_basic_gate_body):
1338 """run the tests used to validate a push 1067 """run the tests used to validate a push
1339 1068
1434 mx.log(' =======') 1163 mx.log(' =======')
1435 mx.log(' ' + str(total.duration)) 1164 mx.log(' ' + str(total.duration))
1436 1165
1437 if args.task_filter: 1166 if args.task_filter:
1438 Task.filters = None 1167 Task.filters = None
1439
1440 def deoptalot(args):
1441 """bootstrap a fastdebug JVMCI VM with DeoptimizeALot and VerifyOops on
1442
1443 If the first argument is a number, the process will be repeated
1444 this number of times. All other arguments are passed to the VM."""
1445 count = 1
1446 if len(args) > 0 and args[0].isdigit():
1447 count = int(args[0])
1448 del args[0]
1449
1450 for _ in range(count):
1451 if not vm(['-XX:+DeoptimizeALot', '-XX:+VerifyOops'] + args + ['-version'], vmbuild='fastdebug') == 0:
1452 mx.abort("Failed")
1453
1454 def longtests(args):
1455
1456 deoptalot(['15', '-Xmx48m'])
1457
1458 dacapo(['100', 'eclipse', '-esa'])
1459 1168
1460 def _igvJdk(): 1169 def _igvJdk():
1461 v8u20 = mx.VersionSpec("1.8.0_20") 1170 v8u20 = mx.VersionSpec("1.8.0_20")
1462 v8u40 = mx.VersionSpec("1.8.0_40") 1171 v8u40 = mx.VersionSpec("1.8.0_40")
1463 v8 = mx.VersionSpec("1.8") 1172 v8 = mx.VersionSpec("1.8")
1545 if mx.get_os() != 'windows': 1254 if mx.get_os() != 'windows':
1546 # Make sure that execution is allowed. The zip file does not always specfiy that correctly 1255 # Make sure that execution is allowed. The zip file does not always specfiy that correctly
1547 os.chmod(executable, 0777) 1256 os.chmod(executable, 0777)
1548 1257
1549 mx.run([executable]) 1258 mx.run([executable])
1550
1551 def bench(args):
1552 """run benchmarks and parse their output for results
1553
1554 Results are JSON formated : {group : {benchmark : score}}."""
1555 resultFile = None
1556 if '-resultfile' in args:
1557 index = args.index('-resultfile')
1558 if index + 1 < len(args):
1559 resultFile = args[index + 1]
1560 del args[index]
1561 del args[index]
1562 else:
1563 mx.abort('-resultfile must be followed by a file name')
1564 vm = _get_vm()
1565 if len(args) is 0:
1566 args = ['all']
1567
1568 vmArgs = [arg for arg in args if arg.startswith('-')]
1569
1570 def benchmarks_in_group(group):
1571 prefix = group + ':'
1572 return [a[len(prefix):] for a in args if a.startswith(prefix)]
1573
1574 results = {}
1575 benchmarks = []
1576 # DaCapo
1577 if 'dacapo' in args or 'all' in args:
1578 benchmarks += sanitycheck.getDacapos(level=sanitycheck.SanityCheckLevel.Benchmark)
1579 else:
1580 dacapos = benchmarks_in_group('dacapo')
1581 for dacapo in dacapos:
1582 if dacapo not in sanitycheck.dacapoSanityWarmup.keys():
1583 mx.abort('Unknown DaCapo : ' + dacapo)
1584 iterations = sanitycheck.dacapoSanityWarmup[dacapo][sanitycheck.SanityCheckLevel.Benchmark]
1585 if iterations > 0:
1586 benchmarks += [sanitycheck.getDacapo(dacapo, iterations)]
1587
1588 if 'scaladacapo' in args or 'all' in args:
1589 benchmarks += sanitycheck.getScalaDacapos(level=sanitycheck.SanityCheckLevel.Benchmark)
1590 else:
1591 scaladacapos = benchmarks_in_group('scaladacapo')
1592 for scaladacapo in scaladacapos:
1593 if scaladacapo not in sanitycheck.dacapoScalaSanityWarmup.keys():
1594 mx.abort('Unknown Scala DaCapo : ' + scaladacapo)
1595 iterations = sanitycheck.dacapoScalaSanityWarmup[scaladacapo][sanitycheck.SanityCheckLevel.Benchmark]
1596 if iterations > 0:
1597 benchmarks += [sanitycheck.getScalaDacapo(scaladacapo, ['-n', str(iterations)])]
1598
1599 # Bootstrap
1600 if 'bootstrap' in args or 'all' in args:
1601 benchmarks += sanitycheck.getBootstraps()
1602 # SPECjvm2008
1603 if 'specjvm2008' in args or 'all' in args:
1604 benchmarks += [sanitycheck.getSPECjvm2008(['-ikv', '-wt', '120', '-it', '120'])]
1605 else:
1606 specjvms = benchmarks_in_group('specjvm2008')
1607 for specjvm in specjvms:
1608 benchmarks += [sanitycheck.getSPECjvm2008(['-ikv', '-wt', '120', '-it', '120', specjvm])]
1609
1610 if 'specjbb2005' in args or 'all' in args:
1611 benchmarks += [sanitycheck.getSPECjbb2005()]
1612
1613 if 'specjbb2013' in args: # or 'all' in args //currently not in default set
1614 benchmarks += [sanitycheck.getSPECjbb2013()]
1615
1616 if 'ctw-full' in args:
1617 benchmarks.append(sanitycheck.getCTW(vm, sanitycheck.CTWMode.Full))
1618 if 'ctw-noinline' in args:
1619 benchmarks.append(sanitycheck.getCTW(vm, sanitycheck.CTWMode.NoInline))
1620
1621 for test in benchmarks:
1622 for (groupName, res) in test.bench(vm, extraVmOpts=vmArgs).items():
1623 group = results.setdefault(groupName, {})
1624 group.update(res)
1625 mx.log(json.dumps(results))
1626 if resultFile:
1627 with open(resultFile, 'w') as f:
1628 f.write(json.dumps(results))
1629
1630 def _get_jmh_path():
1631 path = mx.get_env('JMH_BENCHMARKS', None)
1632 if not path:
1633 probe = join(dirname(_graal_home), 'java-benchmarks')
1634 if exists(probe):
1635 path = probe
1636
1637 if not path:
1638 mx.abort("Please set the JMH_BENCHMARKS environment variable to point to the java-benchmarks workspace")
1639 if not exists(path):
1640 mx.abort("The directory denoted by the JMH_BENCHMARKS environment variable does not exist: " + path)
1641 return path
1642
1643 def makejmhdeps(args):
1644 """creates and installs Maven dependencies required by the JMH benchmarks
1645
1646 The dependencies are specified by files named pom.mxdeps in the
1647 JMH directory tree. Each such file contains a list of dependencies
1648 defined in JSON format. For example:
1649
1650 '[{"artifactId" : "compiler.test", "groupId" : "com.oracle.graal", "deps" : ["com.oracle.graal.compiler.test"]}]'
1651
1652 will result in a dependency being installed in the local Maven repository
1653 that can be referenced in a pom.xml file as follows:
1654
1655 <dependency>
1656 <groupId>com.oracle.graal</groupId>
1657 <artifactId>compiler.test</artifactId>
1658 <version>1.0-SNAPSHOT</version>
1659 </dependency>"""
1660
1661 parser = ArgumentParser(prog='mx makejmhdeps')
1662 parser.add_argument('-s', '--settings', help='alternative path for Maven user settings file', metavar='<path>')
1663 parser.add_argument('-p', '--permissive', action='store_true', help='issue note instead of error if a Maven dependency cannot be built due to missing projects/libraries')
1664 args = parser.parse_args(args)
1665
1666 def makejmhdep(artifactId, groupId, deps):
1667 graalSuite = mx.suite("graal")
1668 path = artifactId + '.jar'
1669 if args.permissive:
1670 allDeps = []
1671 for name in deps:
1672 dist = mx.distribution(name, fatalIfMissing=False)
1673 if dist:
1674 allDeps = allDeps + [d.name for d in dist.sorted_deps(transitive=True)]
1675 else:
1676 if not mx.project(name, fatalIfMissing=False):
1677 if not mx.library(name, fatalIfMissing=False):
1678 mx.log('Skipping dependency ' + groupId + '.' + artifactId + ' as ' + name + ' cannot be resolved')
1679 return
1680 allDeps.append(name)
1681 d = mx.Distribution(graalSuite, name=artifactId, path=path, sourcesPath=path, deps=allDeps, mainClass=None, excludedDependencies=[], distDependencies=[], javaCompliance=None)
1682 d.make_archive()
1683 cmd = ['mvn', 'install:install-file', '-DgroupId=' + groupId, '-DartifactId=' + artifactId,
1684 '-Dversion=1.0-SNAPSHOT', '-Dpackaging=jar', '-Dfile=' + d.path]
1685 if not mx._opts.verbose:
1686 cmd.append('-q')
1687 if args.settings:
1688 cmd = cmd + ['-s', args.settings]
1689 mx.run(cmd)
1690 os.unlink(d.path)
1691
1692 jmhPath = _get_jmh_path()
1693 for root, _, filenames in os.walk(jmhPath):
1694 for f in [join(root, n) for n in filenames if n == 'pom.mxdeps']:
1695 mx.logv('[processing ' + f + ']')
1696 try:
1697 with open(f) as fp:
1698 for d in json.load(fp):
1699 artifactId = d['artifactId']
1700 groupId = d['groupId']
1701 deps = d['deps']
1702 makejmhdep(artifactId, groupId, deps)
1703 except ValueError as e:
1704 mx.abort('Error parsing {0}:\n{1}'.format(f, e))
1705
1706 def buildjmh(args):
1707 """build the JMH benchmarks"""
1708
1709 parser = ArgumentParser(prog='mx buildjmh')
1710 parser.add_argument('-s', '--settings', help='alternative path for Maven user settings file', metavar='<path>')
1711 parser.add_argument('-c', action='store_true', dest='clean', help='clean before building')
1712 args = parser.parse_args(args)
1713
1714 jmhPath = _get_jmh_path()
1715 mx.log('JMH benchmarks: ' + jmhPath)
1716
1717 # Ensure the mx injected dependencies are up to date
1718 makejmhdeps(['-p'] + (['-s', args.settings] if args.settings else []))
1719
1720 timestamp = mx.TimeStampFile(join(_graal_home, 'mx', 'jmh', jmhPath.replace(os.sep, '_') + '.timestamp'))
1721 mustBuild = args.clean
1722 if not mustBuild:
1723 try:
1724 hgfiles = [join(jmhPath, f) for f in subprocess.check_output(['hg', '-R', jmhPath, 'locate']).split('\n')]
1725 mustBuild = timestamp.isOlderThan(hgfiles)
1726 except:
1727 # not a Mercurial repository or hg commands are not available.
1728 mustBuild = True
1729
1730 if mustBuild:
1731 buildOutput = []
1732 def _redirect(x):
1733 if mx._opts.verbose:
1734 mx.log(x[:-1])
1735 else:
1736 buildOutput.append(x)
1737 env = os.environ.copy()
1738 env['JAVA_HOME'] = _jdk(vmToCheck='server')
1739 env['MAVEN_OPTS'] = '-server'
1740 mx.log("Building benchmarks...")
1741 cmd = ['mvn']
1742 if args.settings:
1743 cmd = cmd + ['-s', args.settings]
1744 if args.clean:
1745 cmd.append('clean')
1746 cmd.append('package')
1747 retcode = mx.run(cmd, cwd=jmhPath, out=_redirect, env=env, nonZeroIsFatal=False)
1748 if retcode != 0:
1749 mx.log(''.join(buildOutput))
1750 mx.abort(retcode)
1751 timestamp.touch()
1752 else:
1753 mx.logv('[all Mercurial controlled files in ' + jmhPath + ' are older than ' + timestamp.path + ' - skipping build]')
1754
1755 def jmh(args):
1756 """run the JMH benchmarks
1757
1758 This command respects the standard --vm and --vmbuild options
1759 for choosing which VM to run the benchmarks with."""
1760 if '-h' in args:
1761 mx.help_(['jmh'])
1762 mx.abort(1)
1763
1764 vmArgs, benchmarksAndJsons = _extract_VM_args(args)
1765
1766 benchmarks = [b for b in benchmarksAndJsons if not b.startswith('{')]
1767 jmhArgJsons = [b for b in benchmarksAndJsons if b.startswith('{')]
1768 jmhOutDir = join(_graal_home, 'mx', 'jmh')
1769 if not exists(jmhOutDir):
1770 os.makedirs(jmhOutDir)
1771 jmhOut = join(jmhOutDir, 'jmh.out')
1772 jmhArgs = {'-rff' : jmhOut, '-v' : 'EXTRA' if mx._opts.verbose else 'NORMAL'}
1773
1774 # e.g. '{"-wi" : 20}'
1775 for j in jmhArgJsons:
1776 try:
1777 for n, v in json.loads(j).iteritems():
1778 if v is None:
1779 del jmhArgs[n]
1780 else:
1781 jmhArgs[n] = v
1782 except ValueError as e:
1783 mx.abort('error parsing JSON input: {0}\n{1}'.format(j, e))
1784
1785 jmhPath = _get_jmh_path()
1786 mx.log('Using benchmarks in ' + jmhPath)
1787
1788 matchedSuites = set()
1789 numBench = [0]
1790 for micros in os.listdir(jmhPath):
1791 absoluteMicro = os.path.join(jmhPath, micros)
1792 if not os.path.isdir(absoluteMicro):
1793 continue
1794 if not micros.startswith("micros-"):
1795 mx.logv('JMH: ignored ' + absoluteMicro + " because it doesn't start with 'micros-'")
1796 continue
1797
1798 microJar = os.path.join(absoluteMicro, "target", "microbenchmarks.jar")
1799 if not exists(microJar):
1800 mx.log('Missing ' + microJar + ' - please run "mx buildjmh"')
1801 continue
1802 if benchmarks:
1803 def _addBenchmark(x):
1804 if x.startswith("Benchmark:"):
1805 return
1806 match = False
1807 for b in benchmarks:
1808 match = match or (b in x)
1809
1810 if match:
1811 numBench[0] += 1
1812 matchedSuites.add(micros)
1813
1814 mx.run_java(['-jar', microJar, "-l"], cwd=jmhPath, out=_addBenchmark, addDefaultArgs=False)
1815 else:
1816 matchedSuites.add(micros)
1817
1818 mx.logv("matchedSuites: " + str(matchedSuites))
1819 plural = 's' if not benchmarks or numBench[0] > 1 else ''
1820 number = str(numBench[0]) if benchmarks else "all"
1821 mx.log("Running " + number + " benchmark" + plural + '...')
1822
1823 regex = []
1824 if benchmarks:
1825 regex.append(r".*(" + "|".join(benchmarks) + ").*")
1826
1827 for suite in matchedSuites:
1828 absoluteMicro = os.path.join(jmhPath, suite)
1829 (pfx, exe, vm, forkedVmArgs, _) = _parseVmArgs(vmArgs)
1830 if pfx:
1831 mx.log("JMH ignores prefix: \"" + ' '.join(pfx) + "\"")
1832 javaArgs = ['-jar', os.path.join(absoluteMicro, "target", "microbenchmarks.jar"),
1833 '--jvm', exe,
1834 '--jvmArgs', ' '.join(["-" + vm] + forkedVmArgs)]
1835 for k, v in jmhArgs.iteritems():
1836 javaArgs.append(k)
1837 if len(str(v)):
1838 javaArgs.append(str(v))
1839 mx.run_java(javaArgs + regex, addDefaultArgs=False, cwd=jmhPath)
1840
1841 def specjvm2008(args):
1842 """run one or more SPECjvm2008 benchmarks"""
1843
1844 def launcher(bm, harnessArgs, extraVmOpts):
1845 return sanitycheck.getSPECjvm2008(harnessArgs + [bm]).bench(_get_vm(), extraVmOpts=extraVmOpts)
1846
1847 availableBenchmarks = set(sanitycheck.specjvm2008Names)
1848 for name in sanitycheck.specjvm2008Names:
1849 parts = name.rsplit('.', 1)
1850 if len(parts) > 1:
1851 assert len(parts) == 2
1852 group = parts[0]
1853 availableBenchmarks.add(group)
1854
1855 _run_benchmark(args, sorted(availableBenchmarks), launcher)
1856
1857 def specjbb2013(args):
1858 """run the composite SPECjbb2013 benchmark"""
1859
1860 def launcher(bm, harnessArgs, extraVmOpts):
1861 assert bm is None
1862 return sanitycheck.getSPECjbb2013(harnessArgs).bench(_get_vm(), extraVmOpts=extraVmOpts)
1863
1864 _run_benchmark(args, None, launcher)
1865
1866 def specjbb2005(args):
1867 """run the composite SPECjbb2005 benchmark"""
1868
1869 def launcher(bm, harnessArgs, extraVmOpts):
1870 assert bm is None
1871 return sanitycheck.getSPECjbb2005(harnessArgs).bench(_get_vm(), extraVmOpts=extraVmOpts)
1872
1873 _run_benchmark(args, None, launcher)
1874 1259
1875 def hsdis(args, copyToDir=None): 1260 def hsdis(args, copyToDir=None):
1876 """download the hsdis library 1261 """download the hsdis library
1877 1262
1878 This is needed to support HotSpot's assembly dumping features. 1263 This is needed to support HotSpot's assembly dumping features.
2028 complt += '\t\t;;\n' 1413 complt += '\t\t;;\n'
2029 1414
2030 complt += '\t(args)\n' 1415 complt += '\t(args)\n'
2031 # TODO: improve matcher: if mx args are given, this doesn't work 1416 # TODO: improve matcher: if mx args are given, this doesn't work
2032 complt += '\t\tcase $line[1] in\n' 1417 complt += '\t\tcase $line[1] in\n'
2033 complt += '\t\t\t(vm | vmg | vmfg | unittest | jmh | dacapo | scaladacapo | specjvm2008 | specjbb2013 | specjbb2005)\n'
2034 complt += '\t\t\t\tnoglob \\\n' 1418 complt += '\t\t\t\tnoglob \\\n'
2035 complt += '\t\t\t\t\t_arguments -s -S \\\n' 1419 complt += '\t\t\t\t\t_arguments -s -S \\\n'
2036 complt += _appendOptions("jvmci", r"G\:") 1420 complt += _appendOptions("jvmci", r"G\:")
2037 # TODO: fix -XX:{-,+}Use* flags 1421 # TODO: fix -XX:{-,+}Use* flags
2038 complt += _appendOptions("hotspot", r"XX\:") 1422 complt += _appendOptions("hotspot", r"XX\:")
2172 mx.log('{0}: header does not match RegexpHeader defined in {1}'.format(n, v)) 1556 mx.log('{0}: header does not match RegexpHeader defined in {1}'.format(n, v))
2173 return len(failures) 1557 return len(failures)
2174 1558
2175 def mx_init(suite): 1559 def mx_init(suite):
2176 commands = { 1560 commands = {
2177 'build': [build, ''],
2178 'buildjmh': [buildjmh, '[-options]'],
2179 'buildvars': [buildvars, ''],
2180 'buildvms': [buildvms, '[-options]'],
2181 'c1visualizer' : [c1visualizer, ''],
2182 'checkheaders': [checkheaders, ''], 1561 'checkheaders': [checkheaders, ''],
2183 'clean': [clean, ''], 1562 'clean': [clean, ''],
2184 'ctw': [ctw, '[-vmoptions|noinline|nocomplex|full]'],
2185 'findbugs': [findbugs, ''], 1563 'findbugs': [findbugs, ''],
2186 'generateZshCompletion' : [generateZshCompletion, ''], 1564 'generateZshCompletion' : [generateZshCompletion, ''],
2187 'hsdis': [hsdis, '[att]'],
2188 'hcfdis': [hcfdis, ''],
2189 'igv' : [igv, ''],
2190 'maven-install-truffle' : [maven_install_truffle, ''], 1565 'maven-install-truffle' : [maven_install_truffle, ''],
2191 'jdkhome': [print_jdkhome, ''], 1566 'jdkhome': [print_jdkhome, ''],
2192 'jmh': [jmh, '[VM options] [filters|JMH-args-as-json...]'],
2193 'dacapo': [dacapo, '[VM options] benchmarks...|"all" [DaCapo options]'],
2194 'scaladacapo': [scaladacapo, '[VM options] benchmarks...|"all" [Scala DaCapo options]'],
2195 'specjvm2008': [specjvm2008, '[VM options] benchmarks...|"all" [SPECjvm2008 options]'],
2196 'specjbb2013': [specjbb2013, '[VM options] [-- [SPECjbb2013 options]]'],
2197 'specjbb2005': [specjbb2005, '[VM options] [-- [SPECjbb2005 options]]'],
2198 'gate' : [gate, '[-options]'], 1567 'gate' : [gate, '[-options]'],
2199 'bench' : [bench, '[-resultfile file] [all(default)|dacapo|specjvm2008|bootstrap]'],
2200 'microbench' : [microbench, '[VM options] [-- [JMH options]]'],
2201 'unittest' : [unittest, '[unittest options] [--] [VM options] [filters...]', _unittestHelpSuffix], 1568 'unittest' : [unittest, '[unittest options] [--] [VM options] [filters...]', _unittestHelpSuffix],
2202 'makejmhdeps' : [makejmhdeps, ''],
2203 'shortunittest' : [shortunittest, '[unittest options] [--] [VM options] [filters...]', _unittestHelpSuffix], 1569 'shortunittest' : [shortunittest, '[unittest options] [--] [VM options] [filters...]', _unittestHelpSuffix],
2204 'site' : [site, '[-options]'], 1570 'site' : [site, '[-options]'],
2205 'vm': [vm, '[-options] class [args...]'],
2206 'vmg': [vmg, '[-options] class [args...]'],
2207 'vmfg': [vmfg, '[-options] class [args...]'],
2208 'deoptalot' : [deoptalot, '[n]'],
2209 'longtests' : [longtests, ''],
2210 'sl' : [sl, '[SL args|@VM options]'], 1571 'sl' : [sl, '[SL args|@VM options]'],
2211 'sldebug' : [sldebug, '[SL args|@VM options]'], 1572 'sldebug' : [sldebug, '[SL args|@VM options]'],
2212 'jol' : [jol, ''], 1573 'jol' : [jol, ''],
2213 'makefile' : [mx_graal_makefile.build_makefile, 'build makefiles for JDK build'],
2214 } 1574 }
2215 1575
2216 if _vmSourcesAvailable: 1576 if _vmSourcesAvailable:
2217 mx.add_argument('--vm', action='store', dest='vm', choices=_vmChoices.keys(), help='the VM type to build/run')
2218 mx.add_argument('--vmbuild', action='store', dest='vmbuild', choices=_vmbuildChoices, help='the VM build to build/run (default: ' + _vmbuildChoices[0] + ')')
2219 mx.add_argument('--ecl', action='store_true', dest='make_eclipse_launch', help='create launch configuration for running VM execution(s) in Eclipse') 1577 mx.add_argument('--ecl', action='store_true', dest='make_eclipse_launch', help='create launch configuration for running VM execution(s) in Eclipse')
2220 mx.add_argument('--vmprefix', action='store', dest='vm_prefix', help='prefix for running the VM (e.g. "/usr/bin/gdb --args")', metavar='<prefix>') 1578 mx.add_argument('--vmprefix', action='store', dest='vm_prefix', help='prefix for running the VM (e.g. "/usr/bin/gdb --args")', metavar='<prefix>')
2221 mx.add_argument('--gdb', action='store_const', const='/usr/bin/gdb --args', dest='vm_prefix', help='alias for --vmprefix "/usr/bin/gdb --args"') 1579 mx.add_argument('--gdb', action='store_const', const='/usr/bin/gdb --args', dest='vm_prefix', help='alias for --vmprefix "/usr/bin/gdb --args"')
2222 mx.add_argument('--lldb', action='store_const', const='lldb --', dest='vm_prefix', help='alias for --vmprefix "lldb --"') 1580 mx.add_argument('--lldb', action='store_const', const='lldb --', dest='vm_prefix', help='alias for --vmprefix "lldb --"')
2223 1581
2233 mx.abort('Requires Java version ' + str(_minVersion) + ' or greater for JAVA_HOME, got version ' + str(mx.java().version)) 1591 mx.abort('Requires Java version ' + str(_minVersion) + ' or greater for JAVA_HOME, got version ' + str(mx.java().version))
2234 if _untilVersion and mx.java().version >= _untilVersion: 1592 if _untilVersion and mx.java().version >= _untilVersion:
2235 mx.abort('Requires Java version strictly before ' + str(_untilVersion) + ' for JAVA_HOME, got version ' + str(mx.java().version)) 1593 mx.abort('Requires Java version strictly before ' + str(_untilVersion) + ' for JAVA_HOME, got version ' + str(mx.java().version))
2236 1594
2237 if _vmSourcesAvailable: 1595 if _vmSourcesAvailable:
2238 if hasattr(opts, 'vm') and opts.vm is not None:
2239 global _vm
2240 _vm = opts.vm
2241 _vm = _vm.replace('graal', 'jvmci')
2242 if hasattr(opts, 'vmbuild') and opts.vmbuild is not None:
2243 global _vmbuild
2244 _vmbuild = opts.vmbuild
2245 global _make_eclipse_launch 1596 global _make_eclipse_launch
2246 _make_eclipse_launch = getattr(opts, 'make_eclipse_launch', False) 1597 _make_eclipse_launch = getattr(opts, 'make_eclipse_launch', False)
2247 global _vm_prefix 1598 global _vm_prefix
2248 _vm_prefix = opts.vm_prefix 1599 _vm_prefix = opts.vm_prefix
2249 1600