Mercurial > hg > truffle

comparison mx/mx_graal.py @ 12764:6cdf3b17316d

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
mx: rename commands.py as mx_graal.py
author Mick Jordan <mick.jordan@oracle.com>
date Tue, 12 Nov 2013 19:39:25 -0800
parents mx/commands.py@c6b833f7935e
children 6140eda73e6f
comparison
equal deleted inserted replaced
12763:bff24aae4640 12764:6cdf3b17316d
1 #
2 # commands.py - the GraalVM specific commands
3 #
4 # ----------------------------------------------------------------------------------------------------
5 #
6 # Copyright (c) 2007, 2012, Oracle and/or its affiliates. All rights reserved.
7 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
8 #
9 # This code is free software; you can redistribute it and/or modify it
10 # under the terms of the GNU General Public License version 2 only, as
11 # published by the Free Software Foundation.
12 #
13 # This code is distributed in the hope that it will be useful, but WITHOUT
14 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 # version 2 for more details (a copy is included in the LICENSE file that
17 # accompanied this code).
18 #
19 # You should have received a copy of the GNU General Public License version
20 # 2 along with this work; if not, write to the Free Software Foundation,
21 # Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
22 #
23 # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
24 # or visit www.oracle.com if you need additional information or have any
25 # questions.
26 #
27 # ----------------------------------------------------------------------------------------------------
28
29 import os, sys, shutil, zipfile, tempfile, re, time, datetime, platform, subprocess, multiprocessing
30 from os.path import join, exists, dirname, basename, getmtime
31 from argparse import ArgumentParser, REMAINDER
32 import mx
33 import sanitycheck
34 import itertools
35 import json, textwrap
36
37 # This works because when mx loads this file, it makes sure __file__ gets an absolute path
38 _graal_home = dirname(dirname(__file__))
39
40 """ Used to distinguish an exported GraalVM (see 'mx export'). """
41 _vmSourcesAvailable = exists(join(_graal_home, 'make')) and exists(join(_graal_home, 'src'))
42
43 """ The VMs that can be built and run along with an optional description. Only VMs with a
44 description are listed in the dialogue for setting the default VM (see _get_vm()). """
45 _vmChoices = {
46 'graal' : 'All compilation is performed with Graal. This includes bootstrapping Graal itself unless -XX:-BootstrapGraal is used.',
47 'server' : 'Normal compilation is performed with the tiered system (i.e., client + server), Truffle compilation is performed with Graal. Use this for optimal Truffle performance.',
48 'client' : None, # normal compilation with client compiler, explicit compilation (e.g., by Truffle) with Graal
49 'server-nograal' : None, # all compilation with tiered system (i.e., client + server), Graal omitted
50 'client-nograal' : None, # all compilation with client compiler, Graal omitted
51 'original' : None, # default VM copied from bootstrap JDK
52 }
53
54 """ The VM that will be run by the 'vm' command and built by default by the 'build' command.
55 This can be set via the global '--vm' option or the DEFAULT_VM environment variable.
56 It can also be temporarily set by using of a VM context manager object in a 'with' statement. """
57 _vm = None
58
59 """ The VM builds that will be run by the 'vm' command - default is first in list """
60 _vmbuildChoices = ['product', 'fastdebug', 'debug', 'optimized']
61
62 """ The VM build that will be run by the 'vm' command.
63 This can be set via the global '--vmbuild' option.
64 It can also be temporarily set by using of a VM context manager object in a 'with' statement. """
65 _vmbuild = _vmbuildChoices[0]
66
67 _jacoco = 'off'
68
69 """ The current working directory to switch to before running the VM. """
70 _vm_cwd = None
71
72 """ The base directory in which the JDKs cloned from $JAVA_HOME exist. """
73 _installed_jdks = None
74
75 """ Prefix for running the VM. """
76 _vm_prefix = None
77
78 _make_eclipse_launch = False
79
80 _minVersion = mx.JavaVersion('1.7.0_04')
81
82 def _get_vm():
83 """
84 Gets the configured VM, presenting a dialogue if there is no currently configured VM.
85 """
86 global _vm
87 if _vm:
88 return _vm
89 vm = mx.get_env('DEFAULT_VM')
90 if vm is None:
91 if not sys.stdout.isatty():
92 mx.abort('Need to specify VM with --vm option or DEFAULT_VM environment variable')
93 envPath = join(_graal_home, 'mx', 'env')
94 mx.log('Please select the VM to be executed from the following: ')
95 items = [k for k in _vmChoices.keys() if _vmChoices[k] is not None]
96 descriptions = [_vmChoices[k] for k in _vmChoices.keys() if _vmChoices[k] is not None]
97 vm = mx.select_items(items, descriptions, allowMultiple=False)
98 if mx.ask_yes_no('Persist this choice by adding "DEFAULT_VM=' + vm + '" to ' + envPath, 'y'):
99 with open(envPath, 'a') as fp:
100 print >> fp, 'DEFAULT_VM=' + vm
101 _vm = vm
102 return vm
103
104 """
105 A context manager that can be used with the 'with' statement to set the VM
106 used by all VM executions within the scope of the 'with' statement. For example:
107
108 with VM('server'):
109 dacapo(['pmd'])
110 """
111 class VM:
112 def __init__(self, vm=None, build=None):
113 assert vm is None or vm in _vmChoices.keys()
114 assert build is None or build in _vmbuildChoices
115 self.vm = vm if vm else _vm
116 self.build = build if build else _vmbuild
117 self.previousVm = _vm
118 self.previousBuild = _vmbuild
119
120 def __enter__(self):
121 global _vm, _vmbuild
122 _vm = self.vm
123 _vmbuild = self.build
124
125 def __exit__(self, exc_type, exc_value, traceback):
126 global _vm, _vmbuild
127 _vm = self.previousVm
128 _vmbuild = self.previousBuild
129
130 def _chmodDir(chmodFlags, dirname, fnames):
131 os.chmod(dirname, chmodFlags)
132 for name in fnames:
133 os.chmod(os.path.join(dirname, name), chmodFlags)
134
135 def chmodRecursive(dirname, chmodFlags):
136 os.path.walk(dirname, _chmodDir, chmodFlags)
137
138 def clean(args):
139 """clean the GraalVM source tree"""
140 opts = mx.clean(args, parser=ArgumentParser(prog='mx clean'))
141 if opts.native:
142 def rmIfExists(name):
143 if os.path.isdir(name):
144 shutil.rmtree(name)
145 elif os.path.isfile(name):
146 os.unlink(name)
147
148 rmIfExists(join(_graal_home, 'build'))
149 rmIfExists(join(_graal_home, 'build-nograal'))
150 rmIfExists(_jdksDir())
151 rmIfExists(mx.distribution('GRAAL').path)
152
153 def export(args):
154 """create a GraalVM zip file for distribution"""
155
156 parser = ArgumentParser(prog='mx export')
157 parser.add_argument('--omit-vm-build', action='store_false', dest='vmbuild', help='omit VM build step')
158 parser.add_argument('--omit-dist-init', action='store_false', dest='distInit', help='omit class files and IDE configurations from distribution')
159 parser.add_argument('zipfile', nargs=REMAINDER, metavar='zipfile')
160
161 args = parser.parse_args(args)
162
163 tmp = tempfile.mkdtemp(prefix='tmp', dir=_graal_home)
164 if args.vmbuild:
165 # Make sure the product VM binary is up to date
166 with VM(vmbuild='product'):
167 build([])
168
169 mx.log('Copying Java sources and mx files...')
170 mx.run(('hg archive -I graal -I mx -I mxtool -I mx.sh ' + tmp).split())
171
172 # Copy the GraalVM JDK
173 mx.log('Copying GraalVM JDK...')
174 src = _jdk()
175 dst = join(tmp, basename(src))
176 shutil.copytree(src, dst)
177 zfName = join(_graal_home, 'graalvm-' + mx.get_os() + '.zip')
178 zf = zipfile.ZipFile(zfName, 'w')
179 for root, _, files in os.walk(tmp):
180 for f in files:
181 name = join(root, f)
182 arcname = name[len(tmp) + 1:]
183 zf.write(join(tmp, name), arcname)
184
185 # create class files and IDE configurations
186 if args.distInit:
187 mx.log('Creating class files...')
188 mx.run('mx build'.split(), cwd=tmp)
189 mx.log('Creating IDE configurations...')
190 mx.run('mx ideinit'.split(), cwd=tmp)
191
192 # clean up temp directory
193 mx.log('Cleaning up...')
194 shutil.rmtree(tmp)
195
196 mx.log('Created distribution in ' + zfName)
197
198 def _run_benchmark(args, availableBenchmarks, runBenchmark):
199
200 vmOpts, benchmarksAndOptions = _extract_VM_args(args, useDoubleDash=availableBenchmarks is None)
201
202 if availableBenchmarks is None:
203 harnessArgs = benchmarksAndOptions
204 return runBenchmark(None, harnessArgs, vmOpts)
205
206 if len(benchmarksAndOptions) == 0:
207 mx.abort('at least one benchmark name or "all" must be specified')
208 benchmarks = list(itertools.takewhile(lambda x: not x.startswith('-'), benchmarksAndOptions))
209 harnessArgs = benchmarksAndOptions[len(benchmarks):]
210
211 if 'all' in benchmarks:
212 benchmarks = availableBenchmarks
213 else:
214 for bm in benchmarks:
215 if bm not in availableBenchmarks:
216 mx.abort('unknown benchmark: ' + bm + '\nselect one of: ' + str(availableBenchmarks))
217
218 failed = []
219 for bm in benchmarks:
220 if not runBenchmark(bm, harnessArgs, vmOpts):
221 failed.append(bm)
222
223 if len(failed) != 0:
224 mx.abort('Benchmark failures: ' + str(failed))
225
226 def dacapo(args):
227 """run one or more DaCapo benchmarks"""
228
229 def launcher(bm, harnessArgs, extraVmOpts):
230 return sanitycheck.getDacapo(bm, harnessArgs).test(_get_vm(), extraVmOpts=extraVmOpts)
231
232 _run_benchmark(args, sanitycheck.dacapoSanityWarmup.keys(), launcher)
233
234 def scaladacapo(args):
235 """run one or more Scala DaCapo benchmarks"""
236
237 def launcher(bm, harnessArgs, extraVmOpts):
238 return sanitycheck.getScalaDacapo(bm, harnessArgs).test(_get_vm(), extraVmOpts=extraVmOpts)
239
240 _run_benchmark(args, sanitycheck.dacapoScalaSanityWarmup.keys(), launcher)
241
242 def _arch():
243 machine = platform.uname()[4]
244 if machine in ['amd64', 'AMD64', 'x86_64', 'i86pc']:
245 return 'amd64'
246 if machine in ['sun4v', 'sun4u']:
247 return 'sparcv9'
248 if machine == 'i386' and mx.get_os() == 'darwin':
249 try:
250 # Support for Snow Leopard and earlier version of MacOSX
251 if subprocess.check_output(['sysctl', '-n', 'hw.cpu64bit_capable']).strip() == '1':
252 return 'amd64'
253 except OSError:
254 # sysctl is not available
255 pass
256 mx.abort('unknown or unsupported architecture: os=' + mx.get_os() + ', machine=' + machine)
257
258 def _vmLibDirInJdk(jdk):
259 """
260 Get the directory within a JDK where the server and client
261 subdirectories are located.
262 """
263 if platform.system() == 'Darwin':
264 return join(jdk, 'jre', 'lib')
265 if platform.system() == 'Windows':
266 return join(jdk, 'jre', 'bin')
267 return join(jdk, 'jre', 'lib', _arch())
268
269 def _vmCfgInJdk(jdk):
270 """
271 Get the jvm.cfg file.
272 """
273 if platform.system() == 'Windows':
274 return join(jdk, 'jre', 'lib', _arch(), 'jvm.cfg')
275 return join(_vmLibDirInJdk(jdk), 'jvm.cfg')
276
277 def _jdksDir():
278 return os.path.abspath(join(_installed_jdks if _installed_jdks else _graal_home, 'jdk' + str(mx.java().version)))
279
280 def _handle_missing_VM(bld, vm):
281 mx.log('The ' + bld + ' ' + vm + ' VM has not been created')
282 if sys.stdout.isatty():
283 if mx.ask_yes_no('Build it now', 'y'):
284 with VM(vm, bld):
285 build([])
286 return
287 mx.abort('You need to run "mx --vm ' + vm + ' --vmbuild ' + bld + ' build" to build the selected VM')
288
289 def _jdk(build='product', vmToCheck=None, create=False, installGraalJar=True):
290 """
291 Get the JDK into which Graal is installed, creating it first if necessary.
292 """
293 jdk = join(_jdksDir(), build)
294 if create:
295 srcJdk = mx.java().jdk
296 jdkContents = ['bin', 'include', 'jre', 'lib']
297 if exists(join(srcJdk, 'db')):
298 jdkContents.append('db')
299 if mx.get_os() != 'windows' and exists(join(srcJdk, 'man')):
300 jdkContents.append('man')
301 if not exists(jdk):
302 mx.log('Creating ' + jdk + ' from ' + srcJdk)
303 os.makedirs(jdk)
304 for d in jdkContents:
305 src = join(srcJdk, d)
306 dst = join(jdk, d)
307 if not exists(src):
308 mx.abort('Host JDK directory is missing: ' + src)
309 shutil.copytree(src, dst)
310
311 # Make a copy of the default VM so that this JDK can be
312 # reliably used as the bootstrap for a HotSpot build.
313 jvmCfg = _vmCfgInJdk(jdk)
314 if not exists(jvmCfg):
315 mx.abort(jvmCfg + ' does not exist')
316
317 defaultVM = None
318 jvmCfgLines = []
319 with open(jvmCfg) as f:
320 for line in f:
321 if line.startswith('-') and defaultVM is None:
322 parts = line.split()
323 if len(parts) == 2:
324 assert parts[1] == 'KNOWN', parts[1]
325 defaultVM = parts[0][1:]
326 jvmCfgLines += ['# default VM is a copy of the unmodified ' + defaultVM + ' VM\n']
327 jvmCfgLines += ['-original KNOWN\n']
328 else:
329 # skip lines which we cannot parse (e.g. '-hotspot ALIASED_TO -client')
330 mx.log("WARNING: skipping not parsable line \"" + line + "\"")
331 else:
332 jvmCfgLines += [line]
333
334 assert defaultVM is not None, 'Could not find default VM in ' + jvmCfg
335 if mx.get_os() != 'windows':
336 chmodRecursive(jdk, 0755)
337 shutil.move(join(_vmLibDirInJdk(jdk), defaultVM), join(_vmLibDirInJdk(jdk), 'original'))
338
339
340 with open(jvmCfg, 'w') as fp:
341 for line in jvmCfgLines:
342 fp.write(line)
343
344 # Install a copy of the disassembler library
345 try:
346 hsdis([], copyToDir=_vmLibDirInJdk(jdk))
347 except SystemExit:
348 pass
349 else:
350 if not exists(jdk):
351 if _installed_jdks:
352 mx.log("The selected JDK directory does not (yet) exist: " + jdk)
353 _handle_missing_VM(build, vmToCheck if vmToCheck else 'graal')
354
355 if installGraalJar:
356 _installGraalJarInJdks(mx.distribution('GRAAL'))
357
358 if vmToCheck is not None:
359 jvmCfg = _vmCfgInJdk(jdk)
360 found = False
361 with open(jvmCfg) as f:
362 for line in f:
363 if line.strip() == '-' + vmToCheck + ' KNOWN':
364 found = True
365 break
366 if not found:
367 _handle_missing_VM(build, vmToCheck)
368
369 return jdk
370
371 def _updateInstalledGraalOptionsFile(jdk):
372 graalOptions = join(_graal_home, 'graal.options')
373 jreLibDir = join(jdk, 'jre', 'lib')
374 if exists(graalOptions):
375 shutil.copy(graalOptions, join(jreLibDir, 'graal.options'))
376 else:
377 toDelete = join(jreLibDir, 'graal.options')
378 if exists(toDelete):
379 os.unlink(toDelete)
380
381 def _installGraalJarInJdks(graalDist):
382 graalJar = graalDist.path
383 jdks = _jdksDir()
384 if exists(jdks):
385 for e in os.listdir(jdks):
386 jreLibDir = join(jdks, e, 'jre', 'lib')
387 if exists(jreLibDir):
388 # do a copy and then a move to get atomic updating (on Unix) of graal.jar in the JRE
389 fd, tmp = tempfile.mkstemp(suffix='', prefix='graal.jar', dir=jreLibDir)
390 shutil.copyfile(graalJar, tmp)
391 os.close(fd)
392 shutil.move(tmp, join(jreLibDir, 'graal.jar'))
393
394 # run a command in the windows SDK Debug Shell
395 def _runInDebugShell(cmd, workingDir, logFile=None, findInOutput=None, respondTo=None):
396 if respondTo is None:
397 respondTo = {}
398 newLine = os.linesep
399 startToken = 'RUNINDEBUGSHELL_STARTSEQUENCE'
400 endToken = 'RUNINDEBUGSHELL_ENDSEQUENCE'
401
402 winSDK = mx.get_env('WIN_SDK', 'C:\\Program Files\\Microsoft SDKs\\Windows\\v7.1\\')
403
404 if not exists(winSDK):
405 mx.abort("Could not find Windows SDK : '" + winSDK + "' does not exist")
406
407 if not exists(join(winSDK, 'Bin', 'SetEnv.cmd')):
408 mx.abort("Invalid Windows SDK path (" + winSDK + ") : could not find Bin/SetEnv.cmd (you can use the WIN_SDK environment variable to specify an other path)")
409
410 p = subprocess.Popen('cmd.exe /E:ON /V:ON /K ""' + winSDK + '/Bin/SetEnv.cmd" & echo ' + startToken + '"', \
411 shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, creationflags=subprocess.CREATE_NEW_PROCESS_GROUP)
412 stdout = p.stdout
413 stdin = p.stdin
414 if logFile:
415 log = open(logFile, 'w')
416 ret = False
417 while True:
418
419 # encoding may be None on windows plattforms
420 if sys.stdout.encoding is None:
421 encoding = 'utf-8'
422 else:
423 encoding = sys.stdout.encoding
424
425 line = stdout.readline().decode(encoding)
426 if logFile:
427 log.write(line.encode('utf-8'))
428 line = line.strip()
429 mx.log(line)
430 if line == startToken:
431 stdin.write('cd /D ' + workingDir + ' & ' + cmd + ' & echo ' + endToken + newLine)
432 for regex in respondTo.keys():
433 match = regex.search(line)
434 if match:
435 stdin.write(respondTo[regex] + newLine)
436 if findInOutput:
437 match = findInOutput.search(line)
438 if match:
439 ret = True
440 if line == endToken:
441 if not findInOutput:
442 stdin.write('echo ERRXXX%errorlevel%' + newLine)
443 else:
444 break
445 if line.startswith('ERRXXX'):
446 if line == 'ERRXXX0':
447 ret = True
448 break
449 stdin.write('exit' + newLine)
450 if logFile:
451 log.close()
452 return ret
453
454 def jdkhome(vm=None):
455 """return the JDK directory selected for the 'vm' command"""
456 build = _vmbuild if _vmSourcesAvailable else 'product'
457 return _jdk(build, installGraalJar=False)
458
459 def print_jdkhome(args, vm=None):
460 """print the JDK directory selected for the 'vm' command"""
461 print jdkhome(vm)
462
463 def buildvars(args):
464 """describe the variables that can be set by the -D option to the 'mx build' commmand"""
465
466 buildVars = {
467 'ALT_BOOTDIR' : 'The location of the bootstrap JDK installation (default: ' + mx.java().jdk + ')',
468 'ALT_OUTPUTDIR' : 'Build directory',
469 'HOTSPOT_BUILD_JOBS' : 'Number of CPUs used by make (default: ' + str(multiprocessing.cpu_count()) + ')',
470 'INSTALL' : 'Install the built VM into the JDK? (default: y)',
471 'ZIP_DEBUGINFO_FILES' : 'Install zipped debug symbols file? (default: 0)',
472 }
473
474 mx.log('HotSpot build variables that can be set by the -D option to "mx build":')
475 mx.log('')
476 for n in sorted(buildVars.iterkeys()):
477 mx.log(n)
478 mx.log(textwrap.fill(buildVars[n], initial_indent=' ', subsequent_indent=' ', width=200))
479
480 mx.log('')
481 mx.log('Note that these variables can be given persistent values in the file ' + join(_graal_home, 'mx', 'env') + ' (see \'mx about\').')
482
483 def build(args, vm=None):
484 """build the VM binary
485
486 The global '--vm' and '--vmbuild' options select which VM type and build target to build."""
487
488 # Override to fail quickly if extra arguments are given
489 # at the end of the command line. This allows for a more
490 # helpful error message.
491 class AP(ArgumentParser):
492 def __init__(self):
493 ArgumentParser.__init__(self, prog='mx build')
494 def parse_args(self, args):
495 result = ArgumentParser.parse_args(self, args)
496 if len(result.remainder) != 0:
497 firstBuildTarget = result.remainder[0]
498 mx.abort('To specify the ' + firstBuildTarget + ' VM build target, you need to use the global "--vmbuild" option. For example:\n' +
499 ' mx --vmbuild ' + firstBuildTarget + ' build')
500 return result
501
502 # Call mx.build to compile the Java sources
503 parser = AP()
504 parser.add_argument('--export-dir', help='directory to which graal.jar and graal.options will be copied', metavar='<path>')
505 parser.add_argument('-D', action='append', help='set a HotSpot build variable (run \'mx buildvars\' to list variables)', metavar='name=value')
506 opts2 = mx.build(['--source', '1.7'] + args, parser=parser)
507 assert len(opts2.remainder) == 0
508
509 if opts2.export_dir is not None:
510 if not exists(opts2.export_dir):
511 os.makedirs(opts2.export_dir)
512 else:
513 assert os.path.isdir(opts2.export_dir), '{} is not a directory'.format(opts2.export_dir)
514
515 shutil.copy(mx.distribution('GRAAL').path, opts2.export_dir)
516 graalOptions = join(_graal_home, 'graal.options')
517 if exists(graalOptions):
518 shutil.copy(graalOptions, opts2.export_dir)
519
520 if not _vmSourcesAvailable or not opts2.native:
521 return
522
523 builds = [_vmbuild]
524
525 if vm is None:
526 vm = _get_vm()
527
528 if vm == 'original':
529 pass
530 elif vm.startswith('server'):
531 buildSuffix = ''
532 elif vm.startswith('client'):
533 buildSuffix = '1'
534 else:
535 assert vm == 'graal', vm
536 buildSuffix = 'graal'
537
538 if _installed_jdks and _installed_jdks != _graal_home:
539 if not mx.ask_yes_no("Warning: building while --installed-jdks is set (" + _installed_jdks + ") is not recommanded - are you sure you want to continue", 'n'):
540 mx.abort(1)
541
542 for build in builds:
543 if build == 'ide-build-target':
544 build = os.environ.get('IDE_BUILD_TARGET', None)
545 if build is None or len(build) == 0:
546 continue
547
548 jdk = _jdk(build, create=True)
549
550 if vm == 'original':
551 if build != 'product':
552 mx.log('only product build of original VM exists')
553 continue
554
555 vmDir = join(_vmLibDirInJdk(jdk), vm)
556 if not exists(vmDir):
557 if mx.get_os() != 'windows':
558 chmodRecursive(jdk, 0755)
559 mx.log('Creating VM directory in JDK7: ' + vmDir)
560 os.makedirs(vmDir)
561
562 def filterXusage(line):
563 if not 'Xusage.txt' in line:
564 sys.stderr.write(line + os.linesep)
565
566 # Check if a build really needs to be done
567 timestampFile = join(vmDir, '.build-timestamp')
568 if opts2.force or not exists(timestampFile):
569 mustBuild = True
570 else:
571 mustBuild = False
572 timestamp = os.path.getmtime(timestampFile)
573 sources = []
574 for d in ['src', 'make']:
575 for root, dirnames, files in os.walk(join(_graal_home, d)):
576 # ignore <graal>/src/share/tools
577 if root == join(_graal_home, 'src', 'share'):
578 dirnames.remove('tools')
579 sources += [join(root, name) for name in files]
580 for f in sources:
581 if len(f) != 0 and os.path.getmtime(f) > timestamp:
582 mustBuild = True
583 break
584
585 if not mustBuild:
586 mx.logv('[all files in src and make directories are older than ' + timestampFile[len(_graal_home) + 1:] + ' - skipping native build]')
587 continue
588
589 if platform.system() == 'Windows':
590 compilelogfile = _graal_home + '/graalCompile.log'
591 mksHome = mx.get_env('MKS_HOME', 'C:\\cygwin\\bin')
592
593 variant = {'client': 'compiler1', 'server': 'compiler2'}.get(vm, vm)
594 project_config = variant + '_' + build
595 _runInDebugShell('msbuild ' + _graal_home + r'\build\vs-amd64\jvm.vcproj /p:Configuration=' + project_config + ' /target:clean', _graal_home)
596 winCompileCmd = r'set HotSpotMksHome=' + mksHome + r'& set OUT_DIR=' + jdk + r'& set JAVA_HOME=' + jdk + r'& set path=%JAVA_HOME%\bin;%path%;%HotSpotMksHome%& cd /D "' + _graal_home + r'\make\windows"& call create.bat ' + _graal_home
597 print(winCompileCmd)
598 winCompileSuccess = re.compile(r"^Writing \.vcxproj file:")
599 if not _runInDebugShell(winCompileCmd, _graal_home, compilelogfile, winCompileSuccess):
600 mx.log('Error executing create command')
601 return
602 winBuildCmd = 'msbuild ' + _graal_home + r'\build\vs-amd64\jvm.vcxproj /p:Configuration=' + project_config + ' /p:Platform=x64'
603 if not _runInDebugShell(winBuildCmd, _graal_home, compilelogfile):
604 mx.log('Error building project')
605 return
606 else:
607 cpus = multiprocessing.cpu_count()
608 runCmd = [mx.gmake_cmd()]
609 runCmd.append(build + buildSuffix)
610 env = os.environ.copy()
611
612 if opts2.D:
613 for nv in opts2.D:
614 name, value = nv.split('=', 1)
615 env[name.strip()] = value
616
617 env.setdefault('ARCH_DATA_MODEL', '64')
618 env.setdefault('LANG', 'C')
619 env.setdefault('HOTSPOT_BUILD_JOBS', str(cpus))
620 env.setdefault('ALT_BOOTDIR', mx.java().jdk)
621 if not mx._opts.verbose:
622 runCmd.append('MAKE_VERBOSE=')
623 env['JAVA_HOME'] = jdk
624 if vm.endswith('nograal'):
625 env['INCLUDE_GRAAL'] = 'false'
626 env.setdefault('ALT_OUTPUTDIR', join(_graal_home, 'build-nograal', mx.get_os()))
627 else:
628 env['INCLUDE_GRAAL'] = 'true'
629 env.setdefault('INSTALL', 'y')
630 if mx.get_os() == 'solaris' :
631 # If using sparcWorks, setup flags to avoid make complaining about CC version
632 cCompilerVersion = subprocess.Popen('CC -V', stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True).stderr.readlines()[0]
633 if cCompilerVersion.startswith('CC: Sun C++') :
634 compilerRev = cCompilerVersion.split(' ')[3]
635 env.setdefault('ENFORCE_COMPILER_REV', compilerRev)
636 env.setdefault('ENFORCE_CC_COMPILER_REV', compilerRev)
637 if build == 'jvmg':
638 # I want ALL the symbols when I'm debugging on Solaris
639 # Some Makefile variable are overloaded by environment variable so we need to explicitely
640 # pass them down in the command line. This one is an example of that.
641 runCmd.append('STRIP_POLICY=no_strip')
642 # This removes the need to unzip the *.diz files before debugging in gdb
643 env.setdefault('ZIP_DEBUGINFO_FILES', '0')
644
645 # Clear these 2 variables as having them set can cause very confusing build problems
646 env.pop('LD_LIBRARY_PATH', None)
647 env.pop('CLASSPATH', None)
648
649 mx.run(runCmd, cwd=join(_graal_home, 'make'), err=filterXusage, env=env)
650
651 jvmCfg = _vmCfgInJdk(jdk)
652 if not exists(jvmCfg):
653 mx.abort(jvmCfg + ' does not exist')
654
655 prefix = '-' + vm + ' '
656 vmKnown = prefix + 'KNOWN\n'
657 lines = []
658 found = False
659 with open(jvmCfg) as f:
660 for line in f:
661 if line.strip() == vmKnown.strip():
662 found = True
663 lines.append(line)
664
665 if not found:
666 mx.log('Appending "' + prefix + 'KNOWN" to ' + jvmCfg)
667 if mx.get_os() != 'windows':
668 os.chmod(jvmCfg, 0755)
669 with open(jvmCfg, 'w') as f:
670 for line in lines:
671 if line.startswith(prefix):
672 line = vmKnown
673 found = True
674 f.write(line)
675 if not found:
676 f.write(vmKnown)
677
678 if exists(timestampFile):
679 os.utime(timestampFile, None)
680 else:
681 file(timestampFile, 'a')
682
683 def vmg(args):
684 """run the debug build of VM selected by the '--vm' option"""
685 return vm(args, vmbuild='debug')
686
687 def vmfg(args):
688 """run the fastdebug build of VM selected by the '--vm' option"""
689 return vm(args, vmbuild='fastdebug')
690
691 def vm(args, vm=None, nonZeroIsFatal=True, out=None, err=None, cwd=None, timeout=None, vmbuild=None):
692 """run the VM selected by the '--vm' option"""
693
694 if vm is None:
695 vm = _get_vm()
696
697 if cwd is None:
698 cwd = _vm_cwd
699 elif _vm_cwd is not None and _vm_cwd != cwd:
700 mx.abort("conflicting working directories: do not set --vmcwd for this command")
701
702 build = vmbuild if vmbuild is not None else _vmbuild if _vmSourcesAvailable else 'product'
703 jdk = _jdk(build, vmToCheck=vm, installGraalJar=False)
704 _updateInstalledGraalOptionsFile(jdk)
705 mx.expand_project_in_args(args)
706 if _make_eclipse_launch:
707 mx.make_eclipse_launch(args, 'graal-' + build, name=None, deps=mx.project('com.oracle.graal.hotspot').all_deps([], True))
708 if len([a for a in args if 'PrintAssembly' in a]) != 0:
709 hsdis([], copyToDir=_vmLibDirInJdk(jdk))
710 if mx.java().debug_port is not None:
711 args = ['-Xdebug', '-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=' + str(mx.java().debug_port)] + args
712 if _jacoco == 'on' or _jacoco == 'append':
713 jacocoagent = mx.library("JACOCOAGENT", True)
714 # Exclude all compiler tests and snippets
715 excludes = ['com.oracle.graal.compiler.tests.*', 'com.oracle.graal.jtt.*']
716 for p in mx.projects():
717 excludes += _find_classes_with_annotations(p, None, ['@Snippet', '@ClassSubstitution', '@Test'], includeInnerClasses=True).keys()
718 excludes += p.find_classes_with_matching_source_line(None, lambda line: 'JaCoCo Exclude' in line, includeInnerClasses=True).keys()
719
720 includes = ['com.oracle.graal.*']
721 agentOptions = {
722 'append' : 'true' if _jacoco == 'append' else 'false',
723 'bootclasspath' : 'true',
724 'includes' : ':'.join(includes),
725 'excludes' : ':'.join(excludes),
726 'destfile' : 'jacoco.exec'
727 }
728 args = ['-javaagent:' + jacocoagent.get_path(True) + '=' + ','.join([k + '=' + v for k, v in agentOptions.items()])] + args
729 if '-d64' not in args:
730 args = ['-d64'] + args
731
732 exe = join(jdk, 'bin', mx.exe_suffix('java'))
733 pfx = _vm_prefix.split() if _vm_prefix is not None else []
734
735 if '-version' in args:
736 ignoredArgs = args[args.index('-version') + 1:]
737 if len(ignoredArgs) > 0:
738 mx.log("Warning: The following options will be ignored by the vm because they come after the '-version' argument: " + ' '.join(ignoredArgs))
739
740 return mx.run(pfx + [exe, '-' + vm] + args, nonZeroIsFatal=nonZeroIsFatal, out=out, err=err, cwd=cwd, timeout=timeout)
741
742 def _find_classes_with_annotations(p, pkgRoot, annotations, includeInnerClasses=False):
743 """
744 Scan the sources of project 'p' for Java source files containing a line starting with 'annotation'
745 (ignoring preceding whitespace) and return the fully qualified class name for each Java
746 source file matched in a list.
747 """
748
749 matches = lambda line : len([a for a in annotations if line == a or line.startswith(a + '(')]) != 0
750 return p.find_classes_with_matching_source_line(pkgRoot, matches, includeInnerClasses)
751
752 def _extract_VM_args(args, allowClasspath=False, useDoubleDash=False):
753 """
754 Partitions a command line into a leading sequence of HotSpot VM options and the rest.
755 """
756 for i in range(0, len(args)):
757 if useDoubleDash:
758 if args[i] == '--':
759 vmArgs = args[:i]
760 remainder = args[i + 1:]
761 return vmArgs, remainder
762 else:
763 if not args[i].startswith('-'):
764 if i != 0 and (args[i - 1] == '-cp' or args[i - 1] == '-classpath'):
765 if not allowClasspath:
766 mx.abort('Cannot supply explicit class path option')
767 else:
768 continue
769 vmArgs = args[:i]
770 remainder = args[i:]
771 return vmArgs, remainder
772
773 return args, []
774
775 def _run_tests(args, harness, annotations, testfile):
776
777
778 vmArgs, tests = _extract_VM_args(args)
779 for t in tests:
780 if t.startswith('-'):
781 mx.abort('VM option ' + t + ' must precede ' + tests[0])
782
783 candidates = []
784 for p in mx.projects_opt_limit_to_suites():
785 if mx.java().javaCompliance < p.javaCompliance:
786 continue
787 candidates += _find_classes_with_annotations(p, None, annotations).keys()
788
789 classes = []
790 if len(tests) == 0:
791 classes = candidates
792 else:
793 for t in tests:
794 found = False
795 for c in candidates:
796 if t in c:
797 found = True
798 classes.append(c)
799 if not found:
800 mx.log('warning: no tests matched by substring "' + t)
801
802 projectscp = mx.classpath([pcp.name for pcp in mx.projects_opt_limit_to_suites() if pcp.javaCompliance <= mx.java().javaCompliance])
803
804 if len(classes) != 0:
805 f_testfile = open(testfile, 'w')
806 for c in classes:
807 f_testfile.write(c + '\n')
808 f_testfile.close()
809 harness(projectscp, vmArgs)
810
811 def _unittest(args, annotations):
812 mxdir = dirname(__file__)
813 name = 'JUnitWrapper'
814 javaSource = join(mxdir, name + '.java')
815 javaClass = join(mxdir, name + '.class')
816 testfile = os.environ.get('MX_TESTFILE', None)
817 if testfile is None:
818 (_, testfile) = tempfile.mkstemp(".testclasses", "graal")
819 os.close(_)
820
821 def harness(projectscp, vmArgs):
822 if not exists(javaClass) or getmtime(javaClass) < getmtime(javaSource):
823 subprocess.check_call([mx.java().javac, '-cp', projectscp, '-d', mxdir, javaSource])
824 if _get_vm() != 'graal':
825 prefixArgs = ['-esa', '-ea']
826 else:
827 prefixArgs = ['-XX:-BootstrapGraal', '-esa', '-ea']
828 with open(testfile) as fp:
829 testclasses = [l.rstrip() for l in fp.readlines()]
830 if len(testclasses) == 1:
831 # Execute Junit directly when one test is being run. This simplifies
832 # replaying the VM execution in a native debugger (e.g., gdb).
833 vm(prefixArgs + vmArgs + ['-cp', projectscp, 'org.junit.runner.JUnitCore'] + testclasses)
834 else:
835 vm(prefixArgs + vmArgs + ['-cp', projectscp + os.pathsep + mxdir, name] + [testfile])
836
837 try:
838 _run_tests(args, harness, annotations, testfile)
839 finally:
840 if os.environ.get('MX_TESTFILE') is None:
841 os.remove(testfile)
842
843 _unittestHelpSuffix = """
844
845 If filters are supplied, only tests whose fully qualified name
846 includes a filter as a substring are run.
847
848 For example, this command line:
849
850 mx unittest -G:Dump= -G:MethodFilter=BC_aload.* -G:+PrintCFG BC_aload
851
852 will run all JUnit test classes that contain 'BC_aload' in their
853 fully qualified name and will pass these options to the VM:
854
855 -G:Dump= -G:MethodFilter=BC_aload.* -G:+PrintCFG
856
857 To get around command line length limitations on some OSes, the
858 JUnit class names to be executed are written to a file that a
859 custom JUnit wrapper reads and passes onto JUnit proper. The
860 MX_TESTFILE environment variable can be set to specify a
861 file which will not be deleted once the unittests are done
862 (unlike the temporary file otherwise used).
863
864 As with all other commands, using the global '-v' before 'unittest'
865 command will cause mx to show the complete command line
866 it uses to run the VM.
867 """
868
869 def unittest(args):
870 """run the JUnit tests (all testcases){0}"""
871
872 _unittest(args, ['@Test', '@LongTest', '@Parameters'])
873
874 def shortunittest(args):
875 """run the JUnit tests (short testcases only){0}"""
876
877 _unittest(args, ['@Test'])
878
879 def longunittest(args):
880 """run the JUnit tests (long testcases only){0}"""
881
882 _unittest(args, ['@LongTest', '@Parameters'])
883
884 def buildvms(args):
885 """build one or more VMs in various configurations"""
886
887 vmsDefault = ','.join(_vmChoices.keys())
888 vmbuildsDefault = ','.join(_vmbuildChoices)
889
890 parser = ArgumentParser(prog='mx buildvms')
891 parser.add_argument('--vms', help='a comma separated list of VMs to build (default: ' + vmsDefault + ')', metavar='<args>', default=vmsDefault)
892 parser.add_argument('--builds', help='a comma separated list of build types (default: ' + vmbuildsDefault + ')', metavar='<args>', default=vmbuildsDefault)
893 parser.add_argument('-n', '--no-check', action='store_true', help='omit running "java -version" after each build')
894 parser.add_argument('-c', '--console', action='store_true', help='send build output to console instead of log file')
895
896 args = parser.parse_args(args)
897 vms = args.vms.split(',')
898 builds = args.builds.split(',')
899
900 allStart = time.time()
901 for v in vms:
902 for vmbuild in builds:
903 if v == 'original' and vmbuild != 'product':
904 continue
905 if not args.console:
906 logFile = join(v + '-' + vmbuild + '.log')
907 log = open(join(_graal_home, logFile), 'wb')
908 start = time.time()
909 mx.log('BEGIN: ' + v + '-' + vmbuild + '\t(see: ' + logFile + ')')
910 # Run as subprocess so that output can be directed to a file
911 subprocess.check_call([sys.executable, '-u', join('mxtool', 'mx.py'), '--vm', v, '--vmbuild',
912 vmbuild, 'build'], cwd=_graal_home, stdout=log, stderr=subprocess.STDOUT)
913 duration = datetime.timedelta(seconds=time.time() - start)
914 mx.log('END: ' + v + '-' + vmbuild + '\t[' + str(duration) + ']')
915 else:
916 with VM(v, vmbuild):
917 build([])
918 if not args.no_check:
919 vmargs = ['-version']
920 if v == 'graal':
921 vmargs.insert(0, '-XX:-BootstrapGraal')
922 vm(vmargs, vm=v, vmbuild=vmbuild)
923 allDuration = datetime.timedelta(seconds=time.time() - allStart)
924 mx.log('TOTAL TIME: ' + '[' + str(allDuration) + ']')
925
926 class Task:
927 def __init__(self, title):
928 self.start = time.time()
929 self.title = title
930 self.end = None
931 self.duration = None
932 mx.log(time.strftime('gate: %d %b %Y %H:%M:%S: BEGIN: ') + title)
933 def stop(self):
934 self.end = time.time()
935 self.duration = datetime.timedelta(seconds=self.end - self.start)
936 mx.log(time.strftime('gate: %d %b %Y %H:%M:%S: END: ') + self.title + ' [' + str(self.duration) + ']')
937 return self
938 def abort(self, codeOrMessage):
939 self.end = time.time()
940 self.duration = datetime.timedelta(seconds=self.end - self.start)
941 mx.log(time.strftime('gate: %d %b %Y %H:%M:%S: ABORT: ') + self.title + ' [' + str(self.duration) + ']')
942 mx.abort(codeOrMessage)
943 return self
944
945 def _basic_gate_body(args, tasks):
946 t = Task('BuildHotSpotGraal: fastdebug,product')
947 buildvms(['--vms', 'graal,server', '--builds', 'fastdebug,product'])
948 tasks.append(t.stop())
949
950 with VM('graal', 'fastdebug'):
951 t = Task('BootstrapWithSystemAssertions:fastdebug')
952 vm(['-esa', '-version'])
953 tasks.append(t.stop())
954
955 with VM('graal', 'product'):
956 t = Task('BootstrapWithGCVerification:product')
957 out = mx.DuplicateSuppressingStream(['VerifyAfterGC:', 'VerifyBeforeGC:']).write
958 vm(['-XX:+UnlockDiagnosticVMOptions', '-XX:+VerifyBeforeGC', '-XX:+VerifyAfterGC', '-version'], out=out)
959 tasks.append(t.stop())
960
961 with VM('graal', 'product'):
962 t = Task('BootstrapWithG1GCVerification:product')
963 out = mx.DuplicateSuppressingStream(['VerifyAfterGC:', 'VerifyBeforeGC:']).write
964 vm(['-XX:+UnlockDiagnosticVMOptions', '-XX:-UseSerialGC', '-XX:+UseG1GC', '-XX:+VerifyBeforeGC', '-XX:+VerifyAfterGC', '-version'], out=out)
965 tasks.append(t.stop())
966
967 with VM('graal', 'product'):
968 t = Task('BootstrapWithRegisterPressure:product')
969 vm(['-G:RegisterPressure=rbx,r11,r10,r14,xmm3,xmm11,xmm14', '-esa', '-version'])
970 tasks.append(t.stop())
971
972 with VM('graal', 'product'):
973 t = Task('BootstrapWithAOTConfiguration:product')
974 vm(['-G:+AOTCompilation', '-G:+VerifyPhases', '-esa', '-version'])
975 tasks.append(t.stop())
976
977 with VM('server', 'product'): # hosted mode
978 t = Task('UnitTests:hosted-product')
979 unittest([])
980 tasks.append(t.stop())
981
982 for vmbuild in ['fastdebug', 'product']:
983 for test in sanitycheck.getDacapos(level=sanitycheck.SanityCheckLevel.Gate, gateBuildLevel=vmbuild):
984 t = Task(str(test) + ':' + vmbuild)
985 if not test.test('graal'):
986 t.abort(test.name + ' Failed')
987 tasks.append(t.stop())
988
989 if args.jacocout is not None:
990 jacocoreport([args.jacocout])
991
992 global _jacoco
993 _jacoco = 'off'
994
995 t = Task('CleanAndBuildIdealGraphVisualizer')
996 mx.run(['ant', '-f', join(_graal_home, 'src', 'share', 'tools', 'IdealGraphVisualizer', 'build.xml'), '-q', 'clean', 'build'])
997 tasks.append(t.stop())
998
999 # Prevent Graal modifications from breaking the standard builds
1000 if args.buildNonGraal:
1001 t = Task('BuildHotSpotVarieties')
1002 buildvms(['--vms', 'client,server', '--builds', 'fastdebug,product'])
1003 buildvms(['--vms', 'server-nograal', '--builds', 'product'])
1004 buildvms(['--vms', 'server-nograal', '--builds', 'optimized'])
1005 tasks.append(t.stop())
1006
1007 for vmbuild in ['product', 'fastdebug']:
1008 for theVm in ['client', 'server']:
1009 with VM(theVm, vmbuild):
1010 t = Task('DaCapo_pmd:' + theVm + ':' + vmbuild)
1011 dacapo(['pmd'])
1012 tasks.append(t.stop())
1013
1014 t = Task('UnitTests:' + theVm + ':' + vmbuild)
1015 unittest(['-XX:CompileCommand=exclude,*::run*', 'graal.api'])
1016 tasks.append(t.stop())
1017
1018
1019 def gate(args, gate_body=_basic_gate_body):
1020 """run the tests used to validate a push
1021
1022 If this command exits with a 0 exit code, then the source code is in
1023 a state that would be accepted for integration into the main repository."""
1024
1025 parser = ArgumentParser(prog='mx gate')
1026 parser.add_argument('-j', '--omit-java-clean', action='store_false', dest='cleanJava', help='omit cleaning Java native code')
1027 parser.add_argument('-n', '--omit-native-clean', action='store_false', dest='cleanNative', help='omit cleaning and building native code')
1028 parser.add_argument('-g', '--only-build-graalvm', action='store_false', dest='buildNonGraal', help='only build the Graal VM')
1029 parser.add_argument('--jacocout', help='specify the output directory for jacoco report')
1030
1031 args = parser.parse_args(args)
1032
1033 global _jacoco
1034
1035 tasks = []
1036 total = Task('Gate')
1037 try:
1038
1039 t = Task('Pylint')
1040 mx.pylint([])
1041 tasks.append(t.stop())
1042
1043 t = Task('Clean')
1044 cleanArgs = []
1045 if not args.cleanNative:
1046 cleanArgs.append('--no-native')
1047 if not args.cleanJava:
1048 cleanArgs.append('--no-java')
1049 clean(cleanArgs)
1050 tasks.append(t.stop())
1051
1052 t = Task('IDEConfigCheck')
1053 mx.ideclean([])
1054 mx.ideinit([])
1055 tasks.append(t.stop())
1056
1057 eclipse_exe = os.environ.get('ECLIPSE_EXE')
1058 if eclipse_exe is not None:
1059 t = Task('CodeFormatCheck')
1060 if mx.eclipseformat(['-e', eclipse_exe]) != 0:
1061 t.abort('Formatter modified files - run "mx eclipseformat", check in changes and repush')
1062 tasks.append(t.stop())
1063
1064 t = Task('Canonicalization Check')
1065 mx.log(time.strftime('%d %b %Y %H:%M:%S - Ensuring mx/projects files are canonicalized...'))
1066 if mx.canonicalizeprojects([]) != 0:
1067 t.abort('Rerun "mx canonicalizeprojects" and check-in the modified mx/projects files.')
1068 tasks.append(t.stop())
1069
1070 t = Task('BuildJava')
1071 build(['--no-native', '--jdt-warning-as-error'])
1072 tasks.append(t.stop())
1073
1074 t = Task('Checkstyle')
1075 if mx.checkstyle([]) != 0:
1076 t.abort('Checkstyle warnings were found')
1077 tasks.append(t.stop())
1078
1079 if exists('jacoco.exec'):
1080 os.unlink('jacoco.exec')
1081
1082 if args.jacocout is not None:
1083 _jacoco = 'append'
1084 else:
1085 _jacoco = 'off'
1086
1087 gate_body(args, tasks)
1088
1089 except KeyboardInterrupt:
1090 total.abort(1)
1091
1092 except BaseException as e:
1093 import traceback
1094 traceback.print_exc()
1095 total.abort(str(e))
1096
1097 total.stop()
1098
1099 mx.log('Gate task times:')
1100 for t in tasks:
1101 mx.log(' ' + str(t.duration) + '\t' + t.title)
1102 mx.log(' =======')
1103 mx.log(' ' + str(total.duration))
1104
1105 def deoptalot(args):
1106 """bootstrap a fastdebug Graal VM with DeoptimizeALot and VerifyOops on
1107
1108 If the first argument is a number, the process will be repeated
1109 this number of times. All other arguments are passed to the VM."""
1110 count = 1
1111 if len(args) > 0 and args[0].isdigit():
1112 count = int(args[0])
1113 del args[0]
1114
1115 for _ in range(count):
1116 if not vm(['-XX:+DeoptimizeALot', '-XX:+VerifyOops'] + args + ['-version'], vmbuild='fastdebug') == 0:
1117 mx.abort("Failed")
1118
1119 def longtests(args):
1120
1121 deoptalot(['15', '-Xmx48m'])
1122
1123 dacapo(['100', 'eclipse', '-esa'])
1124
1125 def gv(args):
1126 """run the Graal Visualizer"""
1127 with open(join(_graal_home, '.graal_visualizer.log'), 'w') as fp:
1128 mx.logv('[Graal Visualizer log is in ' + fp.name + ']')
1129 if not exists(join(_graal_home, 'visualizer', 'build.xml')):
1130 mx.logv('[This initial execution may take a while as the NetBeans platform needs to be downloaded]')
1131 mx.run(['ant', '-f', join(_graal_home, 'visualizer', 'build.xml'), '-l', fp.name, 'run'])
1132
1133 def igv(args):
1134 """run the Ideal Graph Visualizer"""
1135 with open(join(_graal_home, '.ideal_graph_visualizer.log'), 'w') as fp:
1136 mx.logv('[Ideal Graph Visualizer log is in ' + fp.name + ']')
1137 if not exists(join(_graal_home, 'src', 'share', 'tools', 'IdealGraphVisualizer', 'nbplatform')):
1138 mx.logv('[This initial execution may take a while as the NetBeans platform needs to be downloaded]')
1139 mx.run(['ant', '-f', join(_graal_home, 'src', 'share', 'tools', 'IdealGraphVisualizer', 'build.xml'), '-l', fp.name, 'run'])
1140
1141 def bench(args):
1142 """run benchmarks and parse their output for results
1143
1144 Results are JSON formated : {group : {benchmark : score}}."""
1145 resultFile = None
1146 if '-resultfile' in args:
1147 index = args.index('-resultfile')
1148 if index + 1 < len(args):
1149 resultFile = args[index + 1]
1150 del args[index]
1151 del args[index]
1152 else:
1153 mx.abort('-resultfile must be followed by a file name')
1154 vm = _get_vm()
1155 if len(args) is 0:
1156 args = ['all']
1157
1158 vmArgs = [arg for arg in args if arg.startswith('-')]
1159
1160 def benchmarks_in_group(group):
1161 prefix = group + ':'
1162 return [a[len(prefix):] for a in args if a.startswith(prefix)]
1163
1164 results = {}
1165 benchmarks = []
1166 # DaCapo
1167 if ('dacapo' in args or 'all' in args):
1168 benchmarks += sanitycheck.getDacapos(level=sanitycheck.SanityCheckLevel.Benchmark)
1169 else:
1170 dacapos = benchmarks_in_group('dacapo')
1171 for dacapo in dacapos:
1172 if dacapo not in sanitycheck.dacapoSanityWarmup.keys():
1173 mx.abort('Unknown DaCapo : ' + dacapo)
1174 iterations = sanitycheck.dacapoSanityWarmup[dacapo][sanitycheck.SanityCheckLevel.Benchmark]
1175 if (iterations > 0):
1176 benchmarks += [sanitycheck.getDacapo(dacapo, iterations)]
1177
1178 if ('scaladacapo' in args or 'all' in args):
1179 benchmarks += sanitycheck.getScalaDacapos(level=sanitycheck.SanityCheckLevel.Benchmark)
1180 else:
1181 scaladacapos = benchmarks_in_group('scaladacapo')
1182 for scaladacapo in scaladacapos:
1183 if scaladacapo not in sanitycheck.dacapoScalaSanityWarmup.keys():
1184 mx.abort('Unknown Scala DaCapo : ' + scaladacapo)
1185 iterations = sanitycheck.dacapoScalaSanityWarmup[scaladacapo][sanitycheck.SanityCheckLevel.Benchmark]
1186 if (iterations > 0):
1187 benchmarks += [sanitycheck.getScalaDacapo(scaladacapo, ['-n', str(iterations)])]
1188
1189 # Bootstrap
1190 if ('bootstrap' in args or 'all' in args):
1191 benchmarks += sanitycheck.getBootstraps()
1192 # SPECjvm2008
1193 if ('specjvm2008' in args or 'all' in args):
1194 benchmarks += [sanitycheck.getSPECjvm2008(['-ikv', '-wt', '120', '-it', '120'])]
1195 else:
1196 specjvms = benchmarks_in_group('specjvm2008')
1197 for specjvm in specjvms:
1198 benchmarks += [sanitycheck.getSPECjvm2008(['-ikv', '-wt', '120', '-it', '120', specjvm])]
1199
1200 if ('specjbb2005' in args or 'all' in args):
1201 benchmarks += [sanitycheck.getSPECjbb2005()]
1202
1203 if ('specjbb2013' in args): # or 'all' in args //currently not in default set
1204 benchmarks += [sanitycheck.getSPECjbb2013()]
1205
1206 if ('ctw-full' in args):
1207 benchmarks.append(sanitycheck.getCTW(vm, sanitycheck.CTWMode.Full))
1208 if ('ctw-noinline' in args):
1209 benchmarks.append(sanitycheck.getCTW(vm, sanitycheck.CTWMode.NoInline))
1210 if ('ctw-nocomplex' in args):
1211 benchmarks.append(sanitycheck.getCTW(vm, sanitycheck.CTWMode.NoComplex))
1212
1213 for test in benchmarks:
1214 for (groupName, res) in test.bench(vm, extraVmOpts=vmArgs).items():
1215 group = results.setdefault(groupName, {})
1216 group.update(res)
1217 mx.log(json.dumps(results))
1218 if resultFile:
1219 with open(resultFile, 'w') as f:
1220 f.write(json.dumps(results))
1221
1222 def specjvm2008(args):
1223 """run one or more SPECjvm2008 benchmarks"""
1224
1225 def launcher(bm, harnessArgs, extraVmOpts):
1226 return sanitycheck.getSPECjvm2008(harnessArgs + [bm]).bench(_get_vm(), extraVmOpts=extraVmOpts)
1227
1228 availableBenchmarks = set(sanitycheck.specjvm2008Names)
1229 for name in sanitycheck.specjvm2008Names:
1230 parts = name.rsplit('.', 1)
1231 if len(parts) > 1:
1232 assert len(parts) == 2
1233 group = parts[0]
1234 availableBenchmarks.add(group)
1235
1236 _run_benchmark(args, sorted(availableBenchmarks), launcher)
1237
1238 def specjbb2013(args):
1239 """runs the composite SPECjbb2013 benchmark"""
1240
1241 def launcher(bm, harnessArgs, extraVmOpts):
1242 assert bm is None
1243 return sanitycheck.getSPECjbb2013(harnessArgs).bench(_get_vm(), extraVmOpts=extraVmOpts)
1244
1245 _run_benchmark(args, None, launcher)
1246
1247 def specjbb2005(args):
1248 """runs the composite SPECjbb2005 benchmark"""
1249
1250 def launcher(bm, harnessArgs, extraVmOpts):
1251 assert bm is None
1252 return sanitycheck.getSPECjbb2005(harnessArgs).bench(_get_vm(), extraVmOpts=extraVmOpts)
1253
1254 _run_benchmark(args, None, launcher)
1255
1256 def hsdis(args, copyToDir=None):
1257 """download the hsdis library
1258
1259 This is needed to support HotSpot's assembly dumping features.
1260 By default it downloads the Intel syntax version, use the 'att' argument to install AT&T syntax."""
1261 flavor = 'intel'
1262 if 'att' in args:
1263 flavor = 'att'
1264 lib = mx.add_lib_suffix('hsdis-' + _arch())
1265 path = join(_graal_home, 'lib', lib)
1266 if not exists(path):
1267 mx.download(path, ['http://lafo.ssw.uni-linz.ac.at/hsdis/' + flavor + "/" + lib])
1268 if copyToDir is not None and exists(copyToDir):
1269 shutil.copy(path, copyToDir)
1270
1271 def hcfdis(args):
1272 """disassemble HexCodeFiles embedded in text files
1273
1274 Run a tool over the input files to convert all embedded HexCodeFiles
1275 to a disassembled format."""
1276
1277 parser = ArgumentParser(prog='mx hcfdis')
1278 parser.add_argument('-m', '--map', help='address to symbol map applied to disassembler output')
1279 parser.add_argument('files', nargs=REMAINDER, metavar='files...')
1280
1281 args = parser.parse_args(args)
1282
1283 path = join(_graal_home, 'lib', 'hcfdis-1.jar')
1284 if not exists(path):
1285 mx.download(path, ['http://lafo.ssw.uni-linz.ac.at/hcfdis-1.jar'])
1286 mx.run_java(['-jar', path] + args.files)
1287
1288 if args.map is not None:
1289 addressRE = re.compile(r'0[xX]([A-Fa-f0-9]+)')
1290 with open(args.map) as fp:
1291 lines = fp.read().splitlines()
1292 symbols = dict()
1293 for l in lines:
1294 addressAndSymbol = l.split(' ', 1)
1295 if len(addressAndSymbol) == 2:
1296 address, symbol = addressAndSymbol
1297 if address.startswith('0x'):
1298 address = long(address, 16)
1299 symbols[address] = symbol
1300 for f in args.files:
1301 with open(f) as fp:
1302 lines = fp.read().splitlines()
1303 updated = False
1304 for i in range(0, len(lines)):
1305 l = lines[i]
1306 for m in addressRE.finditer(l):
1307 sval = m.group(0)
1308 val = long(sval, 16)
1309 sym = symbols.get(val)
1310 if sym:
1311 l = l.replace(sval, sym)
1312 updated = True
1313 lines[i] = l
1314 if updated:
1315 mx.log('updating ' + f)
1316 with open('new_' + f, "w") as fp:
1317 for l in lines:
1318 print >> fp, l
1319
1320 def jacocoreport(args):
1321 """create a JaCoCo coverage report
1322
1323 Creates the report from the 'jacoco.exec' file in the current directory.
1324 Default output directory is 'coverage', but an alternative can be provided as an argument."""
1325 jacocoreport = mx.library("JACOCOREPORT", True)
1326 out = 'coverage'
1327 if len(args) == 1:
1328 out = args[0]
1329 elif len(args) > 1:
1330 mx.abort('jacocoreport takes only one argument : an output directory')
1331 mx.run_java(['-jar', jacocoreport.get_path(True), '-in', 'jacoco.exec', '-g', join(_graal_home, 'graal'), out])
1332
1333 def sl(args):
1334 """run an SL program"""
1335 vmArgs, slArgs = _extract_VM_args(args)
1336 vm(vmArgs + ['-cp', mx.classpath("com.oracle.truffle.sl"), "com.oracle.truffle.sl.SimpleLanguage"] + slArgs)
1337
1338 def isGraalEnabled(vm):
1339 return vm != 'original' and not vm.endswith('nograal')
1340
1341 def site(args):
1342 """create a website containing javadoc and the project dependency graph"""
1343
1344 return mx.site(['--name', 'Graal',
1345 '--jd', '@-tag', '--jd', '@test:X',
1346 '--jd', '@-tag', '--jd', '@run:X',
1347 '--jd', '@-tag', '--jd', '@bug:X',
1348 '--jd', '@-tag', '--jd', '@summary:X',
1349 '--jd', '@-tag', '--jd', '@vmoption:X',
1350 '--overview', join(_graal_home, 'graal', 'overview.html'),
1351 '--title', 'Graal OpenJDK Project Documentation',
1352 '--dot-output-base', 'projects'] + args)
1353
1354 def mx_init(suite):
1355 commands = {
1356 'build': [build, ''],
1357 'buildvars': [buildvars, ''],
1358 'buildvms': [buildvms, '[-options]'],
1359 'clean': [clean, ''],
1360 'hsdis': [hsdis, '[att]'],
1361 'hcfdis': [hcfdis, ''],
1362 'igv' : [igv, ''],
1363 'jdkhome': [print_jdkhome, ''],
1364 'dacapo': [dacapo, '[VM options] benchmarks...|"all" [DaCapo options]'],
1365 'scaladacapo': [scaladacapo, '[VM options] benchmarks...|"all" [Scala DaCapo options]'],
1366 'specjvm2008': [specjvm2008, '[VM options] benchmarks...|"all" [SPECjvm2008 options]'],
1367 'specjbb2013': [specjbb2013, '[VM options] [-- [SPECjbb2013 options]]'],
1368 'specjbb2005': [specjbb2005, '[VM options] [-- [SPECjbb2005 options]]'],
1369 'gate' : [gate, '[-options]'],
1370 'gv' : [gv, ''],
1371 'bench' : [bench, '[-resultfile file] [all(default)|dacapo|specjvm2008|bootstrap]'],
1372 'unittest' : [unittest, '[VM options] [filters...]', _unittestHelpSuffix],
1373 'longunittest' : [longunittest, '[VM options] [filters...]', _unittestHelpSuffix],
1374 'shortunittest' : [shortunittest, '[VM options] [filters...]', _unittestHelpSuffix],
1375 'jacocoreport' : [jacocoreport, '[output directory]'],
1376 'site' : [site, '[-options]'],
1377 'vm': [vm, '[-options] class [args...]'],
1378 'vmg': [vmg, '[-options] class [args...]'],
1379 'vmfg': [vmfg, '[-options] class [args...]'],
1380 'deoptalot' : [deoptalot, '[n]'],
1381 'longtests' : [longtests, ''],
1382 'sl' : [sl, '[SL args|@VM options]']
1383 }
1384
1385 mx.add_argument('--jacoco', help='instruments com.oracle.* classes using JaCoCo', default='off', choices=['off', 'on', 'append'])
1386 mx.add_argument('--vmcwd', dest='vm_cwd', help='current directory will be changed to <path> before the VM is executed', default=None, metavar='<path>')
1387 mx.add_argument('--installed-jdks', help='the base directory in which the JDKs cloned from $JAVA_HOME exist. ' +
1388 'The VM selected by --vm and --vmbuild options is under this directory (i.e., ' +
1389 join('<path>', '<jdk-version>', '<vmbuild>', 'jre', 'lib', '<vm>', mx.add_lib_prefix(mx.add_lib_suffix('jvm'))) + ')', default=None, metavar='<path>')
1390
1391 if (_vmSourcesAvailable):
1392 mx.add_argument('--vm', action='store', dest='vm', choices=_vmChoices.keys(), help='the VM type to build/run')
1393 mx.add_argument('--vmbuild', action='store', dest='vmbuild', choices=_vmbuildChoices, help='the VM build to build/run (default: ' + _vmbuildChoices[0] + ')')
1394 mx.add_argument('--ecl', action='store_true', dest='make_eclipse_launch', help='create launch configuration for running VM execution(s) in Eclipse')
1395 mx.add_argument('--vmprefix', action='store', dest='vm_prefix', help='prefix for running the VM (e.g. "/usr/bin/gdb --args")', metavar='<prefix>')
1396 mx.add_argument('--gdb', action='store_const', const='/usr/bin/gdb --args', dest='vm_prefix', help='alias for --vmprefix "/usr/bin/gdb --args"')
1397
1398 commands.update({
1399 'export': [export, '[-options] [zipfile]'],
1400 })
1401
1402 mx.update_commands(suite, commands)
1403
1404 def mx_post_parse_cmd_line(opts): #
1405 # TODO _minVersion check could probably be part of a Suite in mx?
1406 if (mx.java().version < _minVersion) :
1407 mx.abort('Requires Java version ' + str(_minVersion) + ' or greater, got version ' + str(mx.java().version))
1408
1409 if (_vmSourcesAvailable):
1410 if hasattr(opts, 'vm') and opts.vm is not None:
1411 global _vm
1412 _vm = opts.vm
1413 if hasattr(opts, 'vmbuild') and opts.vmbuild is not None:
1414 global _vmbuild
1415 _vmbuild = opts.vmbuild
1416 global _make_eclipse_launch
1417 _make_eclipse_launch = getattr(opts, 'make_eclipse_launch', False)
1418 global _jacoco
1419 _jacoco = opts.jacoco
1420 global _vm_cwd
1421 _vm_cwd = opts.vm_cwd
1422 global _installed_jdks
1423 _installed_jdks = opts.installed_jdks
1424 global _vm_prefix
1425 _vm_prefix = opts.vm_prefix
1426
1427 mx.distribution('GRAAL').add_update_listener(_installGraalJarInJdks)