Mercurial > hg > truffle

comparison mx.truffle/mx_truffle.py @ 22016:92168cb0e803

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
removed non-Truffle code from mx.truffle/mx_truffle.py
author Doug Simon <doug.simon@oracle.com>
date Wed, 08 Jul 2015 22:30:27 +0200
parents de52ea7de779
children a4c9f8c89c68
comparison
equal deleted inserted replaced
22015:d3750fae58fe 22016:92168cb0e803
24 # or visit www.oracle.com if you need additional information or have any 24 # or visit www.oracle.com if you need additional information or have any
25 # questions. 25 # questions.
26 # 26 #
27 # ---------------------------------------------------------------------------------------------------- 27 # ----------------------------------------------------------------------------------------------------
28 28
29 import os, stat, errno, sys, shutil, zipfile, tarfile, tempfile, re, time, datetime, platform, subprocess, socket
30 from os.path import join, exists, dirname, basename
31 from argparse import ArgumentParser, RawDescriptionHelpFormatter, REMAINDER
32 import mx 29 import mx
33 import xml.dom.minidom
34 import itertools
35 import json, textwrap
36 import fnmatch
37 30
38 # This works because when mx loads this file, it makes sure __file__ gets an absolute path 31 _suite = mx.suite('truffle')
39 _graal_home = dirname(dirname(__file__))
40
41 """ The VM that will be run by the 'vm' command and built by default by the 'build' command.
42 This can be set via the global '--vm' option or the DEFAULT_VM environment variable.
43 It can also be temporarily set by using of a VM context manager object in a 'with' statement. """
44 _vm = None
45
46 _make_eclipse_launch = False
47
48 _minVersion = mx.VersionSpec('1.7')
49
50 # max version (first _unsupported_ version)
51 _untilVersion = None
52
53 class JDKDeployedDist:
54 def __init__(self, name, isExtension=False, usesJVMCIClassLoader=False, partOfHotSpot=False):
55 self.name = name
56 self.isExtension = isExtension
57 self.usesJVMCIClassLoader = usesJVMCIClassLoader
58 self.partOfHotSpot = partOfHotSpot # true when this distribution is delivered with HotSpot
59
60 _jdkDeployedDists = [
61 JDKDeployedDist('TRUFFLE'),
62 ]
63
64 JDK_UNIX_PERMISSIONS_DIR = 0755
65 JDK_UNIX_PERMISSIONS_FILE = 0644
66 JDK_UNIX_PERMISSIONS_EXEC = 0755
67
68 def isVMSupported(vm):
69 if 'client' == vm and len(platform.mac_ver()[0]) != 0:
70 # Client VM not supported: java launcher on Mac OS X translates '-client' to '-server'
71 return False
72 return True
73
74 def _get_vm():
75 """
76 Gets the configured VM, presenting a dialogue if there is no currently configured VM.
77 """
78 global _vm
79 if _vm:
80 return _vm
81 vm = mx.get_env('DEFAULT_VM')
82 if vm is None:
83 extras = mx.get_env('EXTRA_JAVA_HOMES')
84 if not extras is None:
85 for e in extras.split(':'):
86 vm = e
87 break
88 envPath = join(_graal_home, 'mx', 'env')
89 if vm and 'graal' in vm:
90 if exists(envPath):
91 with open(envPath) as fp:
92 if 'DEFAULT_VM=' + vm in fp.read():
93 mx.log('Please update the DEFAULT_VM value in ' + envPath + ' to replace "graal" with "jvmci"')
94 vm = vm.replace('graal', 'jvmci')
95 if vm is None:
96 mx.abort('Need to specify VM with --vm option or DEFAULT_VM environment variable')
97 _vm = vm
98 return vm
99
100 def chmodRecursive(dirname, chmodFlagsDir):
101 if mx.get_os() == 'windows':
102 return
103
104 def _chmodDir(chmodFlags, dirname, fnames):
105 os.chmod(dirname, chmodFlagsDir)
106
107 os.path.walk(dirname, _chmodDir, chmodFlagsDir)
108
109 def clean(args):
110 """clean the source tree"""
111 opts = mx.clean(args, parser=ArgumentParser(prog='mx clean'))
112
113 if opts.native:
114 def handleRemoveReadonly(func, path, exc):
115 excvalue = exc[1]
116 if mx.get_os() == 'windows' and func in (os.rmdir, os.remove) and excvalue.errno == errno.EACCES:
117 os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # 0777
118 func(path)
119 else:
120 raise
121
122 def rmIfExists(name):
123 if os.path.isdir(name):
124 shutil.rmtree(name, ignore_errors=False, onerror=handleRemoveReadonly)
125 elif os.path.isfile(name):
126 os.unlink(name)
127
128 rmIfExists(join(_graal_home, 'build'))
129 rmIfExists(join(_graal_home, 'build-nojvmci'))
130
131 def export(args):
132 """create archives of builds split by vmbuild and vm"""
133
134 parser = ArgumentParser(prog='mx export')
135 args = parser.parse_args(args)
136
137 # collect data about export
138 infos = dict()
139 infos['timestamp'] = time.time()
140
141 hgcfg = mx.HgConfig()
142 hgcfg.check()
143 infos['revision'] = hgcfg.tip('.') + ('+' if hgcfg.isDirty('.') else '')
144 # TODO: infos['repository']
145
146 infos['jdkversion'] = str(mx.java().version)
147
148 infos['architecture'] = mx.get_arch()
149 infos['platform'] = mx.get_os()
150
151 if mx.get_os != 'windows':
152 pass
153 # infos['ccompiler']
154 # infos['linker']
155
156 infos['hostname'] = socket.gethostname()
157
158 def _writeJson(suffix, properties):
159 d = infos.copy()
160 for k, v in properties.iteritems():
161 assert not d.has_key(k)
162 d[k] = v
163
164 jsonFileName = 'export-' + suffix + '.json'
165 with open(jsonFileName, 'w') as f:
166 print >> f, json.dumps(d)
167 return jsonFileName
168
169
170 def _genFileName(archivtype, middle):
171 idPrefix = infos['revision'] + '_'
172 idSuffix = '.tar.gz'
173 return join(_graal_home, "graalvm_" + archivtype + "_" + idPrefix + middle + idSuffix)
174
175 # graal directory
176 graalDirTarName = _genFileName('classfiles', 'javac')
177 mx.logv("creating graal " + graalDirTarName)
178 with tarfile.open(graalDirTarName, 'w:gz') as tar:
179 for root, _, files in os.walk("graal"):
180 for f in [f for f in files if not f.endswith('.java')]:
181 name = join(root, f)
182 # print name
183 tar.add(name, name)
184
185 n = _writeJson("graal", {'javacompiler' : 'javac'})
186 tar.add(n, n)
187
188
189 def _run_benchmark(args, availableBenchmarks, runBenchmark):
190
191 vmOpts, benchmarksAndOptions = _extract_VM_args(args, useDoubleDash=availableBenchmarks is None)
192
193 if availableBenchmarks is None:
194 harnessArgs = benchmarksAndOptions
195 return runBenchmark(None, harnessArgs, vmOpts)
196
197 if len(benchmarksAndOptions) == 0:
198 mx.abort('at least one benchmark name or "all" must be specified')
199 benchmarks = list(itertools.takewhile(lambda x: not x.startswith('-'), benchmarksAndOptions))
200 harnessArgs = benchmarksAndOptions[len(benchmarks):]
201
202 if 'all' in benchmarks:
203 benchmarks = availableBenchmarks
204 else:
205 for bm in benchmarks:
206 if bm not in availableBenchmarks:
207 mx.abort('unknown benchmark: ' + bm + '\nselect one of: ' + str(availableBenchmarks))
208
209 failed = []
210 for bm in benchmarks:
211 if not runBenchmark(bm, harnessArgs, vmOpts):
212 failed.append(bm)
213
214 if len(failed) != 0:
215 mx.abort('Benchmark failures: ' + str(failed))
216
217 def _vmLibDirInJdk(jdk):
218 """
219 Get the directory within a JDK where the server and client
220 subdirectories are located.
221 """
222 mxos = mx.get_os()
223 if mxos == 'darwin':
224 return join(jdk, 'jre', 'lib')
225 if mxos == 'windows' or mxos == 'cygwin':
226 return join(jdk, 'jre', 'bin')
227 return join(jdk, 'jre', 'lib', mx.get_arch())
228
229 def _vmJliLibDirs(jdk):
230 """
231 Get the directories within a JDK where the jli library designates to.
232 """
233 mxos = mx.get_os()
234 if mxos == 'darwin':
235 return [join(jdk, 'jre', 'lib', 'jli')]
236 if mxos == 'windows' or mxos == 'cygwin':
237 return [join(jdk, 'jre', 'bin'), join(jdk, 'bin')]
238 return [join(jdk, 'jre', 'lib', mx.get_arch(), 'jli'), join(jdk, 'lib', mx.get_arch(), 'jli')]
239
240 def _vmCfgInJdk(jdk, jvmCfgFile='jvm.cfg'):
241 """
242 Get the jvm.cfg file.
243 """
244 mxos = mx.get_os()
245 if mxos == "windows" or mxos == "cygwin":
246 return join(jdk, 'jre', 'lib', mx.get_arch(), jvmCfgFile)
247 return join(_vmLibDirInJdk(jdk), jvmCfgFile)
248
249 def _jdksDir():
250 return os.path.abspath(join(_graal_home, 'jdk' + str(mx.java().version)))
251
252 def _handle_missing_VM(bld, vm=None):
253 if not vm:
254 vm = _get_vm()
255 mx.log('The ' + bld + ' ' + vm + ' VM has not been created')
256 mx.abort('You need to run "mx --javahome ' + vm + ' use the selected VM')
257
258 def _jdk(build=None, vmToCheck=None, create=False, installJars=True):
259 """
260 Get the JDK into which Graal is installed, creating it first if necessary.
261 """
262 jdk = join(_jdksDir(), build)
263 if create:
264 srcJdk = mx.java().jdk
265 if not exists(jdk):
266 mx.log('Creating ' + jdk + ' from ' + srcJdk)
267 shutil.copytree(srcJdk, jdk)
268
269 # Make a copy of the default VM so that this JDK can be
270 # reliably used as the bootstrap for a HotSpot build.
271 jvmCfg = _vmCfgInJdk(jdk)
272 if not exists(jvmCfg):
273 mx.abort(jvmCfg + ' does not exist')
274
275 defaultVM = None
276 jvmCfgLines = []
277 with open(jvmCfg) as f:
278 for line in f:
279 if line.startswith('-') and defaultVM is None:
280 parts = line.split()
281 if len(parts) == 2:
282 assert parts[1] == 'KNOWN', parts[1]
283 defaultVM = parts[0][1:]
284 jvmCfgLines += ['# default VM is a copy of the unmodified ' + defaultVM + ' VM\n']
285 jvmCfgLines += ['-original KNOWN\n']
286 else:
287 # skip lines which we cannot parse (e.g. '-hotspot ALIASED_TO -client')
288 mx.log("WARNING: skipping not parsable line \"" + line + "\"")
289 else:
290 jvmCfgLines += [line]
291
292 assert defaultVM is not None, 'Could not find default VM in ' + jvmCfg
293 chmodRecursive(jdk, JDK_UNIX_PERMISSIONS_DIR)
294 shutil.move(join(_vmLibDirInJdk(jdk), defaultVM), join(_vmLibDirInJdk(jdk), 'original'))
295
296 if mx.get_os() != 'windows':
297 os.chmod(jvmCfg, JDK_UNIX_PERMISSIONS_FILE)
298 with open(jvmCfg, 'w') as fp:
299 for line in jvmCfgLines:
300 fp.write(line)
301
302 # patch 'release' file (append graalvm revision)
303 releaseFile = join(jdk, 'release')
304 if exists(releaseFile):
305 releaseFileLines = []
306 with open(releaseFile) as f:
307 for line in f:
308 releaseFileLines.append(line)
309
310 if mx.get_os() != 'windows':
311 os.chmod(releaseFile, JDK_UNIX_PERMISSIONS_FILE)
312 with open(releaseFile, 'w') as fp:
313 for line in releaseFileLines:
314 if line.startswith("SOURCE="):
315 try:
316 sourceLine = line[0:-2] # remove last char
317 hgcfg = mx.HgConfig()
318 hgcfg.check()
319 revision = hgcfg.tip('.')[:12] # take first 12 chars
320 fp.write(sourceLine + ' graal:' + revision + '\"\n')
321 except:
322 fp.write(line)
323 else:
324 fp.write(line)
325
326 # Install a copy of the disassembler library
327 try:
328 hsdis([], copyToDir=_vmLibDirInJdk(jdk))
329 except SystemExit:
330 pass
331 else:
332 if not exists(jdk):
333 _handle_missing_VM(build, vmToCheck)
334
335 if installJars:
336 for jdkDist in _jdkDeployedDists:
337 dist = mx.distribution(jdkDist.name)
338 if exists(dist.path) and jdkDist.partOfHotSpot:
339 _installDistInJdks(jdkDist)
340
341 return jdk
342
343 def _updateInstalledJVMCIOptionsFile(jdk):
344 jvmciOptions = join(_graal_home, 'jvmci.options')
345 jreLibDir = join(jdk, 'jre', 'lib')
346 if exists(jvmciOptions):
347 shutil.copy(jvmciOptions, join(jreLibDir, 'jvmci.options'))
348 else:
349 toDelete = join(jreLibDir, 'jvmci.options')
350 if exists(toDelete):
351 os.unlink(toDelete)
352
353 def _makeHotspotGeneratedSourcesDir():
354 """
355 Gets the directory containing all the HotSpot sources generated from
356 JVMCI Java sources. This directory will be created if it doesn't yet exist.
357 """
358 hsSrcGenDir = join(mx.project('com.oracle.jvmci.hotspot').source_gen_dir(), 'hotspot')
359 if not exists(hsSrcGenDir):
360 os.makedirs(hsSrcGenDir)
361 return hsSrcGenDir
362
363 def _copyToJdk(src, dst, permissions=JDK_UNIX_PERMISSIONS_FILE):
364 name = os.path.basename(src)
365 dstLib = join(dst, name)
366 if mx.get_env('SYMLINK_GRAAL_JAR', None) == 'true':
367 # Using symlinks is much faster than copying but may
368 # cause issues if the lib is being updated while
369 # the VM is running.
370 if not os.path.islink(dstLib) or not os.path.realpath(dstLib) == src:
371 if exists(dstLib):
372 os.remove(dstLib)
373 os.symlink(src, dstLib)
374 else:
375 # do a copy and then a move to get atomic updating (on Unix)
376 fd, tmp = tempfile.mkstemp(suffix='', prefix=name, dir=dst)
377 shutil.copyfile(src, tmp)
378 os.close(fd)
379 shutil.move(tmp, dstLib)
380 os.chmod(dstLib, permissions)
381
382 def _filterJVMCIServices(servicesMap, classpath):
383 """
384 Filters and returns the names in 'serviceImplNames' that denote
385 types available in 'classpath' implementing or extending
386 com.oracle.jvmci.service.Service.
387 """
388 _, binDir = mx._compile_mx_class('FilterTypes', os.pathsep.join(classpath), myDir=dirname(__file__))
389 serialized = [k + '=' + ','.join(v) for k, v in servicesMap.iteritems()]
390 cmd = [mx.java().java, '-cp', mx._cygpathU2W(os.pathsep.join([binDir] + classpath)), 'FilterTypes', 'com.oracle.jvmci.service.Service'] + serialized
391 serialized = subprocess.check_output(cmd)
392 if len(serialized) == 0:
393 return {}
394 servicesMap = {}
395 for e in serialized.split(' '):
396 k, v = e.split('=')
397 impls = v.split(',')
398 servicesMap[k] = impls
399 return servicesMap
400
401 def _extractJVMCIFiles(jdkJars, jvmciJars, servicesDir, optionsDir, cleanDestination=True):
402 if cleanDestination:
403 if exists(servicesDir):
404 shutil.rmtree(servicesDir)
405 if exists(optionsDir):
406 shutil.rmtree(optionsDir)
407 if not exists(servicesDir):
408 os.makedirs(servicesDir)
409 if not exists(optionsDir):
410 os.makedirs(optionsDir)
411 servicesMap = {}
412 optionsFiles = []
413 for jar in jvmciJars:
414 if os.path.isfile(jar):
415 with zipfile.ZipFile(jar) as zf:
416 for member in zf.namelist():
417 if member.startswith('META-INF/services') and member:
418 serviceName = basename(member)
419 if serviceName == "":
420 continue # Zip files may contain empty entries for directories (jar -cf ... creates such)
421 # we don't handle directories
422 assert serviceName and member == 'META-INF/services/' + serviceName
423 with zf.open(member) as serviceFile:
424 serviceImpls = servicesMap.setdefault(serviceName, [])
425 for line in serviceFile.readlines():
426 line = line.strip()
427 if line:
428 serviceImpls.append(line)
429 elif member.startswith('META-INF/options'):
430 filename = basename(member)
431 if filename == "":
432 continue # Zip files may contain empty entries for directories (jar -cf ... creates such)
433 # we don't handle directories
434 assert filename and member == 'META-INF/options/' + filename
435 targetpath = join(optionsDir, filename)
436 optionsFiles.append(filename)
437 with zf.open(member) as optionsFile, \
438 file(targetpath, "wb") as target:
439 shutil.copyfileobj(optionsFile, target)
440 servicesMap = _filterJVMCIServices(servicesMap, jdkJars)
441 for serviceName, serviceImpls in servicesMap.iteritems():
442 fd, tmp = tempfile.mkstemp(prefix=serviceName)
443 f = os.fdopen(fd, 'w+')
444 for serviceImpl in serviceImpls:
445 f.write(serviceImpl + os.linesep)
446 target = join(servicesDir, serviceName)
447 f.close()
448 shutil.move(tmp, target)
449 if mx.get_os() != 'windows':
450 os.chmod(target, JDK_UNIX_PERMISSIONS_FILE)
451
452 def _updateJVMCIFiles(jdkDir):
453 jreJVMCIDir = join(jdkDir, 'jre', 'lib', 'jvmci')
454 jvmciJars = [join(jreJVMCIDir, e) for e in os.listdir(jreJVMCIDir) if e.endswith('.jar')]
455 jreJVMCIServicesDir = join(jreJVMCIDir, 'services')
456 jreJVMCIOptionsDir = join(jreJVMCIDir, 'options')
457 _extractJVMCIFiles(_getJdkDeployedJars(jdkDir), jvmciJars, jreJVMCIServicesDir, jreJVMCIOptionsDir)
458
459 def _patchGraalVersionConstant(dist):
460 """
461 Patches the constant "@@@@@@@@@@@@@@@@graal.version@@@@@@@@@@@@@@@@" in the constant pool of Graal.class
462 with the computed Graal version string.
463 """
464 zf = zipfile.ZipFile(dist.path, 'r')
465 graalClassfilePath = 'com/oracle/graal/api/runtime/Graal.class'
466 try:
467 graalClassfile = zf.read(graalClassfilePath)
468 except KeyError:
469 mx.log(graalClassfilePath + ' is not present in ' + dist.path)
470 return
471 placeholder = '@@@@@@@@@@@@@@@@graal.version@@@@@@@@@@@@@@@@'
472 placeholderLen = len(placeholder)
473 versionSpec = '{:' + str(placeholderLen) + '}'
474 versionStr = versionSpec.format(graal_version())
475
476 if len(versionStr) > placeholderLen:
477 # Truncate the version string if necessary
478 assert versionStr.startswith('unknown'), versionStr
479 versionStr = versionStr[:placeholderLen]
480 if placeholder not in graalClassfile:
481 assert versionStr in graalClassfile, 'could not find "' + placeholder + '" or "' + versionStr + '" constant in ' + dist.path + '!' + graalClassfilePath
482 zf.close()
483 return False
484
485 zfOutFd, zfOutPath = tempfile.mkstemp(suffix='', prefix=basename(dist.path) + '.', dir=dirname(dist.path))
486 zfOut = zipfile.ZipFile(zfOutPath, 'w')
487 for zi in zf.infolist():
488 if zi.filename == graalClassfilePath:
489 data = graalClassfile.replace(placeholder, versionStr)
490 else:
491 data = zf.read(zi)
492 zfOut.writestr(zi, data)
493 zfOut.close()
494 os.close(zfOutFd)
495 zf.close()
496 shutil.move(zfOutPath, dist.path)
497
498 def _installDistInJdks(deployableDist):
499 """
500 Installs the jar(s) for a given Distribution into all existing JVMCI JDKs
501 """
502 if True:
503 return
504 dist = mx.distribution(deployableDist.name)
505 if dist.name == 'GRAAL':
506 _patchGraalVersionConstant(dist)
507
508 jdks = _jdksDir()
509 if exists(jdks):
510 for e in os.listdir(jdks):
511 jdkDir = join(jdks, e)
512 jreLibDir = join(jdkDir, 'jre', 'lib')
513 if exists(jreLibDir):
514 if deployableDist.isExtension:
515 targetDir = join(jreLibDir, 'ext')
516 elif deployableDist.usesJVMCIClassLoader:
517 targetDir = join(jreLibDir, 'jvmci')
518 else:
519 targetDir = jreLibDir
520 if not exists(targetDir):
521 os.makedirs(targetDir)
522 _copyToJdk(dist.path, targetDir)
523 if dist.sourcesPath:
524 _copyToJdk(dist.sourcesPath, jdkDir)
525 if deployableDist.usesJVMCIClassLoader:
526 # deploy service files
527 _updateJVMCIFiles(jdkDir)
528
529 def _getJdkDeployedJars(jdkDir):
530 """
531 Gets jar paths for all deployed distributions in the context of
532 a given JDK directory.
533 """
534 jreLibDir = join(jdkDir, 'jre', 'lib')
535 jars = []
536 for dist in _jdkDeployedDists:
537 jar = basename(mx.distribution(dist.name).path)
538 if dist.isExtension:
539 jars.append(join(jreLibDir, 'ext', jar))
540 elif dist.usesJVMCIClassLoader:
541 jars.append(join(jreLibDir, 'jvmci', jar))
542 else:
543 jars.append(join(jreLibDir, jar))
544 return jars
545
546
547 # run a command in the windows SDK Debug Shell
548 def _runInDebugShell(cmd, workingDir, logFile=None, findInOutput=None, respondTo=None):
549 if respondTo is None:
550 respondTo = {}
551 newLine = os.linesep
552 startToken = 'RUNINDEBUGSHELL_STARTSEQUENCE'
553 endToken = 'RUNINDEBUGSHELL_ENDSEQUENCE'
554
555 winSDK = mx.get_env('WIN_SDK', 'C:\\Program Files\\Microsoft SDKs\\Windows\\v7.1\\')
556
557 if not exists(mx._cygpathW2U(winSDK)):
558 mx.abort("Could not find Windows SDK : '" + winSDK + "' does not exist")
559
560 winSDKSetEnv = mx._cygpathW2U(join(winSDK, 'Bin', 'SetEnv.cmd'))
561 if not exists(winSDKSetEnv):
562 mx.abort("Invalid Windows SDK path (" + winSDK + ") : could not find Bin/SetEnv.cmd (you can use the WIN_SDK environment variable to specify an other path)")
563
564 wincmd = 'cmd.exe /E:ON /V:ON /K "' + mx._cygpathU2W(winSDKSetEnv) + '"'
565 p = subprocess.Popen(wincmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
566 stdout = p.stdout
567 stdin = p.stdin
568 if logFile:
569 log = open(logFile, 'w')
570 ret = False
571
572 def _writeProcess(s):
573 stdin.write(s + newLine)
574
575 _writeProcess("echo " + startToken)
576 while True:
577 # encoding may be None on windows plattforms
578 if sys.stdout.encoding is None:
579 encoding = 'utf-8'
580 else:
581 encoding = sys.stdout.encoding
582
583 line = stdout.readline().decode(encoding)
584 if logFile:
585 log.write(line.encode('utf-8'))
586 line = line.strip()
587 mx.log(line)
588 if line == startToken:
589 _writeProcess('cd /D ' + workingDir + ' & ' + cmd + ' & echo ' + endToken)
590 for regex in respondTo.keys():
591 match = regex.search(line)
592 if match:
593 _writeProcess(respondTo[regex])
594 if findInOutput:
595 match = findInOutput.search(line)
596 if match:
597 ret = True
598 if line == endToken:
599 if not findInOutput:
600 _writeProcess('echo ERRXXX%errorlevel%')
601 else:
602 break
603 if line.startswith('ERRXXX'):
604 if line == 'ERRXXX0':
605 ret = True
606 break
607 _writeProcess("exit")
608 if logFile:
609 log.close()
610 return ret
611
612 def jdkhome(vm=None):
613 """return the JDK directory selected for the 'vm' command"""
614 return _jdk(installJars=False)
615
616 def print_jdkhome(args, vm=None):
617 """print the JDK directory selected for the 'vm' command"""
618 print jdkhome(vm)
619
620 def buildvars(args):
621 """describe the variables that can be set by the -D option to the 'mx build' commmand"""
622
623 buildVars = {
624 'ALT_BOOTDIR' : 'The location of the bootstrap JDK installation (default: ' + mx.java().jdk + ')',
625 'ALT_OUTPUTDIR' : 'Build directory',
626 'HOTSPOT_BUILD_JOBS' : 'Number of CPUs used by make (default: ' + str(mx.cpu_count()) + ')',
627 'INSTALL' : 'Install the built VM into the JDK? (default: y)',
628 'ZIP_DEBUGINFO_FILES' : 'Install zipped debug symbols file? (default: 0)',
629 }
630
631 mx.log('HotSpot build variables that can be set by the -D option to "mx build":')
632 mx.log('')
633 for n in sorted(buildVars.iterkeys()):
634 mx.log(n)
635 mx.log(textwrap.fill(buildVars[n], initial_indent=' ', subsequent_indent=' ', width=200))
636
637 mx.log('')
638 mx.log('Note that these variables can be given persistent values in the file ' + join(_graal_home, 'mx', 'env') + ' (see \'mx about\').')
639
640 cached_graal_version = None
641
642 def graal_version(dev_suffix='dev'):
643 global cached_graal_version
644
645 if not cached_graal_version:
646 # extract latest release tag for graal
647 try:
648 tags = [x.split() for x in subprocess.check_output(['hg', '-R', _graal_home, 'tags']).split('\n') if x.startswith("graal-")]
649 current_id = subprocess.check_output(['hg', '-R', _graal_home, 'log', '--template', '{rev}\n', '--rev', 'tip']).strip()
650 except:
651 # not a mercurial repository or hg commands are not available.
652 tags = None
653
654 if tags and current_id:
655 sorted_tags = sorted(tags, key=lambda e: [int(x) for x in e[0][len("graal-"):].split('.')], reverse=True)
656 most_recent_tag_name, most_recent_tag_revision = sorted_tags[0]
657 most_recent_tag_id = most_recent_tag_revision[:most_recent_tag_revision.index(":")]
658 most_recent_tag_version = most_recent_tag_name[len("graal-"):]
659
660 # tagged commit is one-off with commit that tags it
661 if int(current_id) - int(most_recent_tag_id) <= 1:
662 cached_graal_version = most_recent_tag_version
663 else:
664 major, minor = map(int, most_recent_tag_version.split('.'))
665 cached_graal_version = str(major) + '.' + str(minor + 1) + '-' + dev_suffix
666 else:
667 cached_graal_version = 'unknown-{0}'.format(platform.node())
668
669 return cached_graal_version
670 32
671 def build(args, vm=None): 33 def build(args, vm=None):
672 """build the VM binary 34 """build the Java sources"""
673 35 opts2 = mx.build(['--source', '1.7'] + args)
674 The global '--vm' and '--vmbuild' options select which VM type and build target to build."""
675
676 # Override to fail quickly if extra arguments are given
677 # at the end of the command line. This allows for a more
678 # helpful error message.
679 class AP(ArgumentParser):
680 def __init__(self):
681 ArgumentParser.__init__(self, prog='mx build')
682 def parse_args(self, args):
683 result = ArgumentParser.parse_args(self, args)
684 if len(result.remainder) != 0:
685 firstBuildTarget = result.remainder[0]
686 mx.abort('To specify the ' + firstBuildTarget + ' VM build target, you need to use the global "--vmbuild" option. For example:\n' +
687 ' mx --vmbuild ' + firstBuildTarget + ' build')
688 return result
689
690 # Call mx.build to compile the Java sources
691 parser = AP()
692 parser.add_argument('-D', action='append', help='set a HotSpot build variable (run \'mx buildvars\' to list variables)', metavar='name=value')
693
694 opts2 = mx.build(['--source', '1.7'] + args, parser=parser)
695 assert len(opts2.remainder) == 0 36 assert len(opts2.remainder) == 0
696
697 def vmg(args):
698 """run the debug build of VM selected by the '--vm' option"""
699 return vm(args, vmbuild='debug')
700
701 def vmfg(args):
702 """run the fastdebug build of VM selected by the '--vm' option"""
703 return vm(args, vmbuild='fastdebug')
704
705 def _parseVmArgs(args, vm=None, cwd=None, vmbuild=None):
706 """run the VM selected by the '--vm' option"""
707
708 jdk = mx.java().jdk
709 mx.expand_project_in_args(args)
710 exe = join(jdk, 'bin', mx.exe_suffix('java'))
711 pfx = []
712
713 if '-version' in args:
714 ignoredArgs = args[args.index('-version') + 1:]
715 if len(ignoredArgs) > 0:
716 mx.log("Warning: The following options will be ignored by the vm because they come after the '-version' argument: " + ' '.join(ignoredArgs))
717
718 # Unconditionally prepend Truffle to the boot class path.
719 # This used to be done by the VM itself but was removed to
720 # separate the VM from Truffle.
721 truffle_jar = mx.archive(['@TRUFFLE'])[0]
722 args = ['-Xbootclasspath/p:' + truffle_jar] + args
723
724 args = mx.java().processArgs(args)
725 return (pfx, exe, vm, args, cwd)
726
727 def vm(args, vm=None, nonZeroIsFatal=True, out=None, err=None, cwd=None, timeout=None, vmbuild=None):
728 (pfx_, exe_, _, args_, cwd) = _parseVmArgs(args, vm, cwd, vmbuild)
729 return mx.run(pfx_ + [exe_] + args_, nonZeroIsFatal=nonZeroIsFatal, out=out, err=err, cwd=cwd)
730
731 def _find_classes_with_annotations(p, pkgRoot, annotations, includeInnerClasses=False):
732 """
733 Scan the sources of project 'p' for Java source files containing a line starting with 'annotation'
734 (ignoring preceding whitespace) and return the fully qualified class name for each Java
735 source file matched in a list.
736 """
737
738 matches = lambda line: len([a for a in annotations if line == a or line.startswith(a + '(')]) != 0
739 return p.find_classes_with_matching_source_line(pkgRoot, matches, includeInnerClasses)
740
741 def _extract_VM_args(args, allowClasspath=False, useDoubleDash=False, defaultAllVMArgs=True):
742 """
743 Partitions a command line into a leading sequence of HotSpot VM options and the rest.
744 """
745 for i in range(0, len(args)):
746 if useDoubleDash:
747 if args[i] == '--':
748 vmArgs = args[:i]
749 remainder = args[i + 1:]
750 return vmArgs, remainder
751 else:
752 if not args[i].startswith('-'):
753 if i != 0 and (args[i - 1] == '-cp' or args[i - 1] == '-classpath'):
754 if not allowClasspath:
755 mx.abort('Cannot supply explicit class path option')
756 else:
757 continue
758 vmArgs = args[:i]
759 remainder = args[i:]
760 return vmArgs, remainder
761
762 if defaultAllVMArgs:
763 return args, []
764 else:
765 return [], args
766
767 def _run_tests(args, harness, annotations, testfile, blacklist, whitelist, regex):
768
769
770 vmArgs, tests = _extract_VM_args(args)
771 for t in tests:
772 if t.startswith('-'):
773 mx.abort('VM option ' + t + ' must precede ' + tests[0])
774
775 candidates = {}
776 for p in mx.projects_opt_limit_to_suites():
777 if mx.java().javaCompliance < p.javaCompliance:
778 continue
779 for c in _find_classes_with_annotations(p, None, annotations).keys():
780 candidates[c] = p
781
782 classes = []
783 if len(tests) == 0:
784 classes = candidates.keys()
785 projectsCp = mx.classpath([pcp.name for pcp in mx.projects_opt_limit_to_suites() if pcp.javaCompliance <= mx.java().javaCompliance])
786 else:
787 projs = set()
788 found = False
789 if len(tests) == 1 and '#' in tests[0]:
790 words = tests[0].split('#')
791 if len(words) != 2:
792 mx.abort("Method specification is class#method: " + tests[0])
793 t, method = words
794
795 for c, p in candidates.iteritems():
796 # prefer exact matches first
797 if t == c:
798 found = True
799 classes.append(c)
800 projs.add(p.name)
801 if not found:
802 for c, p in candidates.iteritems():
803 if t in c:
804 found = True
805 classes.append(c)
806 projs.add(p.name)
807 if not found:
808 mx.log('warning: no tests matched by substring "' + t)
809 elif len(classes) != 1:
810 mx.abort('More than one test matches substring {0} {1}'.format(t, classes))
811
812 classes = [c + "#" + method for c in classes]
813 else:
814 for t in tests:
815 if '#' in t:
816 mx.abort('Method specifications can only be used in a single test: ' + t)
817 for c, p in candidates.iteritems():
818 if t in c:
819 found = True
820 classes.append(c)
821 projs.add(p.name)
822 if not found:
823 mx.log('warning: no tests matched by substring "' + t)
824 projectsCp = mx.classpath(projs)
825
826 if blacklist:
827 classes = [c for c in classes if not any((glob.match(c) for glob in blacklist))]
828
829 if whitelist:
830 classes = [c for c in classes if any((glob.match(c) for glob in whitelist))]
831
832 if regex:
833 classes = [c for c in classes if re.search(regex, c)]
834
835 if len(classes) != 0:
836 f_testfile = open(testfile, 'w')
837 for c in classes:
838 f_testfile.write(c + '\n')
839 f_testfile.close()
840 harness(projectsCp, vmArgs)
841
842 def _unittest(args, annotations, prefixCp="", blacklist=None, whitelist=None, verbose=False, fail_fast=False, enable_timing=False, regex=None, color=False, eager_stacktrace=False, gc_after_test=False):
843 testfile = os.environ.get('MX_TESTFILE', None)
844 if testfile is None:
845 (_, testfile) = tempfile.mkstemp(".testclasses", "graal")
846 os.close(_)
847
848 coreCp = mx.classpath(['com.oracle.truffle.tck', 'HCFDIS'])
849
850 coreArgs = []
851 if verbose:
852 coreArgs.append('-JUnitVerbose')
853 if fail_fast:
854 coreArgs.append('-JUnitFailFast')
855 if enable_timing:
856 coreArgs.append('-JUnitEnableTiming')
857 if color:
858 coreArgs.append('-JUnitColor')
859 if eager_stacktrace:
860 coreArgs.append('-JUnitEagerStackTrace')
861 if gc_after_test:
862 coreArgs.append('-JUnitGCAfterTest')
863
864
865 def harness(projectsCp, vmArgs):
866 if _get_vm() != 'jvmci':
867 prefixArgs = ['-esa', '-ea']
868 else:
869 prefixArgs = ['-XX:-BootstrapJVMCI', '-esa', '-ea']
870 if gc_after_test:
871 prefixArgs.append('-XX:-DisableExplicitGC')
872 with open(testfile) as fp:
873 testclasses = [l.rstrip() for l in fp.readlines()]
874
875 # Remove entries from class path that are in graal.jar and
876 # run the VM in a mode where application/test classes can
877 # access core Graal classes.
878 cp = prefixCp + coreCp + os.pathsep + projectsCp
879 if isJVMCIEnabled(_get_vm()):
880 excluded = set()
881 for jdkDist in _jdkDeployedDists:
882 dist = mx.distribution(jdkDist.name)
883 excluded.update([d.output_dir() for d in dist.sorted_deps()])
884 cp = os.pathsep.join([e for e in cp.split(os.pathsep) if e not in excluded])
885
886 # suppress menubar and dock when running on Mac
887 vmArgs = ['-Djava.awt.headless=true'] + vmArgs
888
889 if len(testclasses) == 1:
890 # Execute Junit directly when one test is being run. This simplifies
891 # replaying the VM execution in a native debugger (e.g., gdb).
892 vm(prefixArgs + vmArgs + ['-cp', mx._separatedCygpathU2W(cp), 'com.oracle.truffle.tck.TruffleJUnitCore'] + coreArgs + testclasses)
893 else:
894 vm(prefixArgs + vmArgs + ['-cp', mx._separatedCygpathU2W(cp), 'com.oracle.truffle.tck.TruffleJUnitCore'] + coreArgs + ['@' + mx._cygpathU2W(testfile)])
895
896 try:
897 _run_tests(args, harness, annotations, testfile, blacklist, whitelist, regex)
898 finally:
899 if os.environ.get('MX_TESTFILE') is None:
900 os.remove(testfile)
901
902 _unittestHelpSuffix = """
903 Unittest options:
904
905 --blacklist <file> run all testcases not specified in the blacklist
906 --whitelist <file> run only testcases which are included
907 in the given whitelist
908 --verbose enable verbose JUnit output
909 --fail-fast stop after first JUnit test class that has a failure
910 --enable-timing enable JUnit test timing
911 --regex <regex> run only testcases matching a regular expression
912 --color enable colors output
913 --eager-stacktrace print stacktrace eagerly
914 --gc-after-test force a GC after each test
915
916 To avoid conflicts with VM options '--' can be used as delimiter.
917
918 If filters are supplied, only tests whose fully qualified name
919 includes a filter as a substring are run.
920
921 For example, this command line:
922
923 mx unittest -G:Dump= -G:MethodFilter=BC_aload.* -G:+PrintCFG BC_aload
924
925 will run all JUnit test classes that contain 'BC_aload' in their
926 fully qualified name and will pass these options to the VM:
927
928 -G:Dump= -G:MethodFilter=BC_aload.* -G:+PrintCFG
929
930 To get around command line length limitations on some OSes, the
931 JUnit class names to be executed are written to a file that a
932 custom JUnit wrapper reads and passes onto JUnit proper. The
933 MX_TESTFILE environment variable can be set to specify a
934 file which will not be deleted once the unittests are done
935 (unlike the temporary file otherwise used).
936
937 As with all other commands, using the global '-v' before 'unittest'
938 command will cause mx to show the complete command line
939 it uses to run the VM.
940 """
941
942 def unittest(args):
943 """run the JUnit tests (all testcases){0}"""
944
945 parser = ArgumentParser(prog='mx unittest',
946 description='run the JUnit tests',
947 add_help=False,
948 formatter_class=RawDescriptionHelpFormatter,
949 epilog=_unittestHelpSuffix,
950 )
951 parser.add_argument('--blacklist', help='run all testcases not specified in the blacklist', metavar='<path>')
952 parser.add_argument('--whitelist', help='run testcases specified in whitelist only', metavar='<path>')
953 parser.add_argument('--verbose', help='enable verbose JUnit output', action='store_true')
954 parser.add_argument('--fail-fast', help='stop after first JUnit test class that has a failure', action='store_true')
955 parser.add_argument('--enable-timing', help='enable JUnit test timing', action='store_true')
956 parser.add_argument('--regex', help='run only testcases matching a regular expression', metavar='<regex>')
957 parser.add_argument('--color', help='enable color output', action='store_true')
958 parser.add_argument('--eager-stacktrace', help='print stacktrace eagerly', action='store_true')
959 parser.add_argument('--gc-after-test', help='force a GC after each test', action='store_true')
960
961 ut_args = []
962 delimiter = False
963 # check for delimiter
964 while len(args) > 0:
965 arg = args.pop(0)
966 if arg == '--':
967 delimiter = True
968 break
969 ut_args.append(arg)
970
971 if delimiter:
972 # all arguments before '--' must be recognized
973 parsed_args = parser.parse_args(ut_args)
974 else:
975 # parse all know arguments
976 parsed_args, args = parser.parse_known_args(ut_args)
977
978 if parsed_args.whitelist:
979 try:
980 with open(join(_graal_home, parsed_args.whitelist)) as fp:
981 parsed_args.whitelist = [re.compile(fnmatch.translate(l.rstrip())) for l in fp.readlines() if not l.startswith('#')]
982 except IOError:
983 mx.log('warning: could not read whitelist: ' + parsed_args.whitelist)
984 if parsed_args.blacklist:
985 try:
986 with open(join(_graal_home, parsed_args.blacklist)) as fp:
987 parsed_args.blacklist = [re.compile(fnmatch.translate(l.rstrip())) for l in fp.readlines() if not l.startswith('#')]
988 except IOError:
989 mx.log('warning: could not read blacklist: ' + parsed_args.blacklist)
990
991 _unittest(args, ['@Test', '@Parameters'], **parsed_args.__dict__)
992
993 def shortunittest(args):
994 """alias for 'unittest --whitelist test/whitelist_shortunittest.txt'{0}"""
995
996 args = ['--whitelist', 'test/whitelist_shortunittest.txt'] + args
997 unittest(args)
998
999 class Task:
1000 # None or a list of strings. If not None, only tasks whose title
1001 # matches at least one of the substrings in this list will return
1002 # a non-None value from __enter__. The body of a 'with Task(...) as t'
1003 # statement should check 't' and exit immediately if it is None.
1004 filters = None
1005 filtersExclude = False
1006
1007 def __init__(self, title, tasks=None):
1008 self.tasks = tasks
1009 self.title = title
1010 if tasks is not None and Task.filters is not None:
1011 if Task.filtersExclude:
1012 self.skipped = any([f in title for f in Task.filters])
1013 else:
1014 self.skipped = not any([f in title for f in Task.filters])
1015 else:
1016 self.skipped = False
1017 if not self.skipped:
1018 self.start = time.time()
1019 self.end = None
1020 self.duration = None
1021 mx.log(time.strftime('gate: %d %b %Y %H:%M:%S: BEGIN: ') + title)
1022 def __enter__(self):
1023 assert self.tasks is not None, "using Task with 'with' statement requires to pass the tasks list in the constructor"
1024 if self.skipped:
1025 return None
1026 return self
1027 def __exit__(self, exc_type, exc_value, traceback):
1028 if not self.skipped:
1029 self.tasks.append(self.stop())
1030 def stop(self):
1031 self.end = time.time()
1032 self.duration = datetime.timedelta(seconds=self.end - self.start)
1033 mx.log(time.strftime('gate: %d %b %Y %H:%M:%S: END: ') + self.title + ' [' + str(self.duration) + ']')
1034 return self
1035 def abort(self, codeOrMessage):
1036 self.end = time.time()
1037 self.duration = datetime.timedelta(seconds=self.end - self.start)
1038 mx.log(time.strftime('gate: %d %b %Y %H:%M:%S: ABORT: ') + self.title + ' [' + str(self.duration) + ']')
1039 mx.abort(codeOrMessage)
1040 return self
1041
1042 def _basic_gate_body(args, tasks):
1043 # Run unit tests on server-hosted-jvmci
1044 with Task('UnitTests:hosted-product', tasks) as t:
1045 if t: unittest(['--enable-timing', '--verbose', '--fail-fast'])
1046
1047
1048 def gate(args, gate_body=_basic_gate_body):
1049 """run the tests used to validate a push
1050
1051 If this command exits with a 0 exit code, then the source code is in
1052 a state that would be accepted for integration into the main repository."""
1053
1054 parser = ArgumentParser(prog='mx gate')
1055 parser.add_argument('-j', '--omit-java-clean', action='store_false', dest='cleanJava', help='omit cleaning Java native code')
1056 parser.add_argument('-n', '--omit-native-clean', action='store_false', dest='cleanNative', help='omit cleaning and building native code')
1057 parser.add_argument('-i', '--omit-ide-clean', action='store_false', dest='cleanIde', help='omit cleaning the ide project files')
1058 parser.add_argument('-g', '--only-build-jvmci', action='store_false', dest='buildNonJVMCI', help='only build the JVMCI VM')
1059 parser.add_argument('-t', '--task-filter', help='comma separated list of substrings to select subset of tasks to be run')
1060 parser.add_argument('-x', action='store_true', help='makes --task-filter an exclusion instead of inclusion filter')
1061
1062 args = parser.parse_args(args)
1063
1064 if args.task_filter:
1065 Task.filters = args.task_filter.split(',')
1066 Task.filtersExclude = args.x
1067 elif args.x:
1068 mx.abort('-x option cannot be used without --task-filter option')
1069
1070 # Force
1071 if not mx._opts.strict_compliance:
1072 mx.log("[gate] forcing strict compliance")
1073 mx._opts.strict_compliance = True
1074
1075 tasks = []
1076 total = Task('Gate')
1077 try:
1078 with Task('Pylint', tasks) as t:
1079 if t: mx.pylint([])
1080
1081 def _clean(name='Clean'):
1082 with Task(name, tasks) as t:
1083 if t:
1084 cleanArgs = []
1085 if not args.cleanNative:
1086 cleanArgs.append('--no-native')
1087 if not args.cleanJava:
1088 cleanArgs.append('--no-java')
1089 clean(cleanArgs)
1090 _clean()
1091
1092 with Task('IDEConfigCheck', tasks) as t:
1093 if t:
1094 if args.cleanIde:
1095 mx.ideclean([])
1096 mx.ideinit([])
1097
1098 eclipse_exe = mx.get_env('ECLIPSE_EXE')
1099 if eclipse_exe is not None:
1100 with Task('CodeFormatCheck', tasks) as t:
1101 if t and mx.eclipseformat(['-e', eclipse_exe]) != 0:
1102 t.abort('Formatter modified files - run "mx eclipseformat", check in changes and repush')
1103
1104 with Task('Canonicalization Check', tasks) as t:
1105 if t:
1106 mx.log(time.strftime('%d %b %Y %H:%M:%S - Ensuring mx/projects files are canonicalized...'))
1107 if mx.canonicalizeprojects([]) != 0:
1108 t.abort('Rerun "mx canonicalizeprojects" and check-in the modified mx/projects files.')
1109
1110 if mx.get_env('JDT'):
1111 with Task('BuildJavaWithEcj', tasks):
1112 if t: build(['-p', '--no-native', '--jdt-warning-as-error'])
1113 _clean('CleanAfterEcjBuild')
1114
1115 with Task('BuildJavaWithJavac', tasks):
1116 if t: build(['-p', '--no-native', '--force-javac'])
1117
1118 with Task('Checkstyle', tasks) as t:
1119 if t and mx.checkstyle([]) != 0:
1120 t.abort('Checkstyle warnings were found')
1121
1122 with Task('Checkheaders', tasks) as t:
1123 if t and checkheaders([]) != 0:
1124 t.abort('Checkheaders warnings were found')
1125
1126 with Task('FindBugs', tasks) as t:
1127 if t and findbugs([]) != 0:
1128 t.abort('FindBugs warnings were found')
1129
1130 gate_body(args, tasks)
1131
1132 except KeyboardInterrupt:
1133 total.abort(1)
1134
1135 except BaseException as e:
1136 import traceback
1137 traceback.print_exc()
1138 total.abort(str(e))
1139
1140 total.stop()
1141
1142 mx.log('Gate task times:')
1143 for t in tasks:
1144 mx.log(' ' + str(t.duration) + '\t' + t.title)
1145 mx.log(' =======')
1146 mx.log(' ' + str(total.duration))
1147
1148 if args.task_filter:
1149 Task.filters = None
1150
1151 def _igvJdk():
1152 v8u20 = mx.VersionSpec("1.8.0_20")
1153 v8u40 = mx.VersionSpec("1.8.0_40")
1154 v8 = mx.VersionSpec("1.8")
1155 def _igvJdkVersionCheck(version):
1156 return version >= v8 and (version < v8u20 or version >= v8u40)
1157 return mx.java_version(_igvJdkVersionCheck, versionDescription='>= 1.8 and < 1.8.0u20 or >= 1.8.0u40').jdk
1158
1159 def _igvBuildEnv():
1160 # When the http_proxy environment variable is set, convert it to the proxy settings that ant needs
1161 env = dict(os.environ)
1162 proxy = os.environ.get('http_proxy')
1163 if not (proxy is None) and len(proxy) > 0:
1164 if '://' in proxy:
1165 # Remove the http:// prefix (or any other protocol prefix)
1166 proxy = proxy.split('://', 1)[1]
1167 # Separate proxy server name and port number
1168 proxyName, proxyPort = proxy.split(':', 1)
1169 proxyEnv = '-DproxyHost="' + proxyName + '" -DproxyPort=' + proxyPort
1170 env['ANT_OPTS'] = proxyEnv
1171
1172 env['JAVA_HOME'] = _igvJdk()
1173 return env
1174
1175 def igv(args):
1176 """run the Ideal Graph Visualizer"""
1177 logFile = '.ideal_graph_visualizer.log'
1178 with open(join(_graal_home, logFile), 'w') as fp:
1179 mx.logv('[Ideal Graph Visualizer log is in ' + fp.name + ']')
1180 nbplatform = join(_graal_home, 'src', 'share', 'tools', 'IdealGraphVisualizer', 'nbplatform')
1181
1182 # Remove NetBeans platform if it is earlier than the current supported version
1183 if exists(nbplatform):
1184 updateTrackingFile = join(nbplatform, 'platform', 'update_tracking', 'org-netbeans-core.xml')
1185 if not exists(updateTrackingFile):
1186 mx.log('Could not find \'' + updateTrackingFile + '\', removing NetBeans platform')
1187 shutil.rmtree(nbplatform)
1188 else:
1189 dom = xml.dom.minidom.parse(updateTrackingFile)
1190 currentVersion = mx.VersionSpec(dom.getElementsByTagName('module_version')[0].getAttribute('specification_version'))
1191 supportedVersion = mx.VersionSpec('3.43.1')
1192 if currentVersion < supportedVersion:
1193 mx.log('Replacing NetBeans platform version ' + str(currentVersion) + ' with version ' + str(supportedVersion))
1194 shutil.rmtree(nbplatform)
1195 elif supportedVersion < currentVersion:
1196 mx.log('Supported NetBeans version in igv command should be updated to ' + str(currentVersion))
1197
1198 if not exists(nbplatform):
1199 mx.logv('[This execution may take a while as the NetBeans platform needs to be downloaded]')
1200
1201 env = _igvBuildEnv()
1202 # make the jar for Batik 1.7 available.
1203 env['IGV_BATIK_JAR'] = mx.library('BATIK').get_path(True)
1204 if mx.run(['ant', '-f', mx._cygpathU2W(join(_graal_home, 'src', 'share', 'tools', 'IdealGraphVisualizer', 'build.xml')), '-l', mx._cygpathU2W(fp.name), 'run'], env=env, nonZeroIsFatal=False):
1205 mx.abort("IGV ant build & launch failed. Check '" + logFile + "'. You can also try to delete 'src/share/tools/IdealGraphVisualizer/nbplatform'.")
1206 37
1207 def maven_install_truffle(args): 38 def maven_install_truffle(args):
1208 """install Truffle into your local Maven repository""" 39 """install Truffle into your local Maven repository"""
1209 for name in mx._dists: 40 for name in mx._dists:
1210 dist = mx._dists[name] 41 dist = mx._dists[name]
1215 slash = path.rfind('/') 46 slash = path.rfind('/')
1216 dot = path.rfind('.') 47 dot = path.rfind('.')
1217 if dot <= slash: 48 if dot <= slash:
1218 mx.abort('Dot should be after / in ' + path) 49 mx.abort('Dot should be after / in ' + path)
1219 artifactId = path[slash + 1: dot] 50 artifactId = path[slash + 1: dot]
1220 mx.run(['mvn', 'install:install-file', '-DgroupId=com.oracle.' + dist.suite.name, '-DartifactId=' + artifactId, '-Dversion=' + graal_version('SNAPSHOT'), '-Dpackaging=jar', '-Dfile=' + path]) 51 mx.run(['mvn', 'install:install-file', '-DgroupId=com.oracle.' + dist.suite.name, '-DartifactId=' + artifactId, '-Dversion=' + mx.suite('truffle').release_version('SNAPSHOT'), '-Dpackaging=jar', '-Dfile=' + path])
1221
1222 def c1visualizer(args):
1223 """run the Cl Compiler Visualizer"""
1224 libpath = join(_graal_home, 'lib')
1225 if mx.get_os() == 'windows':
1226 executable = join(libpath, 'c1visualizer', 'bin', 'c1visualizer.exe')
1227 else:
1228 executable = join(libpath, 'c1visualizer', 'bin', 'c1visualizer')
1229
1230 # Check whether the current C1Visualizer installation is the up-to-date
1231 if exists(executable) and not exists(mx.library('C1VISUALIZER_DIST').get_path(resolve=False)):
1232 mx.log('Updating C1Visualizer')
1233 shutil.rmtree(join(libpath, 'c1visualizer'))
1234
1235 archive = mx.library('C1VISUALIZER_DIST').get_path(resolve=True)
1236
1237 if not exists(executable):
1238 zf = zipfile.ZipFile(archive, 'r')
1239 zf.extractall(libpath)
1240
1241 if not exists(executable):
1242 mx.abort('C1Visualizer binary does not exist: ' + executable)
1243
1244 if mx.get_os() != 'windows':
1245 # Make sure that execution is allowed. The zip file does not always specfiy that correctly
1246 os.chmod(executable, 0777)
1247
1248 mx.run([executable])
1249
1250 def hsdis(args, copyToDir=None):
1251 """download the hsdis library
1252
1253 This is needed to support HotSpot's assembly dumping features.
1254 By default it downloads the Intel syntax version, use the 'att' argument to install AT&T syntax."""
1255 flavor = 'intel'
1256 if 'att' in args:
1257 flavor = 'att'
1258 if mx.get_arch() == "sparcv9":
1259 flavor = "sparcv9"
1260 lib = mx.add_lib_suffix('hsdis-' + mx.get_arch())
1261 path = join(_graal_home, 'lib', lib)
1262
1263 sha1s = {
1264 'att/hsdis-amd64.dll' : 'bcbd535a9568b5075ab41e96205e26a2bac64f72',
1265 'att/hsdis-amd64.so' : '58919ba085d4ef7a513f25bae75e7e54ee73c049',
1266 'intel/hsdis-amd64.dll' : '6a388372cdd5fe905c1a26ced614334e405d1f30',
1267 'intel/hsdis-amd64.so' : '844ed9ffed64fe9599638f29a8450c50140e3192',
1268 'intel/hsdis-amd64.dylib' : 'fdb13ef0d7d23d93dacaae9c98837bea0d4fc5a2',
1269 'sparcv9/hsdis-sparcv9.so': '970640a9af0bd63641f9063c11275b371a59ee60',
1270 }
1271
1272 flavoredLib = flavor + "/" + lib
1273 if flavoredLib not in sha1s:
1274 mx.logv("hsdis not supported on this plattform or architecture")
1275 return
1276
1277 if not exists(path):
1278 sha1 = sha1s[flavoredLib]
1279 sha1path = path + '.sha1'
1280 mx.download_file_with_sha1('hsdis', path, ['http://lafo.ssw.uni-linz.ac.at/hsdis/' + flavoredLib], sha1, sha1path, True, True, sources=False)
1281 if copyToDir is not None and exists(copyToDir):
1282 shutil.copy(path, copyToDir)
1283
1284 def hcfdis(args):
1285 """disassemble HexCodeFiles embedded in text files
1286
1287 Run a tool over the input files to convert all embedded HexCodeFiles
1288 to a disassembled format."""
1289
1290 parser = ArgumentParser(prog='mx hcfdis')
1291 parser.add_argument('-m', '--map', help='address to symbol map applied to disassembler output')
1292 parser.add_argument('files', nargs=REMAINDER, metavar='files...')
1293
1294 args = parser.parse_args(args)
1295
1296 path = mx.library('HCFDIS').get_path(resolve=True)
1297 mx.run_java(['-cp', path, 'com.oracle.max.hcfdis.HexCodeFileDis'] + args.files)
1298
1299 if args.map is not None:
1300 addressRE = re.compile(r'0[xX]([A-Fa-f0-9]+)')
1301 with open(args.map) as fp:
1302 lines = fp.read().splitlines()
1303 symbols = dict()
1304 for l in lines:
1305 addressAndSymbol = l.split(' ', 1)
1306 if len(addressAndSymbol) == 2:
1307 address, symbol = addressAndSymbol
1308 if address.startswith('0x'):
1309 address = long(address, 16)
1310 symbols[address] = symbol
1311 for f in args.files:
1312 with open(f) as fp:
1313 lines = fp.read().splitlines()
1314 updated = False
1315 for i in range(0, len(lines)):
1316 l = lines[i]
1317 for m in addressRE.finditer(l):
1318 sval = m.group(0)
1319 val = long(sval, 16)
1320 sym = symbols.get(val)
1321 if sym:
1322 l = l.replace(sval, sym)
1323 updated = True
1324 lines[i] = l
1325 if updated:
1326 mx.log('updating ' + f)
1327 with open('new_' + f, "w") as fp:
1328 for l in lines:
1329 print >> fp, l
1330 52
1331 def sl(args): 53 def sl(args):
1332 """run an SL program""" 54 """run an SL program"""
1333 vmArgs, slArgs = _extract_VM_args(args) 55 vmArgs, slArgs = mx.extract_VM_args(args)
1334 vm(vmArgs + ['-cp', mx.classpath(["TRUFFLE", "com.oracle.truffle.sl"]), "com.oracle.truffle.sl.SLLanguage"] + slArgs) 56 mx.run_java(vmArgs + ['-cp', mx.classpath(["TRUFFLE", "com.oracle.truffle.sl"]), "com.oracle.truffle.sl.SLLanguage"] + slArgs)
1335 57
1336 def sldebug(args): 58 def sldebug(args):
1337 """run a simple command line debugger for the Simple Language""" 59 """run a simple command line debugger for the Simple Language"""
1338 vmArgs, slArgs = _extract_VM_args(args, useDoubleDash=True) 60 vmArgs, slArgs = mx.extract_VM_args(args, useDoubleDash=True)
1339 vm(vmArgs + ['-cp', mx.classpath("com.oracle.truffle.sl.tools"), "com.oracle.truffle.sl.tools.debug.SLREPLServer"] + slArgs) 61 mx.run_java(vmArgs + ['-cp', mx.classpath("com.oracle.truffle.sl.tools"), "com.oracle.truffle.sl.tools.debug.SLREPLServer"] + slArgs)
1340 62
1341 def isJVMCIEnabled(vm): 63 mx.update_commands(_suite, {
1342 return vm != 'original' and not vm.endswith('nojvmci') 64 'maven-install-truffle' : [maven_install_truffle, ''],
1343 65 'sl' : [sl, '[SL args|@VM options]'],
1344 def jol(args): 66 'sldebug' : [sldebug, '[SL args|@VM options]'],
1345 """Java Object Layout""" 67 })
1346 joljar = mx.library('JOL_INTERNALS').get_path(resolve=True)
1347 candidates = mx.findclass(args, logToConsole=False, matcher=lambda s, classname: s == classname or classname.endswith('.' + s) or classname.endswith('$' + s))
1348
1349 if len(candidates) > 0:
1350 candidates = mx.select_items(sorted(candidates))
1351 else:
1352 # mx.findclass can be mistaken, don't give up yet
1353 candidates = args
1354
1355 vm(['-javaagent:' + joljar, '-cp', os.pathsep.join([mx.classpath(), joljar]), "org.openjdk.jol.MainObjectInternals"] + candidates)
1356
1357 def site(args):
1358 """create a website containing javadoc and the project dependency graph"""
1359
1360 return mx.site(['--name', 'Graal',
1361 '--jd', '@-tag', '--jd', '@test:X',
1362 '--jd', '@-tag', '--jd', '@run:X',
1363 '--jd', '@-tag', '--jd', '@bug:X',
1364 '--jd', '@-tag', '--jd', '@summary:X',
1365 '--jd', '@-tag', '--jd', '@vmoption:X',
1366 '--overview', join(_graal_home, 'graal', 'overview.html'),
1367 '--title', 'Graal OpenJDK Project Documentation',
1368 '--dot-output-base', 'projects'] + args)
1369
1370 def findbugs(args):
1371 '''run FindBugs against non-test Java projects'''
1372 findBugsHome = mx.get_env('FINDBUGS_HOME', None)
1373 if findBugsHome:
1374 findbugsJar = join(findBugsHome, 'lib', 'findbugs.jar')
1375 else:
1376 findbugsLib = join(_graal_home, 'lib', 'findbugs-3.0.0')
1377 if not exists(findbugsLib):
1378 tmp = tempfile.mkdtemp(prefix='findbugs-download-tmp', dir=_graal_home)
1379 try:
1380 findbugsDist = mx.library('FINDBUGS_DIST').get_path(resolve=True)
1381 with zipfile.ZipFile(findbugsDist) as zf:
1382 candidates = [e for e in zf.namelist() if e.endswith('/lib/findbugs.jar')]
1383 assert len(candidates) == 1, candidates
1384 libDirInZip = os.path.dirname(candidates[0])
1385 zf.extractall(tmp)
1386 shutil.copytree(join(tmp, libDirInZip), findbugsLib)
1387 finally:
1388 shutil.rmtree(tmp)
1389 findbugsJar = join(findbugsLib, 'findbugs.jar')
1390 assert exists(findbugsJar)
1391 nonTestProjects = [p for p in mx.projects() if not p.name.endswith('.test') and not p.name.endswith('.jtt')]
1392 outputDirs = map(mx._cygpathU2W, [p.output_dir() for p in nonTestProjects])
1393 javaCompliance = max([p.javaCompliance for p in nonTestProjects])
1394 findbugsResults = join(_graal_home, 'findbugs.results')
1395
1396 cmd = ['-jar', mx._cygpathU2W(findbugsJar), '-textui', '-low', '-maxRank', '15']
1397 if mx.is_interactive():
1398 cmd.append('-progress')
1399 cmd = cmd + ['-auxclasspath', mx._separatedCygpathU2W(mx.classpath([d.name for d in _jdkDeployedDists] + [p.name for p in nonTestProjects])), '-output', mx._cygpathU2W(findbugsResults), '-exitcode'] + args + outputDirs
1400 exitcode = mx.run_java(cmd, nonZeroIsFatal=False, javaConfig=mx.java(javaCompliance))
1401 if exitcode != 0:
1402 with open(findbugsResults) as fp:
1403 mx.log(fp.read())
1404 os.unlink(findbugsResults)
1405 return exitcode
1406
1407 def checkheaders(args):
1408 """check Java source headers against any required pattern"""
1409 failures = {}
1410 for p in mx.projects():
1411 if p.native:
1412 continue
1413
1414 csConfig = join(mx.project(p.checkstyleProj).dir, '.checkstyle_checks.xml')
1415 dom = xml.dom.minidom.parse(csConfig)
1416 for module in dom.getElementsByTagName('module'):
1417 if module.getAttribute('name') == 'RegexpHeader':
1418 for prop in module.getElementsByTagName('property'):
1419 if prop.getAttribute('name') == 'header':
1420 value = prop.getAttribute('value')
1421 matcher = re.compile(value, re.MULTILINE)
1422 for sourceDir in p.source_dirs():
1423 for root, _, files in os.walk(sourceDir):
1424 for name in files:
1425 if name.endswith('.java') and name != 'package-info.java':
1426 f = join(root, name)
1427 with open(f) as fp:
1428 content = fp.read()
1429 if not matcher.match(content):
1430 failures[f] = csConfig
1431 for n, v in failures.iteritems():
1432 mx.log('{0}: header does not match RegexpHeader defined in {1}'.format(n, v))
1433 return len(failures)
1434
1435 def mx_init(suite):
1436 commands = {
1437 'checkheaders': [checkheaders, ''],
1438 'clean': [clean, ''],
1439 'findbugs': [findbugs, ''],
1440 'maven-install-truffle' : [maven_install_truffle, ''],
1441 'jdkhome': [print_jdkhome, ''],
1442 'gate' : [gate, '[-options]'],
1443 'unittest' : [unittest, '[unittest options] [--] [VM options] [filters...]', _unittestHelpSuffix],
1444 'shortunittest' : [shortunittest, '[unittest options] [--] [VM options] [filters...]', _unittestHelpSuffix],
1445 'site' : [site, '[-options]'],
1446 'sl' : [sl, '[SL args|@VM options]'],
1447 'sldebug' : [sldebug, '[SL args|@VM options]'],
1448 'jol' : [jol, ''],
1449 }
1450
1451 mx.update_commands(suite, commands)
1452
1453 def mx_post_parse_cmd_line(opts): #
1454 # TODO _minVersion check could probably be part of a Suite in mx?
1455 if mx.java().version < _minVersion:
1456 mx.abort('Requires Java version ' + str(_minVersion) + ' or greater for JAVA_HOME, got version ' + str(mx.java().version))
1457 if _untilVersion and mx.java().version >= _untilVersion:
1458 mx.abort('Requires Java version strictly before ' + str(_untilVersion) + ' for JAVA_HOME, got version ' + str(mx.java().version))
1459
1460 for jdkDist in _jdkDeployedDists:
1461 def _close(jdkDeployable):
1462 def _install(dist):
1463 assert dist.name == jdkDeployable.name, dist.name + "!=" + jdkDeployable.name
1464 if not jdkDist.partOfHotSpot:
1465 _installDistInJdks(jdkDeployable)
1466 return _install
1467 mx.distribution(jdkDist.name).add_update_listener(_close(jdkDist))