Mercurial > hg > truffle

comparison mx.truffle/mx_truffle.py @ 21961:e61e89bd867a

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
rename mx to mx.truffle
author Mick Jordan <mick.jordan@oracle.com>
date Fri, 19 Jun 2015 12:22:49 -0700
parents mx/mx_truffle.py@ef8c90391f1e
children 1ecffe20e460
comparison
equal deleted inserted replaced
21960:a88981c5ce8b 21961:e61e89bd867a
1 #
2 # commands.py - the GraalVM specific commands
3 #
4 # ----------------------------------------------------------------------------------------------------
5 #
6 # Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved.
7 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
8 #
9 # This code is free software; you can redistribute it and/or modify it
10 # under the terms of the GNU General Public License version 2 only, as
11 # published by the Free Software Foundation.
12 #
13 # This code is distributed in the hope that it will be useful, but WITHOUT
14 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
15 # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
16 # version 2 for more details (a copy is included in the LICENSE file that
17 # accompanied this code).
18 #
19 # You should have received a copy of the GNU General Public License version
20 # 2 along with this work; if not, write to the Free Software Foundation,
21 # Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
22 #
23 # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
24 # or visit www.oracle.com if you need additional information or have any
25 # questions.
26 #
27 # ----------------------------------------------------------------------------------------------------
28
29 import os, stat, errno, sys, shutil, zipfile, tarfile, tempfile, re, time, datetime, platform, subprocess, StringIO, socket
30 from os.path import join, exists, dirname, basename
31 from argparse import ArgumentParser, RawDescriptionHelpFormatter, REMAINDER
32 from outputparser import OutputParser, ValuesMatcher
33 import mx
34 import xml.dom.minidom
35 import itertools
36 import json, textwrap
37 import fnmatch
38
39 # This works because when mx loads this file, it makes sure __file__ gets an absolute path
40 _graal_home = dirname(dirname(__file__))
41
42 """ The VM that will be run by the 'vm' command and built by default by the 'build' command.
43 This can be set via the global '--vm' option or the DEFAULT_VM environment variable.
44 It can also be temporarily set by using of a VM context manager object in a 'with' statement. """
45 _vm = None
46
47 _make_eclipse_launch = False
48
49 _minVersion = mx.VersionSpec('1.7')
50
51 # max version (first _unsupported_ version)
52 _untilVersion = None
53
54 class JDKDeployedDist:
55 def __init__(self, name, isExtension=False, usesJVMCIClassLoader=False, partOfHotSpot=False):
56 self.name = name
57 self.isExtension = isExtension
58 self.usesJVMCIClassLoader = usesJVMCIClassLoader
59 self.partOfHotSpot = partOfHotSpot # true when this distribution is delivered with HotSpot
60
61 _jdkDeployedDists = [
62 JDKDeployedDist('TRUFFLE'),
63 ]
64
65 JDK_UNIX_PERMISSIONS_DIR = 0755
66 JDK_UNIX_PERMISSIONS_FILE = 0644
67 JDK_UNIX_PERMISSIONS_EXEC = 0755
68
69 def isVMSupported(vm):
70 if 'client' == vm and len(platform.mac_ver()[0]) != 0:
71 # Client VM not supported: java launcher on Mac OS X translates '-client' to '-server'
72 return False
73 return True
74
75 def _get_vm():
76 """
77 Gets the configured VM, presenting a dialogue if there is no currently configured VM.
78 """
79 global _vm
80 if _vm:
81 return _vm
82 vm = mx.get_env('DEFAULT_VM')
83 if vm is None:
84 extras = mx.get_env('EXTRA_JAVA_HOMES')
85 if not extras is None:
86 for e in extras.split(':'):
87 vm = e
88 break
89 envPath = join(_graal_home, 'mx', 'env')
90 if vm and 'graal' in vm:
91 if exists(envPath):
92 with open(envPath) as fp:
93 if 'DEFAULT_VM=' + vm in fp.read():
94 mx.log('Please update the DEFAULT_VM value in ' + envPath + ' to replace "graal" with "jvmci"')
95 vm = vm.replace('graal', 'jvmci')
96 if vm is None:
97 mx.abort('Need to specify VM with --vm option or DEFAULT_VM environment variable')
98 _vm = vm
99 return vm
100
101 def chmodRecursive(dirname, chmodFlagsDir):
102 if mx.get_os() == 'windows':
103 return
104
105 def _chmodDir(chmodFlags, dirname, fnames):
106 os.chmod(dirname, chmodFlagsDir)
107
108 os.path.walk(dirname, _chmodDir, chmodFlagsDir)
109
110 def clean(args):
111 """clean the source tree"""
112 opts = mx.clean(args, parser=ArgumentParser(prog='mx clean'))
113
114 if opts.native:
115 def handleRemoveReadonly(func, path, exc):
116 excvalue = exc[1]
117 if mx.get_os() == 'windows' and func in (os.rmdir, os.remove) and excvalue.errno == errno.EACCES:
118 os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # 0777
119 func(path)
120 else:
121 raise
122
123 def rmIfExists(name):
124 if os.path.isdir(name):
125 shutil.rmtree(name, ignore_errors=False, onerror=handleRemoveReadonly)
126 elif os.path.isfile(name):
127 os.unlink(name)
128
129 rmIfExists(join(_graal_home, 'build'))
130 rmIfExists(join(_graal_home, 'build-nojvmci'))
131
132 def export(args):
133 """create archives of builds split by vmbuild and vm"""
134
135 parser = ArgumentParser(prog='mx export')
136 args = parser.parse_args(args)
137
138 # collect data about export
139 infos = dict()
140 infos['timestamp'] = time.time()
141
142 hgcfg = mx.HgConfig()
143 hgcfg.check()
144 infos['revision'] = hgcfg.tip('.') + ('+' if hgcfg.isDirty('.') else '')
145 # TODO: infos['repository']
146
147 infos['jdkversion'] = str(mx.java().version)
148
149 infos['architecture'] = mx.get_arch()
150 infos['platform'] = mx.get_os()
151
152 if mx.get_os != 'windows':
153 pass
154 # infos['ccompiler']
155 # infos['linker']
156
157 infos['hostname'] = socket.gethostname()
158
159 def _writeJson(suffix, properties):
160 d = infos.copy()
161 for k, v in properties.iteritems():
162 assert not d.has_key(k)
163 d[k] = v
164
165 jsonFileName = 'export-' + suffix + '.json'
166 with open(jsonFileName, 'w') as f:
167 print >> f, json.dumps(d)
168 return jsonFileName
169
170
171 def _genFileName(archivtype, middle):
172 idPrefix = infos['revision'] + '_'
173 idSuffix = '.tar.gz'
174 return join(_graal_home, "graalvm_" + archivtype + "_" + idPrefix + middle + idSuffix)
175
176 # graal directory
177 graalDirTarName = _genFileName('classfiles', 'javac')
178 mx.logv("creating graal " + graalDirTarName)
179 with tarfile.open(graalDirTarName, 'w:gz') as tar:
180 for root, _, files in os.walk("graal"):
181 for f in [f for f in files if not f.endswith('.java')]:
182 name = join(root, f)
183 # print name
184 tar.add(name, name)
185
186 n = _writeJson("graal", {'javacompiler' : 'javac'})
187 tar.add(n, n)
188
189
190 def _run_benchmark(args, availableBenchmarks, runBenchmark):
191
192 vmOpts, benchmarksAndOptions = _extract_VM_args(args, useDoubleDash=availableBenchmarks is None)
193
194 if availableBenchmarks is None:
195 harnessArgs = benchmarksAndOptions
196 return runBenchmark(None, harnessArgs, vmOpts)
197
198 if len(benchmarksAndOptions) == 0:
199 mx.abort('at least one benchmark name or "all" must be specified')
200 benchmarks = list(itertools.takewhile(lambda x: not x.startswith('-'), benchmarksAndOptions))
201 harnessArgs = benchmarksAndOptions[len(benchmarks):]
202
203 if 'all' in benchmarks:
204 benchmarks = availableBenchmarks
205 else:
206 for bm in benchmarks:
207 if bm not in availableBenchmarks:
208 mx.abort('unknown benchmark: ' + bm + '\nselect one of: ' + str(availableBenchmarks))
209
210 failed = []
211 for bm in benchmarks:
212 if not runBenchmark(bm, harnessArgs, vmOpts):
213 failed.append(bm)
214
215 if len(failed) != 0:
216 mx.abort('Benchmark failures: ' + str(failed))
217
218 def _vmLibDirInJdk(jdk):
219 """
220 Get the directory within a JDK where the server and client
221 subdirectories are located.
222 """
223 mxos = mx.get_os()
224 if mxos == 'darwin':
225 return join(jdk, 'jre', 'lib')
226 if mxos == 'windows' or mxos == 'cygwin':
227 return join(jdk, 'jre', 'bin')
228 return join(jdk, 'jre', 'lib', mx.get_arch())
229
230 def _vmJliLibDirs(jdk):
231 """
232 Get the directories within a JDK where the jli library designates to.
233 """
234 mxos = mx.get_os()
235 if mxos == 'darwin':
236 return [join(jdk, 'jre', 'lib', 'jli')]
237 if mxos == 'windows' or mxos == 'cygwin':
238 return [join(jdk, 'jre', 'bin'), join(jdk, 'bin')]
239 return [join(jdk, 'jre', 'lib', mx.get_arch(), 'jli'), join(jdk, 'lib', mx.get_arch(), 'jli')]
240
241 def _vmCfgInJdk(jdk, jvmCfgFile='jvm.cfg'):
242 """
243 Get the jvm.cfg file.
244 """
245 mxos = mx.get_os()
246 if mxos == "windows" or mxos == "cygwin":
247 return join(jdk, 'jre', 'lib', mx.get_arch(), jvmCfgFile)
248 return join(_vmLibDirInJdk(jdk), jvmCfgFile)
249
250 def _jdksDir():
251 return os.path.abspath(join(_graal_home, 'jdk' + str(mx.java().version)))
252
253 def _handle_missing_VM(bld, vm=None):
254 if not vm:
255 vm = _get_vm()
256 mx.log('The ' + bld + ' ' + vm + ' VM has not been created')
257 mx.abort('You need to run "mx --javahome ' + vm + ' use the selected VM')
258
259 def _jdk(build=None, vmToCheck=None, create=False, installJars=True):
260 """
261 Get the JDK into which Graal is installed, creating it first if necessary.
262 """
263 jdk = join(_jdksDir(), build)
264 if create:
265 srcJdk = mx.java().jdk
266 if not exists(jdk):
267 mx.log('Creating ' + jdk + ' from ' + srcJdk)
268 shutil.copytree(srcJdk, jdk)
269
270 # Make a copy of the default VM so that this JDK can be
271 # reliably used as the bootstrap for a HotSpot build.
272 jvmCfg = _vmCfgInJdk(jdk)
273 if not exists(jvmCfg):
274 mx.abort(jvmCfg + ' does not exist')
275
276 defaultVM = None
277 jvmCfgLines = []
278 with open(jvmCfg) as f:
279 for line in f:
280 if line.startswith('-') and defaultVM is None:
281 parts = line.split()
282 if len(parts) == 2:
283 assert parts[1] == 'KNOWN', parts[1]
284 defaultVM = parts[0][1:]
285 jvmCfgLines += ['# default VM is a copy of the unmodified ' + defaultVM + ' VM\n']
286 jvmCfgLines += ['-original KNOWN\n']
287 else:
288 # skip lines which we cannot parse (e.g. '-hotspot ALIASED_TO -client')
289 mx.log("WARNING: skipping not parsable line \"" + line + "\"")
290 else:
291 jvmCfgLines += [line]
292
293 assert defaultVM is not None, 'Could not find default VM in ' + jvmCfg
294 chmodRecursive(jdk, JDK_UNIX_PERMISSIONS_DIR)
295 shutil.move(join(_vmLibDirInJdk(jdk), defaultVM), join(_vmLibDirInJdk(jdk), 'original'))
296
297 if mx.get_os() != 'windows':
298 os.chmod(jvmCfg, JDK_UNIX_PERMISSIONS_FILE)
299 with open(jvmCfg, 'w') as fp:
300 for line in jvmCfgLines:
301 fp.write(line)
302
303 # patch 'release' file (append graalvm revision)
304 releaseFile = join(jdk, 'release')
305 if exists(releaseFile):
306 releaseFileLines = []
307 with open(releaseFile) as f:
308 for line in f:
309 releaseFileLines.append(line)
310
311 if mx.get_os() != 'windows':
312 os.chmod(releaseFile, JDK_UNIX_PERMISSIONS_FILE)
313 with open(releaseFile, 'w') as fp:
314 for line in releaseFileLines:
315 if line.startswith("SOURCE="):
316 try:
317 sourceLine = line[0:-2] # remove last char
318 hgcfg = mx.HgConfig()
319 hgcfg.check()
320 revision = hgcfg.tip('.')[:12] # take first 12 chars
321 fp.write(sourceLine + ' graal:' + revision + '\"\n')
322 except:
323 fp.write(line)
324 else:
325 fp.write(line)
326
327 # Install a copy of the disassembler library
328 try:
329 hsdis([], copyToDir=_vmLibDirInJdk(jdk))
330 except SystemExit:
331 pass
332 else:
333 if not exists(jdk):
334 _handle_missing_VM(build, vmToCheck)
335
336 if installJars:
337 for jdkDist in _jdkDeployedDists:
338 dist = mx.distribution(jdkDist.name)
339 if exists(dist.path) and jdkDist.partOfHotSpot:
340 _installDistInJdks(jdkDist)
341
342 return jdk
343
344 def _updateInstalledJVMCIOptionsFile(jdk):
345 jvmciOptions = join(_graal_home, 'jvmci.options')
346 jreLibDir = join(jdk, 'jre', 'lib')
347 if exists(jvmciOptions):
348 shutil.copy(jvmciOptions, join(jreLibDir, 'jvmci.options'))
349 else:
350 toDelete = join(jreLibDir, 'jvmci.options')
351 if exists(toDelete):
352 os.unlink(toDelete)
353
354 def _makeHotspotGeneratedSourcesDir():
355 """
356 Gets the directory containing all the HotSpot sources generated from
357 JVMCI Java sources. This directory will be created if it doesn't yet exist.
358 """
359 hsSrcGenDir = join(mx.project('com.oracle.jvmci.hotspot').source_gen_dir(), 'hotspot')
360 if not exists(hsSrcGenDir):
361 os.makedirs(hsSrcGenDir)
362 return hsSrcGenDir
363
364 def _copyToJdk(src, dst, permissions=JDK_UNIX_PERMISSIONS_FILE):
365 name = os.path.basename(src)
366 dstLib = join(dst, name)
367 if mx.get_env('SYMLINK_GRAAL_JAR', None) == 'true':
368 # Using symlinks is much faster than copying but may
369 # cause issues if the lib is being updated while
370 # the VM is running.
371 if not os.path.islink(dstLib) or not os.path.realpath(dstLib) == src:
372 if exists(dstLib):
373 os.remove(dstLib)
374 os.symlink(src, dstLib)
375 else:
376 # do a copy and then a move to get atomic updating (on Unix)
377 fd, tmp = tempfile.mkstemp(suffix='', prefix=name, dir=dst)
378 shutil.copyfile(src, tmp)
379 os.close(fd)
380 shutil.move(tmp, dstLib)
381 os.chmod(dstLib, permissions)
382
383 def _filterJVMCIServices(servicesMap, classpath):
384 """
385 Filters and returns the names in 'serviceImplNames' that denote
386 types available in 'classpath' implementing or extending
387 com.oracle.jvmci.service.Service.
388 """
389 _, binDir = mx._compile_mx_class('FilterTypes', os.pathsep.join(classpath), myDir=dirname(__file__))
390 serialized = [k + '=' + ','.join(v) for k, v in servicesMap.iteritems()]
391 cmd = [mx.java().java, '-cp', mx._cygpathU2W(os.pathsep.join([binDir] + classpath)), 'FilterTypes', 'com.oracle.jvmci.service.Service'] + serialized
392 serialized = subprocess.check_output(cmd)
393 if len(serialized) == 0:
394 return {}
395 servicesMap = {}
396 for e in serialized.split(' '):
397 k, v = e.split('=')
398 impls = v.split(',')
399 servicesMap[k] = impls
400 return servicesMap
401
402 def _extractJVMCIFiles(jdkJars, jvmciJars, servicesDir, optionsDir, cleanDestination=True):
403 if cleanDestination:
404 if exists(servicesDir):
405 shutil.rmtree(servicesDir)
406 if exists(optionsDir):
407 shutil.rmtree(optionsDir)
408 if not exists(servicesDir):
409 os.makedirs(servicesDir)
410 if not exists(optionsDir):
411 os.makedirs(optionsDir)
412 servicesMap = {}
413 optionsFiles = []
414 for jar in jvmciJars:
415 if os.path.isfile(jar):
416 with zipfile.ZipFile(jar) as zf:
417 for member in zf.namelist():
418 if member.startswith('META-INF/services') and member:
419 serviceName = basename(member)
420 if serviceName == "":
421 continue # Zip files may contain empty entries for directories (jar -cf ... creates such)
422 # we don't handle directories
423 assert serviceName and member == 'META-INF/services/' + serviceName
424 with zf.open(member) as serviceFile:
425 serviceImpls = servicesMap.setdefault(serviceName, [])
426 for line in serviceFile.readlines():
427 line = line.strip()
428 if line:
429 serviceImpls.append(line)
430 elif member.startswith('META-INF/options'):
431 filename = basename(member)
432 if filename == "":
433 continue # Zip files may contain empty entries for directories (jar -cf ... creates such)
434 # we don't handle directories
435 assert filename and member == 'META-INF/options/' + filename
436 targetpath = join(optionsDir, filename)
437 optionsFiles.append(filename)
438 with zf.open(member) as optionsFile, \
439 file(targetpath, "wb") as target:
440 shutil.copyfileobj(optionsFile, target)
441 servicesMap = _filterJVMCIServices(servicesMap, jdkJars)
442 for serviceName, serviceImpls in servicesMap.iteritems():
443 fd, tmp = tempfile.mkstemp(prefix=serviceName)
444 f = os.fdopen(fd, 'w+')
445 for serviceImpl in serviceImpls:
446 f.write(serviceImpl + os.linesep)
447 target = join(servicesDir, serviceName)
448 f.close()
449 shutil.move(tmp, target)
450 if mx.get_os() != 'windows':
451 os.chmod(target, JDK_UNIX_PERMISSIONS_FILE)
452
453 def _updateJVMCIFiles(jdkDir):
454 jreJVMCIDir = join(jdkDir, 'jre', 'lib', 'jvmci')
455 jvmciJars = [join(jreJVMCIDir, e) for e in os.listdir(jreJVMCIDir) if e.endswith('.jar')]
456 jreJVMCIServicesDir = join(jreJVMCIDir, 'services')
457 jreJVMCIOptionsDir = join(jreJVMCIDir, 'options')
458 _extractJVMCIFiles(_getJdkDeployedJars(jdkDir), jvmciJars, jreJVMCIServicesDir, jreJVMCIOptionsDir)
459
460 def _patchGraalVersionConstant(dist):
461 """
462 Patches the constant "@@@@@@@@@@@@@@@@graal.version@@@@@@@@@@@@@@@@" in the constant pool of Graal.class
463 with the computed Graal version string.
464 """
465 zf = zipfile.ZipFile(dist.path, 'r')
466 graalClassfilePath = 'com/oracle/graal/api/runtime/Graal.class'
467 try:
468 graalClassfile = zf.read(graalClassfilePath)
469 except KeyError:
470 mx.log(graalClassfilePath + ' is not present in ' + dist.path)
471 return
472 placeholder = '@@@@@@@@@@@@@@@@graal.version@@@@@@@@@@@@@@@@'
473 placeholderLen = len(placeholder)
474 versionSpec = '{:' + str(placeholderLen) + '}'
475 versionStr = versionSpec.format(graal_version())
476
477 if len(versionStr) > placeholderLen:
478 # Truncate the version string if necessary
479 assert versionStr.startswith('unknown'), versionStr
480 versionStr = versionStr[:placeholderLen]
481 if placeholder not in graalClassfile:
482 assert versionStr in graalClassfile, 'could not find "' + placeholder + '" or "' + versionStr + '" constant in ' + dist.path + '!' + graalClassfilePath
483 zf.close()
484 return False
485
486 zfOutFd, zfOutPath = tempfile.mkstemp(suffix='', prefix=basename(dist.path) + '.', dir=dirname(dist.path))
487 zfOut = zipfile.ZipFile(zfOutPath, 'w')
488 for zi in zf.infolist():
489 if zi.filename == graalClassfilePath:
490 data = graalClassfile.replace(placeholder, versionStr)
491 else:
492 data = zf.read(zi)
493 zfOut.writestr(zi, data)
494 zfOut.close()
495 os.close(zfOutFd)
496 zf.close()
497 shutil.move(zfOutPath, dist.path)
498
499 def _installDistInJdks(deployableDist):
500 """
501 Installs the jar(s) for a given Distribution into all existing JVMCI JDKs
502 """
503 if True:
504 return
505 dist = mx.distribution(deployableDist.name)
506 if dist.name == 'GRAAL':
507 _patchGraalVersionConstant(dist)
508
509 jdks = _jdksDir()
510 if exists(jdks):
511 for e in os.listdir(jdks):
512 jdkDir = join(jdks, e)
513 jreLibDir = join(jdkDir, 'jre', 'lib')
514 if exists(jreLibDir):
515 if deployableDist.isExtension:
516 targetDir = join(jreLibDir, 'ext')
517 elif deployableDist.usesJVMCIClassLoader:
518 targetDir = join(jreLibDir, 'jvmci')
519 else:
520 targetDir = jreLibDir
521 if not exists(targetDir):
522 os.makedirs(targetDir)
523 _copyToJdk(dist.path, targetDir)
524 if dist.sourcesPath:
525 _copyToJdk(dist.sourcesPath, jdkDir)
526 if deployableDist.usesJVMCIClassLoader:
527 # deploy service files
528 _updateJVMCIFiles(jdkDir)
529
530 def _getJdkDeployedJars(jdkDir):
531 """
532 Gets jar paths for all deployed distributions in the context of
533 a given JDK directory.
534 """
535 jreLibDir = join(jdkDir, 'jre', 'lib')
536 jars = []
537 for dist in _jdkDeployedDists:
538 jar = basename(mx.distribution(dist.name).path)
539 if dist.isExtension:
540 jars.append(join(jreLibDir, 'ext', jar))
541 elif dist.usesJVMCIClassLoader:
542 jars.append(join(jreLibDir, 'jvmci', jar))
543 else:
544 jars.append(join(jreLibDir, jar))
545 return jars
546
547
548 # run a command in the windows SDK Debug Shell
549 def _runInDebugShell(cmd, workingDir, logFile=None, findInOutput=None, respondTo=None):
550 if respondTo is None:
551 respondTo = {}
552 newLine = os.linesep
553 startToken = 'RUNINDEBUGSHELL_STARTSEQUENCE'
554 endToken = 'RUNINDEBUGSHELL_ENDSEQUENCE'
555
556 winSDK = mx.get_env('WIN_SDK', 'C:\\Program Files\\Microsoft SDKs\\Windows\\v7.1\\')
557
558 if not exists(mx._cygpathW2U(winSDK)):
559 mx.abort("Could not find Windows SDK : '" + winSDK + "' does not exist")
560
561 winSDKSetEnv = mx._cygpathW2U(join(winSDK, 'Bin', 'SetEnv.cmd'))
562 if not exists(winSDKSetEnv):
563 mx.abort("Invalid Windows SDK path (" + winSDK + ") : could not find Bin/SetEnv.cmd (you can use the WIN_SDK environment variable to specify an other path)")
564
565 wincmd = 'cmd.exe /E:ON /V:ON /K "' + mx._cygpathU2W(winSDKSetEnv) + '"'
566 p = subprocess.Popen(wincmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
567 stdout = p.stdout
568 stdin = p.stdin
569 if logFile:
570 log = open(logFile, 'w')
571 ret = False
572
573 def _writeProcess(s):
574 stdin.write(s + newLine)
575
576 _writeProcess("echo " + startToken)
577 while True:
578 # encoding may be None on windows plattforms
579 if sys.stdout.encoding is None:
580 encoding = 'utf-8'
581 else:
582 encoding = sys.stdout.encoding
583
584 line = stdout.readline().decode(encoding)
585 if logFile:
586 log.write(line.encode('utf-8'))
587 line = line.strip()
588 mx.log(line)
589 if line == startToken:
590 _writeProcess('cd /D ' + workingDir + ' & ' + cmd + ' & echo ' + endToken)
591 for regex in respondTo.keys():
592 match = regex.search(line)
593 if match:
594 _writeProcess(respondTo[regex])
595 if findInOutput:
596 match = findInOutput.search(line)
597 if match:
598 ret = True
599 if line == endToken:
600 if not findInOutput:
601 _writeProcess('echo ERRXXX%errorlevel%')
602 else:
603 break
604 if line.startswith('ERRXXX'):
605 if line == 'ERRXXX0':
606 ret = True
607 break
608 _writeProcess("exit")
609 if logFile:
610 log.close()
611 return ret
612
613 def jdkhome(vm=None):
614 """return the JDK directory selected for the 'vm' command"""
615 return _jdk(installJars=False)
616
617 def print_jdkhome(args, vm=None):
618 """print the JDK directory selected for the 'vm' command"""
619 print jdkhome(vm)
620
621 def buildvars(args):
622 """describe the variables that can be set by the -D option to the 'mx build' commmand"""
623
624 buildVars = {
625 'ALT_BOOTDIR' : 'The location of the bootstrap JDK installation (default: ' + mx.java().jdk + ')',
626 'ALT_OUTPUTDIR' : 'Build directory',
627 'HOTSPOT_BUILD_JOBS' : 'Number of CPUs used by make (default: ' + str(mx.cpu_count()) + ')',
628 'INSTALL' : 'Install the built VM into the JDK? (default: y)',
629 'ZIP_DEBUGINFO_FILES' : 'Install zipped debug symbols file? (default: 0)',
630 }
631
632 mx.log('HotSpot build variables that can be set by the -D option to "mx build":')
633 mx.log('')
634 for n in sorted(buildVars.iterkeys()):
635 mx.log(n)
636 mx.log(textwrap.fill(buildVars[n], initial_indent=' ', subsequent_indent=' ', width=200))
637
638 mx.log('')
639 mx.log('Note that these variables can be given persistent values in the file ' + join(_graal_home, 'mx', 'env') + ' (see \'mx about\').')
640
641 cached_graal_version = None
642
643 def graal_version(dev_suffix='dev'):
644 global cached_graal_version
645
646 if not cached_graal_version:
647 # extract latest release tag for graal
648 try:
649 tags = [x.split() for x in subprocess.check_output(['hg', '-R', _graal_home, 'tags']).split('\n') if x.startswith("graal-")]
650 current_id = subprocess.check_output(['hg', '-R', _graal_home, 'log', '--template', '{rev}\n', '--rev', 'tip']).strip()
651 except:
652 # not a mercurial repository or hg commands are not available.
653 tags = None
654
655 if tags and current_id:
656 sorted_tags = sorted(tags, key=lambda e: [int(x) for x in e[0][len("graal-"):].split('.')], reverse=True)
657 most_recent_tag_name, most_recent_tag_revision = sorted_tags[0]
658 most_recent_tag_id = most_recent_tag_revision[:most_recent_tag_revision.index(":")]
659 most_recent_tag_version = most_recent_tag_name[len("graal-"):]
660
661 # tagged commit is one-off with commit that tags it
662 if int(current_id) - int(most_recent_tag_id) <= 1:
663 cached_graal_version = most_recent_tag_version
664 else:
665 major, minor = map(int, most_recent_tag_version.split('.'))
666 cached_graal_version = str(major) + '.' + str(minor + 1) + '-' + dev_suffix
667 else:
668 cached_graal_version = 'unknown-{0}'.format(platform.node())
669
670 return cached_graal_version
671
672 def build(args, vm=None):
673 """build the VM binary
674
675 The global '--vm' and '--vmbuild' options select which VM type and build target to build."""
676
677 # Override to fail quickly if extra arguments are given
678 # at the end of the command line. This allows for a more
679 # helpful error message.
680 class AP(ArgumentParser):
681 def __init__(self):
682 ArgumentParser.__init__(self, prog='mx build')
683 def parse_args(self, args):
684 result = ArgumentParser.parse_args(self, args)
685 if len(result.remainder) != 0:
686 firstBuildTarget = result.remainder[0]
687 mx.abort('To specify the ' + firstBuildTarget + ' VM build target, you need to use the global "--vmbuild" option. For example:\n' +
688 ' mx --vmbuild ' + firstBuildTarget + ' build')
689 return result
690
691 # Call mx.build to compile the Java sources
692 parser = AP()
693 parser.add_argument('-D', action='append', help='set a HotSpot build variable (run \'mx buildvars\' to list variables)', metavar='name=value')
694
695 opts2 = mx.build(['--source', '1.7'] + args, parser=parser)
696 assert len(opts2.remainder) == 0
697
698 def vmg(args):
699 """run the debug build of VM selected by the '--vm' option"""
700 return vm(args, vmbuild='debug')
701
702 def vmfg(args):
703 """run the fastdebug build of VM selected by the '--vm' option"""
704 return vm(args, vmbuild='fastdebug')
705
706 def _parseVmArgs(args, vm=None, cwd=None, vmbuild=None):
707 """run the VM selected by the '--vm' option"""
708
709 jdk = mx.java().jdk
710 mx.expand_project_in_args(args)
711 exe = join(jdk, 'bin', mx.exe_suffix('java'))
712 pfx = []
713
714 if '-version' in args:
715 ignoredArgs = args[args.index('-version') + 1:]
716 if len(ignoredArgs) > 0:
717 mx.log("Warning: The following options will be ignored by the vm because they come after the '-version' argument: " + ' '.join(ignoredArgs))
718
719 # Unconditionally prepend Truffle to the boot class path.
720 # This used to be done by the VM itself but was removed to
721 # separate the VM from Truffle.
722 truffle_jar = mx.archive(['@TRUFFLE'])[0]
723 args = ['-Xbootclasspath/p:' + truffle_jar] + args
724
725 args = mx.java().processArgs(args)
726 return (pfx, exe, vm, args, cwd)
727
728 def vm(args, vm=None, nonZeroIsFatal=True, out=None, err=None, cwd=None, timeout=None, vmbuild=None):
729 (pfx_, exe_, _, args_, cwd) = _parseVmArgs(args, vm, cwd, vmbuild)
730 return mx.run(pfx_ + [exe_] + args_, nonZeroIsFatal=nonZeroIsFatal, out=out, err=err, cwd=cwd)
731
732 def _find_classes_with_annotations(p, pkgRoot, annotations, includeInnerClasses=False):
733 """
734 Scan the sources of project 'p' for Java source files containing a line starting with 'annotation'
735 (ignoring preceding whitespace) and return the fully qualified class name for each Java
736 source file matched in a list.
737 """
738
739 matches = lambda line: len([a for a in annotations if line == a or line.startswith(a + '(')]) != 0
740 return p.find_classes_with_matching_source_line(pkgRoot, matches, includeInnerClasses)
741
742 def _extract_VM_args(args, allowClasspath=False, useDoubleDash=False, defaultAllVMArgs=True):
743 """
744 Partitions a command line into a leading sequence of HotSpot VM options and the rest.
745 """
746 for i in range(0, len(args)):
747 if useDoubleDash:
748 if args[i] == '--':
749 vmArgs = args[:i]
750 remainder = args[i + 1:]
751 return vmArgs, remainder
752 else:
753 if not args[i].startswith('-'):
754 if i != 0 and (args[i - 1] == '-cp' or args[i - 1] == '-classpath'):
755 if not allowClasspath:
756 mx.abort('Cannot supply explicit class path option')
757 else:
758 continue
759 vmArgs = args[:i]
760 remainder = args[i:]
761 return vmArgs, remainder
762
763 if defaultAllVMArgs:
764 return args, []
765 else:
766 return [], args
767
768 def _run_tests(args, harness, annotations, testfile, blacklist, whitelist, regex):
769
770
771 vmArgs, tests = _extract_VM_args(args)
772 for t in tests:
773 if t.startswith('-'):
774 mx.abort('VM option ' + t + ' must precede ' + tests[0])
775
776 candidates = {}
777 for p in mx.projects_opt_limit_to_suites():
778 if mx.java().javaCompliance < p.javaCompliance:
779 continue
780 for c in _find_classes_with_annotations(p, None, annotations).keys():
781 candidates[c] = p
782
783 classes = []
784 if len(tests) == 0:
785 classes = candidates.keys()
786 projectsCp = mx.classpath([pcp.name for pcp in mx.projects_opt_limit_to_suites() if pcp.javaCompliance <= mx.java().javaCompliance])
787 else:
788 projs = set()
789 found = False
790 if len(tests) == 1 and '#' in tests[0]:
791 words = tests[0].split('#')
792 if len(words) != 2:
793 mx.abort("Method specification is class#method: " + tests[0])
794 t, method = words
795
796 for c, p in candidates.iteritems():
797 # prefer exact matches first
798 if t == c:
799 found = True
800 classes.append(c)
801 projs.add(p.name)
802 if not found:
803 for c, p in candidates.iteritems():
804 if t in c:
805 found = True
806 classes.append(c)
807 projs.add(p.name)
808 if not found:
809 mx.log('warning: no tests matched by substring "' + t)
810 elif len(classes) != 1:
811 mx.abort('More than one test matches substring {0} {1}'.format(t, classes))
812
813 classes = [c + "#" + method for c in classes]
814 else:
815 for t in tests:
816 if '#' in t:
817 mx.abort('Method specifications can only be used in a single test: ' + t)
818 for c, p in candidates.iteritems():
819 if t in c:
820 found = True
821 classes.append(c)
822 projs.add(p.name)
823 if not found:
824 mx.log('warning: no tests matched by substring "' + t)
825 projectsCp = mx.classpath(projs)
826
827 if blacklist:
828 classes = [c for c in classes if not any((glob.match(c) for glob in blacklist))]
829
830 if whitelist:
831 classes = [c for c in classes if any((glob.match(c) for glob in whitelist))]
832
833 if regex:
834 classes = [c for c in classes if re.search(regex, c)]
835
836 if len(classes) != 0:
837 f_testfile = open(testfile, 'w')
838 for c in classes:
839 f_testfile.write(c + '\n')
840 f_testfile.close()
841 harness(projectsCp, vmArgs)
842
843 def _unittest(args, annotations, prefixCp="", blacklist=None, whitelist=None, verbose=False, fail_fast=False, enable_timing=False, regex=None, color=False, eager_stacktrace=False, gc_after_test=False):
844 testfile = os.environ.get('MX_TESTFILE', None)
845 if testfile is None:
846 (_, testfile) = tempfile.mkstemp(".testclasses", "graal")
847 os.close(_)
848
849 coreCp = mx.classpath(['com.oracle.truffle.tck', 'HCFDIS'])
850
851 coreArgs = []
852 if verbose:
853 coreArgs.append('-JUnitVerbose')
854 if fail_fast:
855 coreArgs.append('-JUnitFailFast')
856 if enable_timing:
857 coreArgs.append('-JUnitEnableTiming')
858 if color:
859 coreArgs.append('-JUnitColor')
860 if eager_stacktrace:
861 coreArgs.append('-JUnitEagerStackTrace')
862 if gc_after_test:
863 coreArgs.append('-JUnitGCAfterTest')
864
865
866 def harness(projectsCp, vmArgs):
867 if _get_vm() != 'jvmci':
868 prefixArgs = ['-esa', '-ea']
869 else:
870 prefixArgs = ['-XX:-BootstrapJVMCI', '-esa', '-ea']
871 if gc_after_test:
872 prefixArgs.append('-XX:-DisableExplicitGC')
873 with open(testfile) as fp:
874 testclasses = [l.rstrip() for l in fp.readlines()]
875
876 # Remove entries from class path that are in graal.jar and
877 # run the VM in a mode where application/test classes can
878 # access core Graal classes.
879 cp = prefixCp + coreCp + os.pathsep + projectsCp
880 if isJVMCIEnabled(_get_vm()):
881 excluded = set()
882 for jdkDist in _jdkDeployedDists:
883 dist = mx.distribution(jdkDist.name)
884 excluded.update([d.output_dir() for d in dist.sorted_deps()])
885 cp = os.pathsep.join([e for e in cp.split(os.pathsep) if e not in excluded])
886
887 # suppress menubar and dock when running on Mac
888 vmArgs = ['-Djava.awt.headless=true'] + vmArgs
889
890 if len(testclasses) == 1:
891 # Execute Junit directly when one test is being run. This simplifies
892 # replaying the VM execution in a native debugger (e.g., gdb).
893 vm(prefixArgs + vmArgs + ['-cp', mx._separatedCygpathU2W(cp), 'com.oracle.truffle.tck.TruffleJUnitCore'] + coreArgs + testclasses)
894 else:
895 vm(prefixArgs + vmArgs + ['-cp', mx._separatedCygpathU2W(cp), 'com.oracle.truffle.tck.TruffleJUnitCore'] + coreArgs + ['@' + mx._cygpathU2W(testfile)])
896
897 try:
898 _run_tests(args, harness, annotations, testfile, blacklist, whitelist, regex)
899 finally:
900 if os.environ.get('MX_TESTFILE') is None:
901 os.remove(testfile)
902
903 _unittestHelpSuffix = """
904 Unittest options:
905
906 --blacklist <file> run all testcases not specified in the blacklist
907 --whitelist <file> run only testcases which are included
908 in the given whitelist
909 --verbose enable verbose JUnit output
910 --fail-fast stop after first JUnit test class that has a failure
911 --enable-timing enable JUnit test timing
912 --regex <regex> run only testcases matching a regular expression
913 --color enable colors output
914 --eager-stacktrace print stacktrace eagerly
915 --gc-after-test force a GC after each test
916
917 To avoid conflicts with VM options '--' can be used as delimiter.
918
919 If filters are supplied, only tests whose fully qualified name
920 includes a filter as a substring are run.
921
922 For example, this command line:
923
924 mx unittest -G:Dump= -G:MethodFilter=BC_aload.* -G:+PrintCFG BC_aload
925
926 will run all JUnit test classes that contain 'BC_aload' in their
927 fully qualified name and will pass these options to the VM:
928
929 -G:Dump= -G:MethodFilter=BC_aload.* -G:+PrintCFG
930
931 To get around command line length limitations on some OSes, the
932 JUnit class names to be executed are written to a file that a
933 custom JUnit wrapper reads and passes onto JUnit proper. The
934 MX_TESTFILE environment variable can be set to specify a
935 file which will not be deleted once the unittests are done
936 (unlike the temporary file otherwise used).
937
938 As with all other commands, using the global '-v' before 'unittest'
939 command will cause mx to show the complete command line
940 it uses to run the VM.
941 """
942
943 def unittest(args):
944 """run the JUnit tests (all testcases){0}"""
945
946 parser = ArgumentParser(prog='mx unittest',
947 description='run the JUnit tests',
948 add_help=False,
949 formatter_class=RawDescriptionHelpFormatter,
950 epilog=_unittestHelpSuffix,
951 )
952 parser.add_argument('--blacklist', help='run all testcases not specified in the blacklist', metavar='<path>')
953 parser.add_argument('--whitelist', help='run testcases specified in whitelist only', metavar='<path>')
954 parser.add_argument('--verbose', help='enable verbose JUnit output', action='store_true')
955 parser.add_argument('--fail-fast', help='stop after first JUnit test class that has a failure', action='store_true')
956 parser.add_argument('--enable-timing', help='enable JUnit test timing', action='store_true')
957 parser.add_argument('--regex', help='run only testcases matching a regular expression', metavar='<regex>')
958 parser.add_argument('--color', help='enable color output', action='store_true')
959 parser.add_argument('--eager-stacktrace', help='print stacktrace eagerly', action='store_true')
960 parser.add_argument('--gc-after-test', help='force a GC after each test', action='store_true')
961
962 ut_args = []
963 delimiter = False
964 # check for delimiter
965 while len(args) > 0:
966 arg = args.pop(0)
967 if arg == '--':
968 delimiter = True
969 break
970 ut_args.append(arg)
971
972 if delimiter:
973 # all arguments before '--' must be recognized
974 parsed_args = parser.parse_args(ut_args)
975 else:
976 # parse all know arguments
977 parsed_args, args = parser.parse_known_args(ut_args)
978
979 if parsed_args.whitelist:
980 try:
981 with open(join(_graal_home, parsed_args.whitelist)) as fp:
982 parsed_args.whitelist = [re.compile(fnmatch.translate(l.rstrip())) for l in fp.readlines() if not l.startswith('#')]
983 except IOError:
984 mx.log('warning: could not read whitelist: ' + parsed_args.whitelist)
985 if parsed_args.blacklist:
986 try:
987 with open(join(_graal_home, parsed_args.blacklist)) as fp:
988 parsed_args.blacklist = [re.compile(fnmatch.translate(l.rstrip())) for l in fp.readlines() if not l.startswith('#')]
989 except IOError:
990 mx.log('warning: could not read blacklist: ' + parsed_args.blacklist)
991
992 _unittest(args, ['@Test', '@Parameters'], **parsed_args.__dict__)
993
994 def shortunittest(args):
995 """alias for 'unittest --whitelist test/whitelist_shortunittest.txt'{0}"""
996
997 args = ['--whitelist', 'test/whitelist_shortunittest.txt'] + args
998 unittest(args)
999
1000 class Task:
1001 # None or a list of strings. If not None, only tasks whose title
1002 # matches at least one of the substrings in this list will return
1003 # a non-None value from __enter__. The body of a 'with Task(...) as t'
1004 # statement should check 't' and exit immediately if it is None.
1005 filters = None
1006 filtersExclude = False
1007
1008 def __init__(self, title, tasks=None):
1009 self.tasks = tasks
1010 self.title = title
1011 if tasks is not None and Task.filters is not None:
1012 if Task.filtersExclude:
1013 self.skipped = any([f in title for f in Task.filters])
1014 else:
1015 self.skipped = not any([f in title for f in Task.filters])
1016 else:
1017 self.skipped = False
1018 if not self.skipped:
1019 self.start = time.time()
1020 self.end = None
1021 self.duration = None
1022 mx.log(time.strftime('gate: %d %b %Y %H:%M:%S: BEGIN: ') + title)
1023 def __enter__(self):
1024 assert self.tasks is not None, "using Task with 'with' statement requires to pass the tasks list in the constructor"
1025 if self.skipped:
1026 return None
1027 return self
1028 def __exit__(self, exc_type, exc_value, traceback):
1029 if not self.skipped:
1030 self.tasks.append(self.stop())
1031 def stop(self):
1032 self.end = time.time()
1033 self.duration = datetime.timedelta(seconds=self.end - self.start)
1034 mx.log(time.strftime('gate: %d %b %Y %H:%M:%S: END: ') + self.title + ' [' + str(self.duration) + ']')
1035 return self
1036 def abort(self, codeOrMessage):
1037 self.end = time.time()
1038 self.duration = datetime.timedelta(seconds=self.end - self.start)
1039 mx.log(time.strftime('gate: %d %b %Y %H:%M:%S: ABORT: ') + self.title + ' [' + str(self.duration) + ']')
1040 mx.abort(codeOrMessage)
1041 return self
1042
1043 def _basic_gate_body(args, tasks):
1044 # Run unit tests on server-hosted-jvmci
1045 with Task('UnitTests:hosted-product', tasks) as t:
1046 if t: unittest(['--enable-timing', '--verbose', '--fail-fast'])
1047
1048
1049 def gate(args, gate_body=_basic_gate_body):
1050 """run the tests used to validate a push
1051
1052 If this command exits with a 0 exit code, then the source code is in
1053 a state that would be accepted for integration into the main repository."""
1054
1055 parser = ArgumentParser(prog='mx gate')
1056 parser.add_argument('-j', '--omit-java-clean', action='store_false', dest='cleanJava', help='omit cleaning Java native code')
1057 parser.add_argument('-n', '--omit-native-clean', action='store_false', dest='cleanNative', help='omit cleaning and building native code')
1058 parser.add_argument('-i', '--omit-ide-clean', action='store_false', dest='cleanIde', help='omit cleaning the ide project files')
1059 parser.add_argument('-g', '--only-build-jvmci', action='store_false', dest='buildNonJVMCI', help='only build the JVMCI VM')
1060 parser.add_argument('-t', '--task-filter', help='comma separated list of substrings to select subset of tasks to be run')
1061 parser.add_argument('-x', action='store_true', help='makes --task-filter an exclusion instead of inclusion filter')
1062
1063 args = parser.parse_args(args)
1064
1065 if args.task_filter:
1066 Task.filters = args.task_filter.split(',')
1067 Task.filtersExclude = args.x
1068 elif args.x:
1069 mx.abort('-x option cannot be used without --task-filter option')
1070
1071 # Force
1072 if not mx._opts.strict_compliance:
1073 mx.log("[gate] forcing strict compliance")
1074 mx._opts.strict_compliance = True
1075
1076 tasks = []
1077 total = Task('Gate')
1078 try:
1079 with Task('Pylint', tasks) as t:
1080 if t: mx.pylint([])
1081
1082 def _clean(name='Clean'):
1083 with Task(name, tasks) as t:
1084 if t:
1085 cleanArgs = []
1086 if not args.cleanNative:
1087 cleanArgs.append('--no-native')
1088 if not args.cleanJava:
1089 cleanArgs.append('--no-java')
1090 clean(cleanArgs)
1091 _clean()
1092
1093 with Task('IDEConfigCheck', tasks) as t:
1094 if t:
1095 if args.cleanIde:
1096 mx.ideclean([])
1097 mx.ideinit([])
1098
1099 eclipse_exe = mx.get_env('ECLIPSE_EXE')
1100 if eclipse_exe is not None:
1101 with Task('CodeFormatCheck', tasks) as t:
1102 if t and mx.eclipseformat(['-e', eclipse_exe]) != 0:
1103 t.abort('Formatter modified files - run "mx eclipseformat", check in changes and repush')
1104
1105 with Task('Canonicalization Check', tasks) as t:
1106 if t:
1107 mx.log(time.strftime('%d %b %Y %H:%M:%S - Ensuring mx/projects files are canonicalized...'))
1108 if mx.canonicalizeprojects([]) != 0:
1109 t.abort('Rerun "mx canonicalizeprojects" and check-in the modified mx/projects files.')
1110
1111 if mx.get_env('JDT'):
1112 with Task('BuildJavaWithEcj', tasks):
1113 if t: build(['-p', '--no-native', '--jdt-warning-as-error'])
1114 _clean('CleanAfterEcjBuild')
1115
1116 with Task('BuildJavaWithJavac', tasks):
1117 if t: build(['-p', '--no-native', '--force-javac'])
1118
1119 with Task('Checkstyle', tasks) as t:
1120 if t and mx.checkstyle([]) != 0:
1121 t.abort('Checkstyle warnings were found')
1122
1123 with Task('Checkheaders', tasks) as t:
1124 if t and checkheaders([]) != 0:
1125 t.abort('Checkheaders warnings were found')
1126
1127 with Task('FindBugs', tasks) as t:
1128 if t and findbugs([]) != 0:
1129 t.abort('FindBugs warnings were found')
1130
1131 gate_body(args, tasks)
1132
1133 except KeyboardInterrupt:
1134 total.abort(1)
1135
1136 except BaseException as e:
1137 import traceback
1138 traceback.print_exc()
1139 total.abort(str(e))
1140
1141 total.stop()
1142
1143 mx.log('Gate task times:')
1144 for t in tasks:
1145 mx.log(' ' + str(t.duration) + '\t' + t.title)
1146 mx.log(' =======')
1147 mx.log(' ' + str(total.duration))
1148
1149 if args.task_filter:
1150 Task.filters = None
1151
1152 def _igvJdk():
1153 v8u20 = mx.VersionSpec("1.8.0_20")
1154 v8u40 = mx.VersionSpec("1.8.0_40")
1155 v8 = mx.VersionSpec("1.8")
1156 def _igvJdkVersionCheck(version):
1157 return version >= v8 and (version < v8u20 or version >= v8u40)
1158 return mx.java_version(_igvJdkVersionCheck, versionDescription='>= 1.8 and < 1.8.0u20 or >= 1.8.0u40').jdk
1159
1160 def _igvBuildEnv():
1161 # When the http_proxy environment variable is set, convert it to the proxy settings that ant needs
1162 env = dict(os.environ)
1163 proxy = os.environ.get('http_proxy')
1164 if not (proxy is None) and len(proxy) > 0:
1165 if '://' in proxy:
1166 # Remove the http:// prefix (or any other protocol prefix)
1167 proxy = proxy.split('://', 1)[1]
1168 # Separate proxy server name and port number
1169 proxyName, proxyPort = proxy.split(':', 1)
1170 proxyEnv = '-DproxyHost="' + proxyName + '" -DproxyPort=' + proxyPort
1171 env['ANT_OPTS'] = proxyEnv
1172
1173 env['JAVA_HOME'] = _igvJdk()
1174 return env
1175
1176 def igv(args):
1177 """run the Ideal Graph Visualizer"""
1178 logFile = '.ideal_graph_visualizer.log'
1179 with open(join(_graal_home, logFile), 'w') as fp:
1180 mx.logv('[Ideal Graph Visualizer log is in ' + fp.name + ']')
1181 nbplatform = join(_graal_home, 'src', 'share', 'tools', 'IdealGraphVisualizer', 'nbplatform')
1182
1183 # Remove NetBeans platform if it is earlier than the current supported version
1184 if exists(nbplatform):
1185 updateTrackingFile = join(nbplatform, 'platform', 'update_tracking', 'org-netbeans-core.xml')
1186 if not exists(updateTrackingFile):
1187 mx.log('Could not find \'' + updateTrackingFile + '\', removing NetBeans platform')
1188 shutil.rmtree(nbplatform)
1189 else:
1190 dom = xml.dom.minidom.parse(updateTrackingFile)
1191 currentVersion = mx.VersionSpec(dom.getElementsByTagName('module_version')[0].getAttribute('specification_version'))
1192 supportedVersion = mx.VersionSpec('3.43.1')
1193 if currentVersion < supportedVersion:
1194 mx.log('Replacing NetBeans platform version ' + str(currentVersion) + ' with version ' + str(supportedVersion))
1195 shutil.rmtree(nbplatform)
1196 elif supportedVersion < currentVersion:
1197 mx.log('Supported NetBeans version in igv command should be updated to ' + str(currentVersion))
1198
1199 if not exists(nbplatform):
1200 mx.logv('[This execution may take a while as the NetBeans platform needs to be downloaded]')
1201
1202 env = _igvBuildEnv()
1203 # make the jar for Batik 1.7 available.
1204 env['IGV_BATIK_JAR'] = mx.library('BATIK').get_path(True)
1205 if mx.run(['ant', '-f', mx._cygpathU2W(join(_graal_home, 'src', 'share', 'tools', 'IdealGraphVisualizer', 'build.xml')), '-l', mx._cygpathU2W(fp.name), 'run'], env=env, nonZeroIsFatal=False):
1206 mx.abort("IGV ant build & launch failed. Check '" + logFile + "'. You can also try to delete 'src/share/tools/IdealGraphVisualizer/nbplatform'.")
1207
1208 def maven_install_truffle(args):
1209 """install Truffle into your local Maven repository"""
1210 for name in ['TRUFFLE', 'TRUFFLE_TCK', 'TRUFFLE_DSL_PROCESSOR', 'TRUFFLE_SL']:
1211 mx.archive(["@" + name])
1212 path = mx._dists[name].path
1213 mx.run(['mvn', 'install:install-file', '-DgroupId=com.oracle', '-DartifactId=' + name.lower(), '-Dversion=' + graal_version('SNAPSHOT'), '-Dpackaging=jar', '-Dfile=' + path])
1214
1215 def c1visualizer(args):
1216 """run the Cl Compiler Visualizer"""
1217 libpath = join(_graal_home, 'lib')
1218 if mx.get_os() == 'windows':
1219 executable = join(libpath, 'c1visualizer', 'bin', 'c1visualizer.exe')
1220 else:
1221 executable = join(libpath, 'c1visualizer', 'bin', 'c1visualizer')
1222
1223 # Check whether the current C1Visualizer installation is the up-to-date
1224 if exists(executable) and not exists(mx.library('C1VISUALIZER_DIST').get_path(resolve=False)):
1225 mx.log('Updating C1Visualizer')
1226 shutil.rmtree(join(libpath, 'c1visualizer'))
1227
1228 archive = mx.library('C1VISUALIZER_DIST').get_path(resolve=True)
1229
1230 if not exists(executable):
1231 zf = zipfile.ZipFile(archive, 'r')
1232 zf.extractall(libpath)
1233
1234 if not exists(executable):
1235 mx.abort('C1Visualizer binary does not exist: ' + executable)
1236
1237 if mx.get_os() != 'windows':
1238 # Make sure that execution is allowed. The zip file does not always specfiy that correctly
1239 os.chmod(executable, 0777)
1240
1241 mx.run([executable])
1242
1243 def hsdis(args, copyToDir=None):
1244 """download the hsdis library
1245
1246 This is needed to support HotSpot's assembly dumping features.
1247 By default it downloads the Intel syntax version, use the 'att' argument to install AT&T syntax."""
1248 flavor = 'intel'
1249 if 'att' in args:
1250 flavor = 'att'
1251 if mx.get_arch() == "sparcv9":
1252 flavor = "sparcv9"
1253 lib = mx.add_lib_suffix('hsdis-' + mx.get_arch())
1254 path = join(_graal_home, 'lib', lib)
1255
1256 sha1s = {
1257 'att/hsdis-amd64.dll' : 'bcbd535a9568b5075ab41e96205e26a2bac64f72',
1258 'att/hsdis-amd64.so' : '58919ba085d4ef7a513f25bae75e7e54ee73c049',
1259 'intel/hsdis-amd64.dll' : '6a388372cdd5fe905c1a26ced614334e405d1f30',
1260 'intel/hsdis-amd64.so' : '844ed9ffed64fe9599638f29a8450c50140e3192',
1261 'intel/hsdis-amd64.dylib' : 'fdb13ef0d7d23d93dacaae9c98837bea0d4fc5a2',
1262 'sparcv9/hsdis-sparcv9.so': '970640a9af0bd63641f9063c11275b371a59ee60',
1263 }
1264
1265 flavoredLib = flavor + "/" + lib
1266 if flavoredLib not in sha1s:
1267 mx.logv("hsdis not supported on this plattform or architecture")
1268 return
1269
1270 if not exists(path):
1271 sha1 = sha1s[flavoredLib]
1272 sha1path = path + '.sha1'
1273 mx.download_file_with_sha1('hsdis', path, ['http://lafo.ssw.uni-linz.ac.at/hsdis/' + flavoredLib], sha1, sha1path, True, True, sources=False)
1274 if copyToDir is not None and exists(copyToDir):
1275 shutil.copy(path, copyToDir)
1276
1277 def hcfdis(args):
1278 """disassemble HexCodeFiles embedded in text files
1279
1280 Run a tool over the input files to convert all embedded HexCodeFiles
1281 to a disassembled format."""
1282
1283 parser = ArgumentParser(prog='mx hcfdis')
1284 parser.add_argument('-m', '--map', help='address to symbol map applied to disassembler output')
1285 parser.add_argument('files', nargs=REMAINDER, metavar='files...')
1286
1287 args = parser.parse_args(args)
1288
1289 path = mx.library('HCFDIS').get_path(resolve=True)
1290 mx.run_java(['-cp', path, 'com.oracle.max.hcfdis.HexCodeFileDis'] + args.files)
1291
1292 if args.map is not None:
1293 addressRE = re.compile(r'0[xX]([A-Fa-f0-9]+)')
1294 with open(args.map) as fp:
1295 lines = fp.read().splitlines()
1296 symbols = dict()
1297 for l in lines:
1298 addressAndSymbol = l.split(' ', 1)
1299 if len(addressAndSymbol) == 2:
1300 address, symbol = addressAndSymbol
1301 if address.startswith('0x'):
1302 address = long(address, 16)
1303 symbols[address] = symbol
1304 for f in args.files:
1305 with open(f) as fp:
1306 lines = fp.read().splitlines()
1307 updated = False
1308 for i in range(0, len(lines)):
1309 l = lines[i]
1310 for m in addressRE.finditer(l):
1311 sval = m.group(0)
1312 val = long(sval, 16)
1313 sym = symbols.get(val)
1314 if sym:
1315 l = l.replace(sval, sym)
1316 updated = True
1317 lines[i] = l
1318 if updated:
1319 mx.log('updating ' + f)
1320 with open('new_' + f, "w") as fp:
1321 for l in lines:
1322 print >> fp, l
1323
1324 def sl(args):
1325 """run an SL program"""
1326 vmArgs, slArgs = _extract_VM_args(args)
1327 vm(vmArgs + ['-cp', mx.classpath(["TRUFFLE", "com.oracle.truffle.sl"]), "com.oracle.truffle.sl.SLLanguage"] + slArgs)
1328
1329 def sldebug(args):
1330 """run a simple command line debugger for the Simple Language"""
1331 vmArgs, slArgs = _extract_VM_args(args, useDoubleDash=True)
1332 vm(vmArgs + ['-cp', mx.classpath("com.oracle.truffle.sl.tools"), "com.oracle.truffle.sl.tools.debug.SLREPLServer"] + slArgs)
1333
1334 def isJVMCIEnabled(vm):
1335 return vm != 'original' and not vm.endswith('nojvmci')
1336
1337 def jol(args):
1338 """Java Object Layout"""
1339 joljar = mx.library('JOL_INTERNALS').get_path(resolve=True)
1340 candidates = mx.findclass(args, logToConsole=False, matcher=lambda s, classname: s == classname or classname.endswith('.' + s) or classname.endswith('$' + s))
1341
1342 if len(candidates) > 0:
1343 candidates = mx.select_items(sorted(candidates))
1344 else:
1345 # mx.findclass can be mistaken, don't give up yet
1346 candidates = args
1347
1348 vm(['-javaagent:' + joljar, '-cp', os.pathsep.join([mx.classpath(), joljar]), "org.openjdk.jol.MainObjectInternals"] + candidates)
1349
1350 def site(args):
1351 """create a website containing javadoc and the project dependency graph"""
1352
1353 return mx.site(['--name', 'Graal',
1354 '--jd', '@-tag', '--jd', '@test:X',
1355 '--jd', '@-tag', '--jd', '@run:X',
1356 '--jd', '@-tag', '--jd', '@bug:X',
1357 '--jd', '@-tag', '--jd', '@summary:X',
1358 '--jd', '@-tag', '--jd', '@vmoption:X',
1359 '--overview', join(_graal_home, 'graal', 'overview.html'),
1360 '--title', 'Graal OpenJDK Project Documentation',
1361 '--dot-output-base', 'projects'] + args)
1362
1363 def generateZshCompletion(args):
1364 """generate zsh completion for mx"""
1365 try:
1366 from genzshcomp import CompletionGenerator
1367 except ImportError:
1368 mx.abort("install genzshcomp (pip install genzshcomp)")
1369
1370 # need to fake module for the custom mx arg parser, otherwise a check in genzshcomp fails
1371 originalModule = mx._argParser.__module__
1372 mx._argParser.__module__ = "argparse"
1373 generator = CompletionGenerator("mx", mx._argParser)
1374 mx._argParser.__module__ = originalModule
1375
1376 # strip last line and define local variable "ret"
1377 complt = "\n".join(generator.get().split('\n')[0:-1]).replace('context state line', 'context state line ret=1')
1378
1379 # add array of possible subcommands (as they are not part of the argument parser)
1380 complt += '\n ": :->command" \\\n'
1381 complt += ' "*::args:->args" && ret=0\n'
1382 complt += '\n'
1383 complt += 'case $state in\n'
1384 complt += '\t(command)\n'
1385 complt += '\t\tlocal -a main_commands\n'
1386 complt += '\t\tmain_commands=(\n'
1387 for cmd in sorted(mx._commands.iterkeys()):
1388 c, _ = mx._commands[cmd][:2]
1389 doc = c.__doc__
1390 complt += '\t\t\t"{0}'.format(cmd)
1391 if doc:
1392 complt += ':{0}'.format(_fixQuotes(doc.split('\n', 1)[0]))
1393 complt += '"\n'
1394 complt += '\t\t)\n'
1395 complt += '\t\t_describe -t main_commands command main_commands && ret=0\n'
1396 complt += '\t\t;;\n'
1397
1398 complt += '\t(args)\n'
1399 # TODO: improve matcher: if mx args are given, this doesn't work
1400 complt += '\t\tcase $line[1] in\n'
1401 complt += '\t\t\t\tnoglob \\\n'
1402 complt += '\t\t\t\t\t_arguments -s -S \\\n'
1403 complt += _appendOptions("jvmci", r"G\:")
1404 # TODO: fix -XX:{-,+}Use* flags
1405 complt += _appendOptions("hotspot", r"XX\:")
1406 complt += '\t\t\t\t\t"-version" && ret=0 \n'
1407 complt += '\t\t\t\t;;\n'
1408 complt += '\t\tesac\n'
1409 complt += '\t\t;;\n'
1410 complt += 'esac\n'
1411 complt += '\n'
1412 complt += 'return $ret'
1413 print complt
1414
1415 def _fixQuotes(arg):
1416 return arg.replace('\"', '').replace('\'', '').replace('`', '').replace('{', '\\{').replace('}', '\\}').replace('[', '\\[').replace(']', '\\]')
1417
1418 def _appendOptions(optionType, optionPrefix):
1419 def isBoolean(vmap, field):
1420 return vmap[field] == "Boolean" or vmap[field] == "bool"
1421
1422 def hasDescription(vmap):
1423 return vmap['optDefault'] or vmap['optDoc']
1424
1425 complt = ""
1426 for vmap in _parseVMOptions(optionType):
1427 complt += '\t\t\t\t\t-"'
1428 complt += optionPrefix
1429 if isBoolean(vmap, 'optType'):
1430 complt += '"{-,+}"'
1431 complt += vmap['optName']
1432 if not isBoolean(vmap, 'optType'):
1433 complt += '='
1434 if hasDescription(vmap):
1435 complt += "["
1436 if vmap['optDefault']:
1437 complt += r"(default\: " + vmap['optDefault'] + ")"
1438 if vmap['optDoc']:
1439 complt += _fixQuotes(vmap['optDoc'])
1440 if hasDescription(vmap):
1441 complt += "]"
1442 complt += '" \\\n'
1443 return complt
1444
1445 def _parseVMOptions(optionType):
1446 parser = OutputParser()
1447 # TODO: the optDoc part can wrapped accross multiple lines, currently only the first line will be captured
1448 # TODO: fix matching for float literals
1449 jvmOptions = re.compile(
1450 r"^[ \t]*"
1451 r"(?P<optType>(Boolean|Integer|Float|Double|String|bool|intx|uintx|ccstr|double)) "
1452 r"(?P<optName>[a-zA-Z0-9]+)"
1453 r"[ \t]+=[ \t]*"
1454 r"(?P<optDefault>([\-0-9]+(\.[0-9]+(\.[0-9]+\.[0-9]+))?|false|true|null|Name|sun\.boot\.class\.path))?"
1455 r"[ \t]*"
1456 r"(?P<optDoc>.+)?",
1457 re.MULTILINE)
1458 parser.addMatcher(ValuesMatcher(jvmOptions, {
1459 'optType' : '<optType>',
1460 'optName' : '<optName>',
1461 'optDefault' : '<optDefault>',
1462 'optDoc' : '<optDoc>',
1463 }))
1464
1465 # gather JVMCI options
1466 output = StringIO.StringIO()
1467 vm(['-XX:-BootstrapJVMCI', '-XX:+UnlockDiagnosticVMOptions', '-G:+PrintFlags' if optionType == "jvmci" else '-XX:+PrintFlagsWithComments'],
1468 vm="jvmci",
1469 vmbuild="optimized",
1470 nonZeroIsFatal=False,
1471 out=output.write,
1472 err=subprocess.STDOUT)
1473
1474 valueMap = parser.parse(output.getvalue())
1475 return valueMap
1476
1477 def findbugs(args):
1478 '''run FindBugs against non-test Java projects'''
1479 findBugsHome = mx.get_env('FINDBUGS_HOME', None)
1480 if findBugsHome:
1481 findbugsJar = join(findBugsHome, 'lib', 'findbugs.jar')
1482 else:
1483 findbugsLib = join(_graal_home, 'lib', 'findbugs-3.0.0')
1484 if not exists(findbugsLib):
1485 tmp = tempfile.mkdtemp(prefix='findbugs-download-tmp', dir=_graal_home)
1486 try:
1487 findbugsDist = mx.library('FINDBUGS_DIST').get_path(resolve=True)
1488 with zipfile.ZipFile(findbugsDist) as zf:
1489 candidates = [e for e in zf.namelist() if e.endswith('/lib/findbugs.jar')]
1490 assert len(candidates) == 1, candidates
1491 libDirInZip = os.path.dirname(candidates[0])
1492 zf.extractall(tmp)
1493 shutil.copytree(join(tmp, libDirInZip), findbugsLib)
1494 finally:
1495 shutil.rmtree(tmp)
1496 findbugsJar = join(findbugsLib, 'findbugs.jar')
1497 assert exists(findbugsJar)
1498 nonTestProjects = [p for p in mx.projects() if not p.name.endswith('.test') and not p.name.endswith('.jtt')]
1499 outputDirs = map(mx._cygpathU2W, [p.output_dir() for p in nonTestProjects])
1500 javaCompliance = max([p.javaCompliance for p in nonTestProjects])
1501 findbugsResults = join(_graal_home, 'findbugs.results')
1502
1503 cmd = ['-jar', mx._cygpathU2W(findbugsJar), '-textui', '-low', '-maxRank', '15']
1504 if mx.is_interactive():
1505 cmd.append('-progress')
1506 cmd = cmd + ['-auxclasspath', mx._separatedCygpathU2W(mx.classpath([d.name for d in _jdkDeployedDists] + [p.name for p in nonTestProjects])), '-output', mx._cygpathU2W(findbugsResults), '-exitcode'] + args + outputDirs
1507 exitcode = mx.run_java(cmd, nonZeroIsFatal=False, javaConfig=mx.java(javaCompliance))
1508 if exitcode != 0:
1509 with open(findbugsResults) as fp:
1510 mx.log(fp.read())
1511 os.unlink(findbugsResults)
1512 return exitcode
1513
1514 def checkheaders(args):
1515 """check Java source headers against any required pattern"""
1516 failures = {}
1517 for p in mx.projects():
1518 if p.native:
1519 continue
1520
1521 csConfig = join(mx.project(p.checkstyleProj).dir, '.checkstyle_checks.xml')
1522 dom = xml.dom.minidom.parse(csConfig)
1523 for module in dom.getElementsByTagName('module'):
1524 if module.getAttribute('name') == 'RegexpHeader':
1525 for prop in module.getElementsByTagName('property'):
1526 if prop.getAttribute('name') == 'header':
1527 value = prop.getAttribute('value')
1528 matcher = re.compile(value, re.MULTILINE)
1529 for sourceDir in p.source_dirs():
1530 for root, _, files in os.walk(sourceDir):
1531 for name in files:
1532 if name.endswith('.java') and name != 'package-info.java':
1533 f = join(root, name)
1534 with open(f) as fp:
1535 content = fp.read()
1536 if not matcher.match(content):
1537 failures[f] = csConfig
1538 for n, v in failures.iteritems():
1539 mx.log('{0}: header does not match RegexpHeader defined in {1}'.format(n, v))
1540 return len(failures)
1541
1542 def mx_init(suite):
1543 commands = {
1544 'checkheaders': [checkheaders, ''],
1545 'clean': [clean, ''],
1546 'findbugs': [findbugs, ''],
1547 'generateZshCompletion' : [generateZshCompletion, ''],
1548 'maven-install-truffle' : [maven_install_truffle, ''],
1549 'jdkhome': [print_jdkhome, ''],
1550 'gate' : [gate, '[-options]'],
1551 'unittest' : [unittest, '[unittest options] [--] [VM options] [filters...]', _unittestHelpSuffix],
1552 'shortunittest' : [shortunittest, '[unittest options] [--] [VM options] [filters...]', _unittestHelpSuffix],
1553 'site' : [site, '[-options]'],
1554 'sl' : [sl, '[SL args|@VM options]'],
1555 'sldebug' : [sldebug, '[SL args|@VM options]'],
1556 'jol' : [jol, ''],
1557 }
1558
1559 mx.update_commands(suite, commands)
1560
1561 def mx_post_parse_cmd_line(opts): #
1562 # TODO _minVersion check could probably be part of a Suite in mx?
1563 if mx.java().version < _minVersion:
1564 mx.abort('Requires Java version ' + str(_minVersion) + ' or greater for JAVA_HOME, got version ' + str(mx.java().version))
1565 if _untilVersion and mx.java().version >= _untilVersion:
1566 mx.abort('Requires Java version strictly before ' + str(_untilVersion) + ' for JAVA_HOME, got version ' + str(mx.java().version))
1567
1568 for jdkDist in _jdkDeployedDists:
1569 def _close(jdkDeployable):
1570 def _install(dist):
1571 assert dist.name == jdkDeployable.name, dist.name + "!=" + jdkDeployable.name
1572 if not jdkDist.partOfHotSpot:
1573 _installDistInJdks(jdkDeployable)
1574 return _install
1575 mx.distribution(jdkDist.name).add_update_listener(_close(jdkDist))