Mercurial > hg > graal-jvmci-8

comparison mx.jvmci/mx_jvmci.py @ 22139:ed35cb998428

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Initial split off from monolithic basic-graal repo
author Doug Simon <doug.simon@oracle.com>
date Mon, 06 Jul 2015 14:10:14 +0200
parents mx.graal/mx_graal.py@51ceda0cf404
children 60d9e50d5481
comparison
equal deleted inserted replaced
22138:103f53747955 22139:ed35cb998428
1 #
2 # ----------------------------------------------------------------------------------------------------
3 #
4 # Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved.
5 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
6 #
7 # This code is free software; you can redistribute it and/or modify it
8 # under the terms of the GNU General Public License version 2 only, as
9 # published by the Free Software Foundation.
10 #
11 # This code is distributed in the hope that it will be useful, but WITHOUT
12 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
13 # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
14 # version 2 for more details (a copy is included in the LICENSE file that
15 # accompanied this code).
16 #
17 # You should have received a copy of the GNU General Public License version
18 # 2 along with this work; if not, write to the Free Software Foundation,
19 # Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
20 #
21 # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
22 # or visit www.oracle.com if you need additional information or have any
23 # questions.
24 #
25 # ----------------------------------------------------------------------------------------------------
26
27 import os, stat, errno, sys, shutil, zipfile, tarfile, tempfile, re, time, datetime, platform, subprocess, socket
28 from os.path import join, exists, dirname, basename
29 from argparse import ArgumentParser, REMAINDER
30 import xml.dom.minidom
31 import json, textwrap
32
33 import mx
34 import mx_unittest
35 import mx_findbugs
36 import mx_jvmci_makefile
37
38 _suite = mx.suite('jvmci')
39
40 """ The VMs that can be built and run along with an optional description. Only VMs with a
41 description are listed in the dialogue for setting the default VM (see get_vm()). """
42 _vmChoices = {
43 'jvmci' : 'VM triggered compilation is performed with a tiered system (C1 + Graal) and Graal is available for hosted compilation.',
44 'server' : 'Normal compilation is performed with a tiered system (C1 + C2) and Graal is available for hosted compilation.',
45 'client' : None, # VM compilation with client compiler, hosted compilation with Graal
46 'server-nojvmci' : None, # all compilation with tiered system (i.e., client + server), JVMCI omitted
47 'client-nojvmci' : None, # all compilation with client compiler, JVMCI omitted
48 'original' : None, # default VM copied from bootstrap JDK
49 'graal' : None, # alias for jvmci
50 'server-nograal' : None, # alias for server-nojvmci
51 'client-nograal' : None, # alias for client-nojvmci
52 }
53
54 """ The VM that will be run by the 'vm' command and built by default by the 'build' command.
55 This can be set via the global '--vm' option or the DEFAULT_VM environment variable.
56 It can also be temporarily set by using of a VM context manager object in a 'with' statement. """
57 _vm = None
58
59 """ The VM builds that will be run by the 'vm' command - default is first in list """
60 _vmbuildChoices = ['product', 'fastdebug', 'debug', 'optimized']
61
62 """ The VM build that will be run by the 'vm' command.
63 This can be set via the global '--vmbuild' option.
64 It can also be temporarily set by using of a VM context manager object in a 'with' statement. """
65 _vmbuild = _vmbuildChoices[0]
66
67 _jacoco = 'off'
68
69 """ The current working directory to switch to before running the VM. """
70 _vm_cwd = None
71
72 """ The base directory in which the JDKs cloned from $JAVA_HOME exist. """
73 _installed_jdks = None
74
75 """ Prefix for running the VM. """
76 _vm_prefix = None
77
78 _make_eclipse_launch = False
79
80 _minVersion = mx.VersionSpec('1.8')
81
82 # max version (first _unsupported_ version)
83 _untilVersion = None
84
85 class JDKDeployedDist:
86 def __init__(self, name, isExtension=False, usesJVMCIClassLoader=False, partOfHotSpot=False):
87 self.name = name
88 self.isExtension = isExtension
89 self.usesJVMCIClassLoader = usesJVMCIClassLoader
90 self.partOfHotSpot = partOfHotSpot # true when this distribution is delivered with HotSpot
91
92 # A hook called after the jar(s) for this dist have been deployed into a JDK. If not None,
93 # this hook is called with these arguments:
94 # jdkDir: the root directory of the JDK
95 # targetDir: the directory into which the jar(s) were installed
96 self.postJdkInstall = None
97
98 """
99 List of distributions that are deployed into a JDK by mx.
100 """
101 jdkDeployedDists = [
102 JDKDeployedDist('JVMCI_SERVICE', partOfHotSpot=True),
103 JDKDeployedDist('JVMCI_API', usesJVMCIClassLoader=True, partOfHotSpot=True),
104 JDKDeployedDist('JVMCI_HOTSPOT', usesJVMCIClassLoader=True, partOfHotSpot=True),
105 ]
106
107 JDK_UNIX_PERMISSIONS_DIR = 0755
108 JDK_UNIX_PERMISSIONS_FILE = 0644
109 JDK_UNIX_PERMISSIONS_EXEC = 0755
110
111 def isVMSupported(vm):
112 if 'client' == vm and len(platform.mac_ver()[0]) != 0:
113 # Client VM not supported: java launcher on Mac OS X translates '-client' to '-server'
114 return False
115 return True
116
117 def get_vm():
118 """
119 Gets the configured VM, presenting a dialogue if there is no currently configured VM.
120 """
121 global _vm
122 if _vm:
123 return _vm
124 vm = mx.get_env('DEFAULT_VM')
125 envPath = join(_suite.mxDir, 'env')
126 if vm and 'graal' in vm:
127 if exists(envPath):
128 with open(envPath) as fp:
129 if 'DEFAULT_VM=' + vm in fp.read():
130 mx.log('Please update the DEFAULT_VM value in ' + envPath + ' to replace "graal" with "jvmci"')
131 vm = vm.replace('graal', 'jvmci')
132 if vm is None:
133 if not mx.is_interactive():
134 mx.abort('Need to specify VM with --vm option or DEFAULT_VM environment variable')
135 mx.log('Please select the VM to be executed from the following: ')
136 items = [k for k in _vmChoices.keys() if _vmChoices[k] is not None]
137 descriptions = [_vmChoices[k] for k in _vmChoices.keys() if _vmChoices[k] is not None]
138 vm = mx.select_items(items, descriptions, allowMultiple=False)
139 mx.ask_persist_env('DEFAULT_VM', vm)
140 _vm = vm
141 return vm
142
143 """
144 A context manager that can be used with the 'with' statement to set the VM
145 used by all VM executions within the scope of the 'with' statement. For example:
146
147 with VM('server'):
148 dacapo(['pmd'])
149 """
150 class VM:
151 def __init__(self, vm=None, build=None):
152 assert vm is None or vm in _vmChoices.keys()
153 assert build is None or build in _vmbuildChoices
154 self.vm = vm if vm else _vm
155 self.build = build if build else _vmbuild
156 self.previousVm = _vm
157 self.previousBuild = _vmbuild
158
159 def __enter__(self):
160 global _vm, _vmbuild
161 _vm = self.vm
162 _vmbuild = self.build
163
164 def __exit__(self, exc_type, exc_value, traceback):
165 global _vm, _vmbuild
166 _vm = self.previousVm
167 _vmbuild = self.previousBuild
168
169 def chmodRecursive(dirname, chmodFlagsDir):
170 if mx.get_os() == 'windows':
171 return
172
173 def _chmodDir(chmodFlags, dirname, fnames):
174 os.chmod(dirname, chmodFlagsDir)
175
176 os.path.walk(dirname, _chmodDir, chmodFlagsDir)
177
178 def clean(args):
179 """clean the source tree"""
180 opts = mx.clean(args, parser=ArgumentParser(prog='mx clean'))
181
182 if opts.native:
183 def handleRemoveReadonly(func, path, exc):
184 excvalue = exc[1]
185 if mx.get_os() == 'windows' and func in (os.rmdir, os.remove) and excvalue.errno == errno.EACCES:
186 os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # 0777
187 func(path)
188 else:
189 raise
190
191 def rmIfExists(name):
192 if os.path.isdir(name):
193 shutil.rmtree(name, ignore_errors=False, onerror=handleRemoveReadonly)
194 elif os.path.isfile(name):
195 os.unlink(name)
196
197 rmIfExists(join(_suite.dir, 'build'))
198 rmIfExists(join(_suite.dir, 'build-nojvmci'))
199 rmIfExists(_jdksDir())
200
201 def export(args):
202 """create archives of builds split by vmbuild and vm"""
203
204 parser = ArgumentParser(prog='mx export')
205 args = parser.parse_args(args)
206
207 # collect data about export
208 infos = dict()
209 infos['timestamp'] = time.time()
210
211 hgcfg = mx.HgConfig()
212 hgcfg.check()
213 infos['revision'] = hgcfg.tip('.') + ('+' if hgcfg.isDirty('.') else '')
214 # TODO: infos['repository']
215
216 infos['jdkversion'] = str(mx.java().version)
217
218 infos['architecture'] = mx.get_arch()
219 infos['platform'] = mx.get_os()
220
221 if mx.get_os != 'windows':
222 pass
223 # infos['ccompiler']
224 # infos['linker']
225
226 infos['hostname'] = socket.gethostname()
227
228 def _writeJson(suffix, properties):
229 d = infos.copy()
230 for k, v in properties.iteritems():
231 assert not d.has_key(k)
232 d[k] = v
233
234 jsonFileName = 'export-' + suffix + '.json'
235 with open(jsonFileName, 'w') as f:
236 print >> f, json.dumps(d)
237 return jsonFileName
238
239
240 def _genFileName(archivtype, middle):
241 idPrefix = infos['revision'] + '_'
242 idSuffix = '.tar.gz'
243 return join(_suite.dir, "graalvm_" + archivtype + "_" + idPrefix + middle + idSuffix)
244
245 def _genFileArchPlatformName(archivtype, middle):
246 return _genFileName(archivtype, infos['platform'] + '_' + infos['architecture'] + '_' + middle)
247
248
249 # archive different build types of hotspot
250 for vmBuild in _vmbuildChoices:
251 jdkpath = join(_jdksDir(), vmBuild)
252 if not exists(jdkpath):
253 mx.logv("skipping " + vmBuild)
254 continue
255
256 tarName = _genFileArchPlatformName('basejdk', vmBuild)
257 mx.logv("creating basejdk " + tarName)
258 vmSet = set()
259 with tarfile.open(tarName, 'w:gz') as tar:
260 for root, _, files in os.walk(jdkpath):
261 if basename(root) in _vmChoices.keys():
262 # TODO: add some assert to check path assumption
263 vmSet.add(root)
264 continue
265
266 for f in files:
267 name = join(root, f)
268 # print name
269 tar.add(name, name)
270
271 n = _writeJson("basejdk-" + vmBuild, {'vmbuild' : vmBuild})
272 tar.add(n, n)
273
274 # create a separate archive for each VM
275 for vm in vmSet:
276 bVm = basename(vm)
277 vmTarName = _genFileArchPlatformName('vm', vmBuild + '_' + bVm)
278 mx.logv("creating vm " + vmTarName)
279
280 debugFiles = set()
281 with tarfile.open(vmTarName, 'w:gz') as tar:
282 for root, _, files in os.walk(vm):
283 for f in files:
284 # TODO: mac, windows, solaris?
285 if any(map(f.endswith, [".debuginfo"])):
286 debugFiles.add(f)
287 else:
288 name = join(root, f)
289 # print name
290 tar.add(name, name)
291
292 n = _writeJson("vm-" + vmBuild + "-" + bVm, {'vmbuild' : vmBuild, 'vm' : bVm})
293 tar.add(n, n)
294
295 if len(debugFiles) > 0:
296 debugTarName = _genFileArchPlatformName('debugfilesvm', vmBuild + '_' + bVm)
297 mx.logv("creating debugfilesvm " + debugTarName)
298 with tarfile.open(debugTarName, 'w:gz') as tar:
299 for f in debugFiles:
300 name = join(root, f)
301 # print name
302 tar.add(name, name)
303
304 n = _writeJson("debugfilesvm-" + vmBuild + "-" + bVm, {'vmbuild' : vmBuild, 'vm' : bVm})
305 tar.add(n, n)
306
307 # jvmci directory
308 jvmciDirTarName = _genFileName('classfiles', 'javac')
309 mx.logv("creating jvmci " + jvmciDirTarName)
310 with tarfile.open(jvmciDirTarName, 'w:gz') as tar:
311 for root, _, files in os.walk("jvmci"):
312 for f in [f for f in files if not f.endswith('.java')]:
313 name = join(root, f)
314 # print name
315 tar.add(name, name)
316
317 n = _writeJson("jvmci", {'javacompiler' : 'javac'})
318 tar.add(n, n)
319
320 def _vmLibDirInJdk(jdk):
321 """
322 Gets the directory within a JDK where the server and client
323 sub-directories are located.
324 """
325 mxos = mx.get_os()
326 if mxos == 'darwin':
327 return join(jdk, 'jre', 'lib')
328 if mxos == 'windows' or mxos == 'cygwin':
329 return join(jdk, 'jre', 'bin')
330 return join(jdk, 'jre', 'lib', mx.get_arch())
331
332 def _vmJliLibDirs(jdk):
333 """
334 Get the directories within a JDK where the jli library designates to.
335 """
336 mxos = mx.get_os()
337 if mxos == 'darwin':
338 return [join(jdk, 'jre', 'lib', 'jli')]
339 if mxos == 'windows' or mxos == 'cygwin':
340 return [join(jdk, 'jre', 'bin'), join(jdk, 'bin')]
341 return [join(jdk, 'jre', 'lib', mx.get_arch(), 'jli'), join(jdk, 'lib', mx.get_arch(), 'jli')]
342
343 def _vmCfgInJdk(jdk, jvmCfgFile='jvm.cfg'):
344 """
345 Get the jvm.cfg file.
346 """
347 mxos = mx.get_os()
348 if mxos == "windows" or mxos == "cygwin":
349 return join(jdk, 'jre', 'lib', mx.get_arch(), jvmCfgFile)
350 return join(_vmLibDirInJdk(jdk), jvmCfgFile)
351
352 def _jdksDir():
353 return os.path.abspath(join(_installed_jdks if _installed_jdks else _suite.dir, 'jdk' + str(mx.java().version)))
354
355 def _handle_missing_VM(bld, vm=None):
356 if not vm:
357 vm = get_vm()
358 mx.log('The ' + bld + ' ' + vm + ' VM has not been created')
359 if mx.is_interactive():
360 if mx.ask_yes_no('Build it now', 'y'):
361 with VM(vm, bld):
362 build([])
363 return
364 mx.abort('You need to run "mx --vm ' + vm + ' --vmbuild ' + bld + ' build" to build the selected VM')
365
366 def _jdk(build=None, vmToCheck=None, create=False, installJars=True):
367 """
368 Get the JDK into which JVMCI is installed, creating it first if necessary.
369 """
370 if not build:
371 build = _vmbuild
372 jdk = join(_jdksDir(), build)
373 if create:
374 srcJdk = mx.java().jdk
375 if not exists(jdk):
376 mx.log('Creating ' + jdk + ' from ' + srcJdk)
377 shutil.copytree(srcJdk, jdk)
378
379 # Make a copy of the default VM so that this JDK can be
380 # reliably used as the bootstrap for a HotSpot build.
381 jvmCfg = _vmCfgInJdk(jdk)
382 if not exists(jvmCfg):
383 mx.abort(jvmCfg + ' does not exist')
384
385 defaultVM = None
386 jvmCfgLines = []
387 with open(jvmCfg) as f:
388 for line in f:
389 if line.startswith('-') and defaultVM is None:
390 parts = line.split()
391 if len(parts) == 2:
392 assert parts[1] == 'KNOWN', parts[1]
393 defaultVM = parts[0][1:]
394 jvmCfgLines += ['# default VM is a copy of the unmodified ' + defaultVM + ' VM\n']
395 jvmCfgLines += ['-original KNOWN\n']
396 else:
397 # skip lines which we cannot parse (e.g. '-hotspot ALIASED_TO -client')
398 mx.log("WARNING: skipping not parsable line \"" + line + "\"")
399 else:
400 jvmCfgLines += [line]
401
402 assert defaultVM is not None, 'Could not find default VM in ' + jvmCfg
403 chmodRecursive(jdk, JDK_UNIX_PERMISSIONS_DIR)
404 shutil.move(join(_vmLibDirInJdk(jdk), defaultVM), join(_vmLibDirInJdk(jdk), 'original'))
405
406 if mx.get_os() != 'windows':
407 os.chmod(jvmCfg, JDK_UNIX_PERMISSIONS_FILE)
408 with open(jvmCfg, 'w') as fp:
409 for line in jvmCfgLines:
410 fp.write(line)
411
412 # patch 'release' file (append jvmci revision)
413 releaseFile = join(jdk, 'release')
414 if exists(releaseFile):
415 releaseFileLines = []
416 with open(releaseFile) as f:
417 for line in f:
418 releaseFileLines.append(line)
419
420 if mx.get_os() != 'windows':
421 os.chmod(releaseFile, JDK_UNIX_PERMISSIONS_FILE)
422 with open(releaseFile, 'w') as fp:
423 for line in releaseFileLines:
424 if line.startswith("SOURCE="):
425 try:
426 sourceLine = line[0:-2] # remove last char
427 hgcfg = mx.HgConfig()
428 hgcfg.check()
429 revision = hgcfg.tip('.')[:12] # take first 12 chars
430 fp.write(sourceLine + ' jvmci:' + revision + '\"\n')
431 except:
432 fp.write(line)
433 else:
434 fp.write(line)
435
436 # Install a copy of the disassembler library
437 try:
438 hsdis([], copyToDir=_vmLibDirInJdk(jdk))
439 except SystemExit:
440 pass
441 else:
442 if not exists(jdk):
443 if _installed_jdks:
444 mx.log("The selected JDK directory does not (yet) exist: " + jdk)
445 _handle_missing_VM(build, vmToCheck)
446
447 if installJars:
448 for jdkDist in jdkDeployedDists:
449 dist = mx.distribution(jdkDist.name)
450 if exists(dist.path) and jdkDist.partOfHotSpot:
451 _installDistInJdks(jdkDist)
452
453 if vmToCheck is not None:
454 jvmCfg = _vmCfgInJdk(jdk)
455 found = False
456 with open(jvmCfg) as f:
457 for line in f:
458 if line.strip() == '-' + vmToCheck + ' KNOWN':
459 found = True
460 break
461 if not found:
462 _handle_missing_VM(build, vmToCheck)
463
464 return jdk
465
466 def _updateInstalledJVMCIOptionsFile(jdk):
467 jvmciOptions = join(_suite.dir, 'jvmci.options')
468 jreLibDir = join(jdk, 'jre', 'lib')
469 if exists(jvmciOptions):
470 shutil.copy(jvmciOptions, join(jreLibDir, 'jvmci.options'))
471 else:
472 toDelete = join(jreLibDir, 'jvmci.options')
473 if exists(toDelete):
474 os.unlink(toDelete)
475
476 def _makeHotspotGeneratedSourcesDir():
477 """
478 Gets the directory containing all the HotSpot sources generated from
479 JVMCI Java sources. This directory will be created if it doesn't yet exist.
480 """
481 hsSrcGenDir = join(mx.project('jdk.internal.jvmci.hotspot').source_gen_dir(), 'hotspot')
482 if not exists(hsSrcGenDir):
483 os.makedirs(hsSrcGenDir)
484 return hsSrcGenDir
485
486 def _copyToJdk(src, dst, permissions=JDK_UNIX_PERMISSIONS_FILE):
487 name = os.path.basename(src)
488 dstLib = join(dst, name)
489 if mx.get_env('SYMLINK_GRAAL_JAR', None) == 'true':
490 # Using symlinks is much faster than copying but may
491 # cause issues if the lib is being updated while
492 # the VM is running.
493 if not os.path.islink(dstLib) or not os.path.realpath(dstLib) == src:
494 if exists(dstLib):
495 os.remove(dstLib)
496 os.symlink(src, dstLib)
497 else:
498 # do a copy and then a move to get atomic updating (on Unix)
499 fd, tmp = tempfile.mkstemp(suffix='', prefix=name, dir=dst)
500 shutil.copyfile(src, tmp)
501 os.close(fd)
502 shutil.move(tmp, dstLib)
503 os.chmod(dstLib, permissions)
504
505 def _extractJVMCIFiles(jdkJars, jvmciJars, servicesDir, optionsDir):
506
507 oldServices = os.listdir(servicesDir) if exists(servicesDir) else os.makedirs(servicesDir)
508 oldOptions = os.listdir(optionsDir) if exists(optionsDir) else os.makedirs(optionsDir)
509
510 jvmciServices = {}
511 optionsFiles = []
512 for jar in jvmciJars:
513 if os.path.isfile(jar):
514 with zipfile.ZipFile(jar) as zf:
515 for member in zf.namelist():
516 if member.startswith('META-INF/jvmci.services/') and member != 'META-INF/jvmci.services/':
517 service = basename(member)
518 assert service != "", member
519 with zf.open(member) as serviceFile:
520 providers = jvmciServices.setdefault(service, [])
521 for line in serviceFile.readlines():
522 line = line.strip()
523 if line:
524 providers.append(line)
525 elif member.startswith('META-INF/jvmci.options/') and member != 'META-INF/jvmci.options/':
526 filename = basename(member)
527 assert filename != "", member
528 targetpath = join(optionsDir, filename)
529 optionsFiles.append(filename)
530 with zf.open(member) as optionsFile, \
531 file(targetpath, "wb") as target:
532 shutil.copyfileobj(optionsFile, target)
533 if oldOptions and filename in oldOptions:
534 oldOptions.remove(filename)
535 for service, providers in jvmciServices.iteritems():
536 fd, tmp = tempfile.mkstemp(prefix=service)
537 f = os.fdopen(fd, 'w+')
538 for provider in providers:
539 f.write(provider + os.linesep)
540 target = join(servicesDir, service)
541 f.close()
542 shutil.move(tmp, target)
543 if oldServices and service in oldServices:
544 oldServices.remove(service)
545 if mx.get_os() != 'windows':
546 os.chmod(target, JDK_UNIX_PERMISSIONS_FILE)
547
548 if mx.is_interactive():
549 for d, files in [(servicesDir, oldServices), (optionsDir, oldOptions)]:
550 if files and mx.ask_yes_no('These files in ' + d + ' look obsolete:\n ' + '\n '.join(files) + '\nDelete them', 'n'):
551 for f in files:
552 path = join(d, f)
553 os.remove(path)
554 mx.log('Deleted ' + path)
555
556 def _updateJVMCIFiles(jdkDir):
557 jreJVMCIDir = join(jdkDir, 'jre', 'lib', 'jvmci')
558 jvmciJars = [join(jreJVMCIDir, e) for e in os.listdir(jreJVMCIDir) if e.endswith('.jar')]
559 jreJVMCIServicesDir = join(jreJVMCIDir, 'services')
560 jreJVMCIOptionsDir = join(jreJVMCIDir, 'options')
561 _extractJVMCIFiles(_getJdkDeployedJars(jdkDir), jvmciJars, jreJVMCIServicesDir, jreJVMCIOptionsDir)
562
563 def _installDistInJdks(deployableDist):
564 """
565 Installs the jar(s) for a given Distribution into all existing JVMCI JDKs
566 """
567 dist = mx.distribution(deployableDist.name)
568 jdks = _jdksDir()
569 if exists(jdks):
570 for e in os.listdir(jdks):
571 jdkDir = join(jdks, e)
572 jreLibDir = join(jdkDir, 'jre', 'lib')
573 if exists(jreLibDir):
574 if deployableDist.isExtension:
575 targetDir = join(jreLibDir, 'ext')
576 elif deployableDist.usesJVMCIClassLoader:
577 targetDir = join(jreLibDir, 'jvmci')
578 else:
579 targetDir = jreLibDir
580 if not exists(targetDir):
581 os.makedirs(targetDir)
582 _copyToJdk(dist.path, targetDir)
583 if dist.sourcesPath:
584 _copyToJdk(dist.sourcesPath, jdkDir)
585 if deployableDist.usesJVMCIClassLoader:
586 # deploy service files
587 _updateJVMCIFiles(jdkDir)
588 if deployableDist.postJdkInstall:
589 deployableDist.postJdkInstall(jdkDir, targetDir)
590
591 def _getJdkDeployedJars(jdkDir):
592 """
593 Gets jar paths for all deployed distributions in the context of
594 a given JDK directory.
595 """
596 jreLibDir = join(jdkDir, 'jre', 'lib')
597 jars = []
598 for dist in jdkDeployedDists:
599 jar = basename(mx.distribution(dist.name).path)
600 if dist.isExtension:
601 jars.append(join(jreLibDir, 'ext', jar))
602 elif dist.usesJVMCIClassLoader:
603 jars.append(join(jreLibDir, 'jvmci', jar))
604 else:
605 jars.append(join(jreLibDir, jar))
606 return jars
607
608
609 # run a command in the windows SDK Debug Shell
610 def _runInDebugShell(cmd, workingDir, logFile=None, findInOutput=None, respondTo=None):
611 if respondTo is None:
612 respondTo = {}
613 newLine = os.linesep
614 startToken = 'RUNINDEBUGSHELL_STARTSEQUENCE'
615 endToken = 'RUNINDEBUGSHELL_ENDSEQUENCE'
616
617 winSDK = mx.get_env('WIN_SDK', 'C:\\Program Files\\Microsoft SDKs\\Windows\\v7.1\\')
618
619 if not exists(mx._cygpathW2U(winSDK)):
620 mx.abort("Could not find Windows SDK : '" + winSDK + "' does not exist")
621
622 winSDKSetEnv = mx._cygpathW2U(join(winSDK, 'Bin', 'SetEnv.cmd'))
623 if not exists(winSDKSetEnv):
624 mx.abort("Invalid Windows SDK path (" + winSDK + ") : could not find Bin/SetEnv.cmd (you can use the WIN_SDK environment variable to specify an other path)")
625
626 wincmd = 'cmd.exe /E:ON /V:ON /K "' + mx._cygpathU2W(winSDKSetEnv) + '"'
627 p = subprocess.Popen(wincmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
628 stdout = p.stdout
629 stdin = p.stdin
630 if logFile:
631 log = open(logFile, 'w')
632 ret = False
633
634 def _writeProcess(s):
635 stdin.write(s + newLine)
636
637 _writeProcess("echo " + startToken)
638 while True:
639 # encoding may be None on windows plattforms
640 if sys.stdout.encoding is None:
641 encoding = 'utf-8'
642 else:
643 encoding = sys.stdout.encoding
644
645 line = stdout.readline().decode(encoding)
646 if logFile:
647 log.write(line.encode('utf-8'))
648 line = line.strip()
649 mx.log(line)
650 if line == startToken:
651 _writeProcess('cd /D ' + workingDir + ' & ' + cmd + ' & echo ' + endToken)
652 for regex in respondTo.keys():
653 match = regex.search(line)
654 if match:
655 _writeProcess(respondTo[regex])
656 if findInOutput:
657 match = findInOutput.search(line)
658 if match:
659 ret = True
660 if line == endToken:
661 if not findInOutput:
662 _writeProcess('echo ERRXXX%errorlevel%')
663 else:
664 break
665 if line.startswith('ERRXXX'):
666 if line == 'ERRXXX0':
667 ret = True
668 break
669 _writeProcess("exit")
670 if logFile:
671 log.close()
672 return ret
673
674 def jdkhome(vm=None):
675 """return the JDK directory selected for the 'vm' command"""
676 return _jdk(installJars=False)
677
678 def print_jdkhome(args, vm=None):
679 """print the JDK directory selected for the 'vm' command"""
680 print jdkhome(vm)
681
682 def buildvars(args):
683 """describe the variables that can be set by the -D option to the 'mx build' commmand"""
684
685 buildVars = {
686 'ALT_BOOTDIR' : 'The location of the bootstrap JDK installation (default: ' + mx.java().jdk + ')',
687 'ALT_OUTPUTDIR' : 'Build directory',
688 'HOTSPOT_BUILD_JOBS' : 'Number of CPUs used by make (default: ' + str(mx.cpu_count()) + ')',
689 'INSTALL' : 'Install the built VM into the JDK? (default: y)',
690 'ZIP_DEBUGINFO_FILES' : 'Install zipped debug symbols file? (default: 0)',
691 }
692
693 mx.log('HotSpot build variables that can be set by the -D option to "mx build":')
694 mx.log('')
695 for n in sorted(buildVars.iterkeys()):
696 mx.log(n)
697 mx.log(textwrap.fill(buildVars[n], initial_indent=' ', subsequent_indent=' ', width=200))
698
699 mx.log('')
700 mx.log('Note that these variables can be given persistent values in the file ' + join(_suite.mxDir, 'env') + ' (see \'mx about\').')
701
702 cached_graal_version = None
703
704 def build(args, vm=None):
705 """build the VM binary
706
707 The global '--vm' and '--vmbuild' options select which VM type and build target to build."""
708
709 # Override to fail quickly if extra arguments are given
710 # at the end of the command line. This allows for a more
711 # helpful error message.
712 class AP(ArgumentParser):
713 def __init__(self):
714 ArgumentParser.__init__(self, prog='mx build')
715 def parse_args(self, args):
716 result = ArgumentParser.parse_args(self, args)
717 if len(result.remainder) != 0:
718 firstBuildTarget = result.remainder[0]
719 mx.abort('To specify the ' + firstBuildTarget + ' VM build target, you need to use the global "--vmbuild" option. For example:\n' +
720 ' mx --vmbuild ' + firstBuildTarget + ' build')
721 return result
722
723 # Call mx.build to compile the Java sources
724 parser = AP()
725 parser.add_argument('-D', action='append', help='set a HotSpot build variable (run \'mx buildvars\' to list variables)', metavar='name=value')
726
727 opts2 = mx.build(['--source', '1.7'] + args, parser=parser)
728 assert len(opts2.remainder) == 0
729
730 if not opts2.native:
731 return
732
733 builds = [_vmbuild]
734
735 if os.environ.get('BUILDING_FROM_IDE', None) == 'true':
736 build = os.environ.get('IDE_BUILD_TARGET', None)
737 if build is None or len(build) == 0:
738 return
739 if build not in _vmbuildChoices:
740 mx.abort('VM build "' + build + '" specified by IDE_BUILD_TARGET environment variable is unknown (must be one of ' +
741 str(_vmbuildChoices) + ')')
742 builds = [build]
743
744 if vm is None:
745 vm = get_vm()
746
747 if vm == 'original':
748 pass
749 elif vm.startswith('server'):
750 buildSuffix = ''
751 elif vm.startswith('client'):
752 buildSuffix = '1'
753 else:
754 assert vm == 'jvmci', vm
755 buildSuffix = 'jvmci'
756
757 if _installed_jdks and _installed_jdks != _suite.dir:
758 if not mx.ask_yes_no("Warning: building while --installed-jdks is set (" + _installed_jdks + ") is not recommanded - are you sure you want to continue", 'n'):
759 mx.abort(1)
760
761 isWindows = platform.system() == 'Windows' or "CYGWIN" in platform.system()
762 for build in builds:
763 installJars = vm != 'original' and (isWindows or not opts2.java)
764 jdk = _jdk(build, create=True, installJars=installJars)
765
766 if vm == 'original':
767 if build != 'product':
768 mx.log('only product build of original VM exists')
769 continue
770
771 if not isVMSupported(vm):
772 mx.log('The ' + vm + ' VM is not supported on this platform - skipping')
773 continue
774
775 vmDir = join(_vmLibDirInJdk(jdk), vm)
776 if not exists(vmDir):
777 chmodRecursive(jdk, JDK_UNIX_PERMISSIONS_DIR)
778 mx.log('Creating VM directory in JDK: ' + vmDir)
779 os.makedirs(vmDir)
780
781 def filterXusage(line):
782 if not 'Xusage.txt' in line:
783 sys.stderr.write(line + os.linesep)
784
785 # Check if a build really needs to be done
786 timestampFile = join(vmDir, '.build-timestamp')
787 if opts2.force or not exists(timestampFile):
788 mustBuild = True
789 else:
790 mustBuild = False
791 timestamp = os.path.getmtime(timestampFile)
792 sources = []
793 for d in ['src', 'make', join('jvmci', 'jdk.internal.jvmci.hotspot', 'src_gen', 'hotspot')]:
794 for root, dirnames, files in os.walk(join(_suite.dir, d)):
795 # ignore <graal>/src/share/tools
796 if root == join(_suite.dir, 'src', 'share'):
797 dirnames.remove('tools')
798 sources += [join(root, name) for name in files]
799 for f in sources:
800 if len(f) != 0 and os.path.getmtime(f) > timestamp:
801 mustBuild = True
802 break
803
804 if not mustBuild:
805 mx.logv('[all files in src and make directories are older than ' + timestampFile[len(_suite.dir) + 1:] + ' - skipping native build]')
806 continue
807
808 if isWindows:
809 t_compilelogfile = mx._cygpathU2W(os.path.join(_suite.dir, "graalCompile.log"))
810 mksHome = mx.get_env('MKS_HOME', 'C:\\cygwin\\bin')
811
812 variant = {'client': 'compiler1', 'server': 'compiler2'}.get(vm, vm)
813 project_config = variant + '_' + build
814 jvmciHome = mx._cygpathU2W(_suite.dir)
815 _runInDebugShell('msbuild ' + jvmciHome + r'\build\vs-amd64\jvm.vcproj /p:Configuration=' + project_config + ' /target:clean', jvmciHome)
816 winCompileCmd = r'set HotSpotMksHome=' + mksHome + r'& set OUT_DIR=' + mx._cygpathU2W(jdk) + r'& set JAVA_HOME=' + mx._cygpathU2W(jdk) + r'& set path=%JAVA_HOME%\bin;%path%;%HotSpotMksHome%& cd /D "' + jvmciHome + r'\make\windows"& call create.bat ' + jvmciHome
817 print winCompileCmd
818 winCompileSuccess = re.compile(r"^Writing \.vcxproj file:")
819 if not _runInDebugShell(winCompileCmd, jvmciHome, t_compilelogfile, winCompileSuccess):
820 mx.log('Error executing create command')
821 return
822 winBuildCmd = 'msbuild ' + jvmciHome + r'\build\vs-amd64\jvm.vcxproj /p:Configuration=' + project_config + ' /p:Platform=x64'
823 if not _runInDebugShell(winBuildCmd, jvmciHome, t_compilelogfile):
824 mx.log('Error building project')
825 return
826 else:
827 cpus = mx.cpu_count()
828 makeDir = join(_suite.dir, 'make')
829 runCmd = [mx.gmake_cmd(), '-C', makeDir]
830
831 env = os.environ.copy()
832
833 # These must be passed as environment variables
834 env.setdefault('LANG', 'C')
835 env['JAVA_HOME'] = jdk
836
837 def setMakeVar(name, default, env=None):
838 """Sets a make variable on the command line to the value
839 of the variable in 'env' with the same name if defined
840 and 'env' is not None otherwise to 'default'
841 """
842 runCmd.append(name + '=' + (env.get(name, default) if env else default))
843
844 if opts2.D:
845 for nv in opts2.D:
846 name, value = nv.split('=', 1)
847 setMakeVar(name.strip(), value)
848
849 setMakeVar('ARCH_DATA_MODEL', '64', env=env)
850 setMakeVar('HOTSPOT_BUILD_JOBS', str(cpus), env=env)
851 setMakeVar('ALT_BOOTDIR', mx.java().jdk, env=env)
852 setMakeVar("EXPORT_PATH", jdk)
853
854 setMakeVar('MAKE_VERBOSE', 'y' if mx._opts.verbose else '')
855 if vm.endswith('nojvmci'):
856 setMakeVar('INCLUDE_JVMCI', 'false')
857 setMakeVar('ALT_OUTPUTDIR', join(_suite.dir, 'build-nojvmci', mx.get_os()), env=env)
858 else:
859 version = _suite.release_version()
860 setMakeVar('USER_RELEASE_SUFFIX', 'jvmci-' + version)
861 setMakeVar('INCLUDE_JVMCI', 'true')
862 setMakeVar('INSTALL', 'y', env=env)
863 if mx.get_os() == 'darwin' and platform.mac_ver()[0] != '':
864 # Force use of clang on MacOS
865 setMakeVar('USE_CLANG', 'true')
866 if mx.get_os() == 'solaris':
867 # If using sparcWorks, setup flags to avoid make complaining about CC version
868 cCompilerVersion = subprocess.Popen('CC -V', stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True).stderr.readlines()[0]
869 if cCompilerVersion.startswith('CC: Sun C++'):
870 compilerRev = cCompilerVersion.split(' ')[3]
871 setMakeVar('ENFORCE_COMPILER_REV', compilerRev, env=env)
872 setMakeVar('ENFORCE_CC_COMPILER_REV', compilerRev, env=env)
873 if build == 'jvmg':
874 # We want ALL the symbols when debugging on Solaris
875 setMakeVar('STRIP_POLICY', 'no_strip')
876 # This removes the need to unzip the *.diz files before debugging in gdb
877 setMakeVar('ZIP_DEBUGINFO_FILES', '0', env=env)
878
879 if buildSuffix == "1":
880 setMakeVar("BUILD_CLIENT_ONLY", "true")
881
882 # Clear this variable as having it set can cause very confusing build problems
883 env.pop('CLASSPATH', None)
884
885 # Issue an env prefix that can be used to run the make on the command line
886 if not mx._opts.verbose:
887 mx.log('--------------- make command line ----------------------')
888
889 envPrefix = ' '.join([key + '=' + env[key] for key in env.iterkeys() if not os.environ.has_key(key) or env[key] != os.environ[key]])
890 if len(envPrefix):
891 mx.log('env ' + envPrefix + ' \\')
892
893 runCmd.append(build + buildSuffix)
894 runCmd.append("docs")
895 runCmd.append("export_" + build)
896
897 if not mx._opts.verbose:
898 mx.log(' '.join(runCmd))
899 mx.log('--------------------------------------------------------')
900 mx.run(runCmd, err=filterXusage, env=env)
901
902 jvmCfg = _vmCfgInJdk(jdk)
903 if not exists(jvmCfg):
904 mx.abort(jvmCfg + ' does not exist')
905
906 prefix = '-' + vm + ' '
907 vmKnown = prefix + 'KNOWN\n'
908 lines = []
909 found = False
910 with open(jvmCfg) as f:
911 for line in f:
912 if line.strip() == vmKnown.strip():
913 found = True
914 lines.append(line)
915
916 if not found:
917 mx.log('Prepending "' + prefix + 'KNOWN" to ' + jvmCfg)
918 if mx.get_os() != 'windows':
919 os.chmod(jvmCfg, JDK_UNIX_PERMISSIONS_FILE)
920 with open(jvmCfg, 'w') as f:
921 written = False
922 for line in lines:
923 if line.startswith('#'):
924 f.write(line)
925 continue
926 if not written:
927 f.write(vmKnown)
928 if vm == 'jvmci':
929 # Legacy support
930 f.write('-graal ALIASED_TO -jvmci\n')
931 written = True
932 if line.startswith(prefix):
933 line = vmKnown
934 if written:
935 continue
936 f.write(line)
937
938 for jdkDist in jdkDeployedDists: # Install non HotSpot distribution
939 if not jdkDist.partOfHotSpot:
940 _installDistInJdks(jdkDist)
941 if exists(timestampFile):
942 os.utime(timestampFile, None)
943 else:
944 file(timestampFile, 'a')
945
946 """
947 The base list of JaCoCo includes.
948 """
949 jacocoIncludes = ['@Test']
950
951 """
952 The list of annotations which if present denote a class that should
953 be excluded from JaCoCo analysis.
954 """
955 jacocoExcludedAnnotations = ['@Test']
956
957 def parseVmArgs(args, vm=None, cwd=None, vmbuild=None):
958 """run the VM selected by the '--vm' option"""
959
960 if vm is None:
961 vm = get_vm()
962
963 if not isVMSupported(vm):
964 mx.abort('The ' + vm + ' is not supported on this platform')
965
966 if cwd is None:
967 cwd = _vm_cwd
968 elif _vm_cwd is not None and _vm_cwd != cwd:
969 mx.abort("conflicting working directories: do not set --vmcwd for this command")
970
971 build = vmbuild if vmbuild else _vmbuild
972 jdk = _jdk(build, vmToCheck=vm, installJars=False)
973 _updateInstalledJVMCIOptionsFile(jdk)
974 mx.expand_project_in_args(args)
975 if _make_eclipse_launch:
976 mx.make_eclipse_launch(_suite, args, _suite.name + '-' + build, name=None, deps=mx.sorted_deps(includeLibs=True))
977 if _jacoco == 'on' or _jacoco == 'append':
978 jacocoagent = mx.library("JACOCOAGENT", True)
979 # Exclude all compiler tests and snippets
980
981 includes = list(jacocoIncludes)
982 baseExcludes = []
983 for p in mx.projects():
984 projsetting = getattr(p, 'jacoco', '')
985 if projsetting == 'exclude':
986 baseExcludes.append(p.name)
987 if projsetting == 'include':
988 includes.append(p.name + '.*')
989
990 def _filter(l):
991 # filter out specific classes which are already covered by a baseExclude package
992 return [clazz for clazz in l if not any([clazz.startswith(package) for package in baseExcludes])]
993 excludes = []
994 for p in mx.projects():
995 excludes += _filter(p.find_classes_with_annotations(None, jacocoExcludedAnnotations, includeInnerClasses=True).keys())
996 excludes += _filter(p.find_classes_with_matching_source_line(None, lambda line: 'JaCoCo Exclude' in line, includeInnerClasses=True).keys())
997
998 excludes += [package + '.*' for package in baseExcludes]
999 agentOptions = {
1000 'append' : 'true' if _jacoco == 'append' else 'false',
1001 'bootclasspath' : 'true',
1002 'includes' : ':'.join(includes),
1003 'excludes' : ':'.join(excludes),
1004 'destfile' : 'jacoco.exec'
1005 }
1006 args = ['-javaagent:' + jacocoagent.get_path(True) + '=' + ','.join([k + '=' + v for k, v in agentOptions.items()])] + args
1007 exe = join(jdk, 'bin', mx.exe_suffix('java'))
1008 pfx = _vm_prefix.split() if _vm_prefix is not None else []
1009
1010 if '-version' in args:
1011 ignoredArgs = args[args.index('-version') + 1:]
1012 if len(ignoredArgs) > 0:
1013 mx.log("Warning: The following options will be ignored by the vm because they come after the '-version' argument: " + ' '.join(ignoredArgs))
1014
1015 args = mx.java().processArgs(args)
1016 return (pfx, exe, vm, args, cwd)
1017
1018 def vm(args, vm=None, nonZeroIsFatal=True, out=None, err=None, cwd=None, timeout=None, vmbuild=None):
1019 (pfx_, exe_, vm_, args_, cwd) = parseVmArgs(args, vm, cwd, vmbuild)
1020 return mx.run(pfx_ + [exe_, '-' + vm_] + args_, nonZeroIsFatal=nonZeroIsFatal, out=out, err=err, cwd=cwd, timeout=timeout)
1021
1022 def unittest(args):
1023 def vmLauncher(vmArgs, mainClass, mainClassArgs):
1024 if isJVMCIEnabled(get_vm()):
1025 # Remove entries from class path that are in JVMCI loaded jars
1026 cpIndex, cp = mx.find_classpath_arg(vmArgs)
1027 if cp:
1028 excluded = set()
1029 for jdkDist in jdkDeployedDists:
1030 dist = mx.distribution(jdkDist.name)
1031 excluded.update([d.output_dir() for d in dist.sorted_deps()])
1032 cp = os.pathsep.join([e for e in cp.split(os.pathsep) if e not in excluded])
1033 vmArgs[cpIndex] = cp
1034
1035 # Run the VM in a mode where application/test classes can
1036 # access JVMCI loaded classes.
1037 vmArgs = ['-XX:-UseJVMCIClassLoader'] + vmArgs
1038
1039 vm(vmArgs + [mainClass] + mainClassArgs)
1040 mx_unittest.unittest(args, vmLauncher=vmLauncher)
1041
1042 def shortunittest(args):
1043 """alias for 'unittest --whitelist test/whitelist_shortunittest.txt'"""
1044
1045 args = ['--whitelist', 'test/whitelist_shortunittest.txt'] + args
1046 unittest(args)
1047
1048 def buildvms(args):
1049 """build one or more VMs in various configurations"""
1050
1051 vmsDefault = ','.join(_vmChoices.keys())
1052 vmbuildsDefault = ','.join(_vmbuildChoices)
1053
1054 parser = ArgumentParser(prog='mx buildvms')
1055 parser.add_argument('--vms', help='a comma separated list of VMs to build (default: ' + vmsDefault + ')', metavar='<args>', default=vmsDefault)
1056 parser.add_argument('--builds', help='a comma separated list of build types (default: ' + vmbuildsDefault + ')', metavar='<args>', default=vmbuildsDefault)
1057 parser.add_argument('--check-distributions', action='store_true', dest='check_distributions', help='check built distributions for overlap')
1058 parser.add_argument('-n', '--no-check', action='store_true', help='omit running "java -version" after each build')
1059 parser.add_argument('-c', '--console', action='store_true', help='send build output to console instead of log file')
1060
1061 args = parser.parse_args(args)
1062 vms = args.vms.split(',')
1063 builds = args.builds.split(',')
1064
1065 allStart = time.time()
1066 check_dists_args = ['--check-distributions'] if args.check_distributions else []
1067 for v in vms:
1068 if not isVMSupported(v):
1069 mx.log('The ' + v + ' VM is not supported on this platform - skipping')
1070 continue
1071
1072 for vmbuild in builds:
1073 if v == 'original' and vmbuild != 'product':
1074 continue
1075 if not args.console:
1076 logFile = join(v + '-' + vmbuild + '.log')
1077 log = open(join(_suite.dir, logFile), 'wb')
1078 start = time.time()
1079 mx.log('BEGIN: ' + v + '-' + vmbuild + '\t(see: ' + logFile + ')')
1080 verbose = ['-v'] if mx._opts.verbose else []
1081 # Run as subprocess so that output can be directed to a file
1082 cmd = [sys.executable, '-u', join('mxtool', 'mx.py')] + verbose + ['--vm', v, '--vmbuild', vmbuild, 'build'] + check_dists_args
1083 mx.logv("executing command: " + str(cmd))
1084 subprocess.check_call(cmd, cwd=_suite.dir, stdout=log, stderr=subprocess.STDOUT)
1085 duration = datetime.timedelta(seconds=time.time() - start)
1086 mx.log('END: ' + v + '-' + vmbuild + '\t[' + str(duration) + ']')
1087 else:
1088 with VM(v, vmbuild):
1089 build(check_dists_args)
1090 if not args.no_check:
1091 vmargs = ['-version']
1092 if v == 'jvmci':
1093 vmargs.insert(0, '-XX:-BootstrapJVMCI')
1094 vm(vmargs, vm=v, vmbuild=vmbuild)
1095 allDuration = datetime.timedelta(seconds=time.time() - allStart)
1096 mx.log('TOTAL TIME: ' + '[' + str(allDuration) + ']')
1097
1098
1099 """
1100 Context manager for a single gate task that can prevent the
1101 task from executing or time and log its execution.
1102 """
1103 class Task:
1104 # None or a list of strings. If not None, only tasks whose title
1105 # matches at least one of the substrings in this list will return
1106 # a non-None value from __enter__. The body of a 'with Task(...) as t'
1107 # statement should check 't' and exit immediately if it is None.
1108 filters = None
1109 filtersExclude = False
1110
1111 def __init__(self, title, tasks=None, disableJacoco=False):
1112 self.tasks = tasks
1113 self.title = title
1114 if tasks is not None and Task.filters is not None:
1115 if Task.filtersExclude:
1116 self.skipped = any([f in title for f in Task.filters])
1117 else:
1118 self.skipped = not any([f in title for f in Task.filters])
1119 else:
1120 self.skipped = False
1121 if not self.skipped:
1122 self.start = time.time()
1123 self.end = None
1124 self.duration = None
1125 self.disableJacoco = disableJacoco
1126 mx.log(time.strftime('gate: %d %b %Y %H:%M:%S: BEGIN: ') + title)
1127 def __enter__(self):
1128 assert self.tasks is not None, "using Task with 'with' statement requires to pass the tasks list in the constructor"
1129 if self.skipped:
1130 return None
1131 if self.disableJacoco:
1132 self.jacacoSave = _jacoco
1133 return self
1134 def __exit__(self, exc_type, exc_value, traceback):
1135 if not self.skipped:
1136 self.tasks.append(self.stop())
1137 if self.disableJacoco:
1138 global _jacoco
1139 _jacoco = self.jacacoSave
1140
1141 def stop(self):
1142 self.end = time.time()
1143 self.duration = datetime.timedelta(seconds=self.end - self.start)
1144 mx.log(time.strftime('gate: %d %b %Y %H:%M:%S: END: ') + self.title + ' [' + str(self.duration) + ']')
1145 return self
1146 def abort(self, codeOrMessage):
1147 self.end = time.time()
1148 self.duration = datetime.timedelta(seconds=self.end - self.start)
1149 mx.log(time.strftime('gate: %d %b %Y %H:%M:%S: ABORT: ') + self.title + ' [' + str(self.duration) + ']')
1150 mx.abort(codeOrMessage)
1151 return self
1152
1153 def ctw(args):
1154 """run CompileTheWorld"""
1155
1156 defaultCtwopts = '-Inline'
1157
1158 parser = ArgumentParser(prog='mx ctw')
1159 parser.add_argument('--ctwopts', action='store', help='space separated JVMCI options used for CTW compilations (default: --ctwopts="' + defaultCtwopts + '")', default=defaultCtwopts, metavar='<options>')
1160 parser.add_argument('--jar', action='store', help='jar of classes to compiled instead of rt.jar', metavar='<path>')
1161
1162 args, vmargs = parser.parse_known_args(args)
1163
1164 if args.ctwopts:
1165 vmargs.append('-G:CompileTheWorldConfig=' + args.ctwopts)
1166
1167 if args.jar:
1168 jar = os.path.abspath(args.jar)
1169 else:
1170 jar = join(_jdk(installJars=False), 'jre', 'lib', 'rt.jar')
1171 vmargs.append('-G:CompileTheWorldExcludeMethodFilter=sun.awt.X11.*.*')
1172
1173 vmargs += ['-XX:+CompileTheWorld']
1174 vm_ = get_vm()
1175 if isJVMCIEnabled(vm_):
1176 if vm_ == 'jvmci':
1177 vmargs += ['-XX:+BootstrapJVMCI']
1178 vmargs += ['-G:CompileTheWorldClasspath=' + jar]
1179 else:
1180 vmargs += ['-Xbootclasspath/p:' + jar]
1181
1182 # suppress menubar and dock when running on Mac; exclude x11 classes as they may cause vm crashes (on Solaris)
1183 vmargs = ['-Djava.awt.headless=true'] + vmargs
1184
1185 vm(vmargs)
1186
1187 def _jvmci_gate_runner(args, tasks):
1188 # Build server-hosted-jvmci now so we can run the unit tests
1189 with Task('BuildHotSpotJVMCIHosted: product', tasks) as t:
1190 if t: buildvms(['--vms', 'server', '--builds', 'product', '--check-distributions'])
1191
1192 # Run unit tests on server-hosted-jvmci
1193 with VM('server', 'product'):
1194 with Task('UnitTests:hosted-product', tasks) as t:
1195 if t: unittest(['--enable-timing', '--verbose', '--fail-fast'])
1196
1197 # Run unit tests on server-hosted-jvmci with -G:+SSA_LIR
1198 with VM('server', 'product'):
1199 with Task('UnitTestsSSA:hosted-product', tasks) as t:
1200 if t: unittest(['--enable-timing', '--verbose', '--fail-fast', '-G:+SSA_LIR'])
1201 # Run ctw against rt.jar on server-hosted-jvmci
1202 with VM('server', 'product'):
1203 with Task('CTW:hosted-product', tasks) as t:
1204 if t: ctw(['--ctwopts', '-Inline +ExitVMOnException', '-esa', '-G:+CompileTheWorldMultiThreaded', '-G:-InlineDuringParsing', '-G:-CompileTheWorldVerbose'])
1205
1206 # Build the other VM flavors
1207 with Task('BuildHotSpotGraalOthers: fastdebug,product', tasks) as t:
1208 if t: buildvms(['--vms', 'jvmci,server', '--builds', 'fastdebug,product', '--check-distributions'])
1209
1210 with VM('jvmci', 'fastdebug'):
1211 with Task('BootstrapWithSystemAssertions:fastdebug', tasks) as t:
1212 if t: vm(['-esa', '-XX:-TieredCompilation', '-version'])
1213
1214 with VM('jvmci', 'fastdebug'):
1215 with Task('BootstrapEconomyWithSystemAssertions:fastdebug', tasks) as t:
1216 if t: vm(['-esa', '-XX:-TieredCompilation', '-G:CompilerConfiguration=economy', '-version'])
1217
1218 with VM('jvmci', 'fastdebug'):
1219 with Task('BootstrapWithSystemAssertionsNoCoop:fastdebug', tasks) as t:
1220 if t: vm(['-esa', '-XX:-TieredCompilation', '-XX:-UseCompressedOops', '-version'])
1221
1222 with VM('jvmci', 'fastdebug'):
1223 with Task('BootstrapWithExceptionEdges:fastdebug', tasks) as t:
1224 if t: vm(['-esa', '-XX:-TieredCompilation', '-G:+StressInvokeWithExceptionNode', '-version'])
1225
1226 with VM('jvmci', 'product'):
1227 with Task('BootstrapWithGCVerification:product', tasks) as t:
1228 if t:
1229 out = mx.DuplicateSuppressingStream(['VerifyAfterGC:', 'VerifyBeforeGC:']).write
1230 vm(['-XX:-TieredCompilation', '-XX:+UnlockDiagnosticVMOptions', '-XX:+VerifyBeforeGC', '-XX:+VerifyAfterGC', '-version'], out=out)
1231
1232 with VM('jvmci', 'product'):
1233 with Task('BootstrapWithG1GCVerification:product', tasks) as t:
1234 if t:
1235 out = mx.DuplicateSuppressingStream(['VerifyAfterGC:', 'VerifyBeforeGC:']).write
1236 vm(['-XX:-TieredCompilation', '-XX:+UnlockDiagnosticVMOptions', '-XX:-UseSerialGC', '-XX:+UseG1GC', '-XX:+VerifyBeforeGC', '-XX:+VerifyAfterGC', '-version'], out=out)
1237
1238 with VM('jvmci', 'product'):
1239 with Task('BootstrapWithRegisterPressure:product', tasks) as t:
1240 if t:
1241 registers = 'o0,o1,o2,o3,f8,f9,d32,d34' if platform.processor() == 'sparc' else 'rbx,r11,r10,r14,xmm3,xmm11,xmm14'
1242 vm(['-XX:-TieredCompilation', '-G:RegisterPressure=' + registers, '-esa', '-version'])
1243
1244 with VM('jvmci', 'product'):
1245 with Task('BootstrapSSAWithRegisterPressure:product', tasks) as t:
1246 if t:
1247 registers = 'o0,o1,o2,o3,f8,f9,d32,d34' if platform.processor() == 'sparc' else 'rbx,r11,r10,r14,xmm3,xmm11,xmm14'
1248 vm(['-XX:-TieredCompilation', '-G:+SSA_LIR', '-G:RegisterPressure=' + registers, '-esa', '-version'])
1249
1250 with VM('jvmci', 'product'):
1251 with Task('BootstrapWithImmutableCode:product', tasks) as t:
1252 if t: vm(['-XX:-TieredCompilation', '-G:+ImmutableCode', '-G:+VerifyPhases', '-esa', '-version'])
1253
1254 with Task('CleanAndBuildIdealGraphVisualizer', tasks, disableJacoco=True) as t:
1255 if t and platform.processor() != 'sparc':
1256 buildxml = mx._cygpathU2W(join(_suite.dir, 'src', 'share', 'tools', 'IdealGraphVisualizer', 'build.xml'))
1257 mx.run(['ant', '-f', buildxml, '-q', 'clean', 'build'], env=_igvBuildEnv())
1258
1259 # Prevent JVMCI modifications from breaking the standard builds
1260 if args.buildNonJVMCI:
1261 with Task('BuildHotSpotVarieties', tasks, disableJacoco=True) as t:
1262 if t:
1263 buildvms(['--vms', 'client,server', '--builds', 'fastdebug,product'])
1264 if mx.get_os() not in ['windows', 'cygwin']:
1265 buildvms(['--vms', 'server-nojvmci', '--builds', 'product,optimized'])
1266
1267 """
1268 List of function called by the gate once common gate tasks have been executed.
1269 Each function is called with these arguments:
1270 args: the argparse.Namespace object containing result of parsing gate command line
1271 tasks: list of Task to which extra Tasks should be added
1272 """
1273 gateRunners = [_jvmci_gate_runner]
1274
1275 def gate(args):
1276 """run the tests used to validate a push
1277
1278 If this command exits with a 0 exit code, then the source code is in
1279 a state that would be accepted for integration into the main repository."""
1280
1281 parser = ArgumentParser(prog='mx gate')
1282 parser.add_argument('-j', '--omit-java-clean', action='store_false', dest='cleanJava', help='omit cleaning Java native code')
1283 parser.add_argument('-n', '--omit-native-clean', action='store_false', dest='cleanNative', help='omit cleaning and building native code')
1284 parser.add_argument('-i', '--omit-ide-clean', action='store_false', dest='cleanIde', help='omit cleaning the ide project files')
1285 parser.add_argument('-g', '--only-build-jvmci', action='store_false', dest='buildNonJVMCI', help='only build the JVMCI VM')
1286 parser.add_argument('-t', '--task-filter', help='comma separated list of substrings to select subset of tasks to be run')
1287 parser.add_argument('-x', action='store_true', help='makes --task-filter an exclusion instead of inclusion filter')
1288 parser.add_argument('--jacocout', help='specify the output directory for jacoco report')
1289
1290 args = parser.parse_args(args)
1291
1292 global _jacoco
1293 if args.task_filter:
1294 Task.filters = args.task_filter.split(',')
1295 Task.filtersExclude = args.x
1296 elif args.x:
1297 mx.abort('-x option cannot be used without --task-filter option')
1298
1299 # Force
1300 if not mx._opts.strict_compliance:
1301 mx.log("[gate] forcing strict compliance")
1302 mx._opts.strict_compliance = True
1303
1304 tasks = []
1305 total = Task('Gate')
1306 try:
1307 with Task('Check jvmci.make in sync with suite.py', tasks) as t:
1308 if t:
1309 jvmciMake = join('make', 'jvmci.make')
1310 if mx_jvmci_makefile.build_makefile(['-o', jvmciMake]) != 0:
1311 t.abort('Rerun "mx makefile -o ' + jvmciMake + ' and check-in the modified ' + jvmciMake)
1312
1313 with Task('Pylint', tasks) as t:
1314 if t: mx.pylint([])
1315
1316 def _clean(name='Clean'):
1317 with Task(name, tasks) as t:
1318 if t:
1319 cleanArgs = []
1320 if not args.cleanNative:
1321 cleanArgs.append('--no-native')
1322 if not args.cleanJava:
1323 cleanArgs.append('--no-java')
1324 clean(cleanArgs)
1325 _clean()
1326
1327 with Task('IDEConfigCheck', tasks) as t:
1328 if t:
1329 if args.cleanIde:
1330 mx.ideclean([])
1331 mx.ideinit([])
1332
1333 eclipse_exe = mx.get_env('ECLIPSE_EXE')
1334 if eclipse_exe is not None:
1335 with Task('CodeFormatCheck', tasks) as t:
1336 if t and mx.eclipseformat(['-e', eclipse_exe]) != 0:
1337 t.abort('Formatter modified files - run "mx eclipseformat", check in changes and repush')
1338
1339 with Task('Canonicalization Check', tasks) as t:
1340 if t:
1341 mx.log(time.strftime('%d %b %Y %H:%M:%S - Ensuring mx/projects files are canonicalized...'))
1342 if mx.canonicalizeprojects([]) != 0:
1343 t.abort('Rerun "mx canonicalizeprojects" and check-in the modified mx/suite*.py files.')
1344
1345 if mx.get_env('JDT'):
1346 with Task('BuildJavaWithEcj', tasks):
1347 if t: build(['-p', '--no-native', '--jdt-warning-as-error'])
1348 _clean('CleanAfterEcjBuild')
1349
1350 with Task('BuildJavaWithJavac', tasks):
1351 if t: build(['-p', '--no-native', '--force-javac'])
1352
1353 with Task('Checkstyle', tasks) as t:
1354 if t and mx.checkstyle([]) != 0:
1355 t.abort('Checkstyle warnings were found')
1356
1357 with Task('Checkheaders', tasks) as t:
1358 if t and checkheaders([]) != 0:
1359 t.abort('Checkheaders warnings were found')
1360
1361 with Task('FindBugs', tasks) as t:
1362 if t and mx_findbugs.findbugs([]) != 0:
1363 t.abort('FindBugs warnings were found')
1364
1365 if exists('jacoco.exec'):
1366 os.unlink('jacoco.exec')
1367
1368 if args.jacocout is not None:
1369 _jacoco = 'append'
1370 else:
1371 _jacoco = 'off'
1372
1373 for runner in gateRunners:
1374 runner(args, tasks)
1375
1376 if args.jacocout is not None:
1377 jacocoreport([args.jacocout])
1378 _jacoco = 'off'
1379
1380 except KeyboardInterrupt:
1381 total.abort(1)
1382
1383 except BaseException as e:
1384 import traceback
1385 traceback.print_exc()
1386 total.abort(str(e))
1387
1388 total.stop()
1389
1390 mx.log('Gate task times:')
1391 for t in tasks:
1392 mx.log(' ' + str(t.duration) + '\t' + t.title)
1393 mx.log(' =======')
1394 mx.log(' ' + str(total.duration))
1395
1396 if args.task_filter:
1397 Task.filters = None
1398
1399 def deoptalot(args):
1400 """bootstrap a VM with DeoptimizeALot and VerifyOops on
1401
1402 If the first argument is a number, the process will be repeated
1403 this number of times. All other arguments are passed to the VM."""
1404 count = 1
1405 if len(args) > 0 and args[0].isdigit():
1406 count = int(args[0])
1407 del args[0]
1408
1409 for _ in range(count):
1410 if not vm(['-XX:-TieredCompilation', '-XX:+DeoptimizeALot', '-XX:+VerifyOops'] + args + ['-version']) == 0:
1411 mx.abort("Failed")
1412
1413 def longtests(args):
1414
1415 deoptalot(['15', '-Xmx48m'])
1416
1417 def _igvJdk():
1418 v8u20 = mx.VersionSpec("1.8.0_20")
1419 v8u40 = mx.VersionSpec("1.8.0_40")
1420 v8 = mx.VersionSpec("1.8")
1421 def _igvJdkVersionCheck(version):
1422 return version >= v8 and (version < v8u20 or version >= v8u40)
1423 return mx.java(_igvJdkVersionCheck, versionDescription='>= 1.8 and < 1.8.0u20 or >= 1.8.0u40', purpose="building & running IGV").jdk
1424
1425 def _igvBuildEnv():
1426 # When the http_proxy environment variable is set, convert it to the proxy settings that ant needs
1427 env = dict(os.environ)
1428 proxy = os.environ.get('http_proxy')
1429 if not (proxy is None) and len(proxy) > 0:
1430 if '://' in proxy:
1431 # Remove the http:// prefix (or any other protocol prefix)
1432 proxy = proxy.split('://', 1)[1]
1433 # Separate proxy server name and port number
1434 proxyName, proxyPort = proxy.split(':', 1)
1435 proxyEnv = '-DproxyHost="' + proxyName + '" -DproxyPort=' + proxyPort
1436 env['ANT_OPTS'] = proxyEnv
1437
1438 env['JAVA_HOME'] = _igvJdk()
1439 return env
1440
1441 def igv(args):
1442 """run the Ideal Graph Visualizer"""
1443 logFile = '.ideal_graph_visualizer.log'
1444 with open(join(_suite.dir, logFile), 'w') as fp:
1445 mx.logv('[Ideal Graph Visualizer log is in ' + fp.name + ']')
1446 nbplatform = join(_suite.dir, 'src', 'share', 'tools', 'IdealGraphVisualizer', 'nbplatform')
1447
1448 # Remove NetBeans platform if it is earlier than the current supported version
1449 if exists(nbplatform):
1450 updateTrackingFile = join(nbplatform, 'platform', 'update_tracking', 'org-netbeans-core.xml')
1451 if not exists(updateTrackingFile):
1452 mx.log('Could not find \'' + updateTrackingFile + '\', removing NetBeans platform')
1453 shutil.rmtree(nbplatform)
1454 else:
1455 dom = xml.dom.minidom.parse(updateTrackingFile)
1456 currentVersion = mx.VersionSpec(dom.getElementsByTagName('module_version')[0].getAttribute('specification_version'))
1457 supportedVersion = mx.VersionSpec('3.43.1')
1458 if currentVersion < supportedVersion:
1459 mx.log('Replacing NetBeans platform version ' + str(currentVersion) + ' with version ' + str(supportedVersion))
1460 shutil.rmtree(nbplatform)
1461 elif supportedVersion < currentVersion:
1462 mx.log('Supported NetBeans version in igv command should be updated to ' + str(currentVersion))
1463
1464 if not exists(nbplatform):
1465 mx.logv('[This execution may take a while as the NetBeans platform needs to be downloaded]')
1466
1467 env = _igvBuildEnv()
1468 # make the jar for Batik 1.7 available.
1469 env['IGV_BATIK_JAR'] = mx.library('BATIK').get_path(True)
1470 if mx.run(['ant', '-f', mx._cygpathU2W(join(_suite.dir, 'src', 'share', 'tools', 'IdealGraphVisualizer', 'build.xml')), '-l', mx._cygpathU2W(fp.name), 'run'], env=env, nonZeroIsFatal=False):
1471 mx.abort("IGV ant build & launch failed. Check '" + logFile + "'. You can also try to delete 'src/share/tools/IdealGraphVisualizer/nbplatform'.")
1472
1473 def c1visualizer(args):
1474 """run the Cl Compiler Visualizer"""
1475 libpath = join(_suite.dir, 'lib')
1476 if mx.get_os() == 'windows':
1477 executable = join(libpath, 'c1visualizer', 'bin', 'c1visualizer.exe')
1478 else:
1479 executable = join(libpath, 'c1visualizer', 'bin', 'c1visualizer')
1480
1481 # Check whether the current C1Visualizer installation is the up-to-date
1482 if exists(executable) and not exists(mx.library('C1VISUALIZER_DIST').get_path(resolve=False)):
1483 mx.log('Updating C1Visualizer')
1484 shutil.rmtree(join(libpath, 'c1visualizer'))
1485
1486 archive = mx.library('C1VISUALIZER_DIST').get_path(resolve=True)
1487
1488 if not exists(executable):
1489 zf = zipfile.ZipFile(archive, 'r')
1490 zf.extractall(libpath)
1491
1492 if not exists(executable):
1493 mx.abort('C1Visualizer binary does not exist: ' + executable)
1494
1495 if mx.get_os() != 'windows':
1496 # Make sure that execution is allowed. The zip file does not always specfiy that correctly
1497 os.chmod(executable, 0777)
1498
1499 mx.run([executable])
1500
1501 def _get_jmh_path():
1502 path = mx.get_env('JMH_BENCHMARKS', None)
1503 if not path:
1504 probe = join(dirname(_suite.dir), 'java-benchmarks')
1505 if exists(probe):
1506 path = probe
1507
1508 if not path:
1509 mx.abort("Please set the JMH_BENCHMARKS environment variable to point to the java-benchmarks workspace")
1510 if not exists(path):
1511 mx.abort("The directory denoted by the JMH_BENCHMARKS environment variable does not exist: " + path)
1512 return path
1513
1514 def makejmhdeps(args):
1515 """creates and installs Maven dependencies required by the JMH benchmarks
1516
1517 The dependencies are specified by files named pom.mxdeps in the
1518 JMH directory tree. Each such file contains a list of dependencies
1519 defined in JSON format. For example:
1520
1521 '[{"artifactId" : "compiler.test", "groupId" : "com.oracle.graal", "deps" : ["com.oracle.graal.compiler.test"]}]'
1522
1523 will result in a dependency being installed in the local Maven repository
1524 that can be referenced in a pom.xml file as follows:
1525
1526 <dependency>
1527 <groupId>com.oracle.graal</groupId>
1528 <artifactId>compiler.test</artifactId>
1529 <version>1.0-SNAPSHOT</version>
1530 </dependency>"""
1531
1532 parser = ArgumentParser(prog='mx makejmhdeps')
1533 parser.add_argument('-s', '--settings', help='alternative path for Maven user settings file', metavar='<path>')
1534 parser.add_argument('-p', '--permissive', action='store_true', help='issue note instead of error if a Maven dependency cannot be built due to missing projects/libraries')
1535 args = parser.parse_args(args)
1536
1537 def makejmhdep(artifactId, groupId, deps):
1538 path = artifactId + '.jar'
1539 if args.permissive:
1540 allDeps = []
1541 for name in deps:
1542 dist = mx.distribution(name, fatalIfMissing=False)
1543 if dist:
1544 allDeps = allDeps + [d.name for d in dist.sorted_deps(transitive=True)]
1545 else:
1546 if not mx.project(name, fatalIfMissing=False):
1547 if not mx.library(name, fatalIfMissing=False):
1548 mx.log('Skipping dependency ' + groupId + '.' + artifactId + ' as ' + name + ' cannot be resolved')
1549 return
1550 allDeps.append(name)
1551 d = mx.Distribution(_suite, name=artifactId, path=path, sourcesPath=path, deps=allDeps, mainClass=None, excludedDependencies=[], distDependencies=[], javaCompliance=None)
1552 d.make_archive()
1553 cmd = ['mvn', 'install:install-file', '-DgroupId=' + groupId, '-DartifactId=' + artifactId,
1554 '-Dversion=1.0-SNAPSHOT', '-Dpackaging=jar', '-Dfile=' + d.path]
1555 if not mx._opts.verbose:
1556 cmd.append('-q')
1557 if args.settings:
1558 cmd = cmd + ['-s', args.settings]
1559 mx.run(cmd)
1560 os.unlink(d.path)
1561
1562 jmhPath = _get_jmh_path()
1563 for root, _, filenames in os.walk(jmhPath):
1564 for f in [join(root, n) for n in filenames if n == 'pom.mxdeps']:
1565 mx.logv('[processing ' + f + ']')
1566 try:
1567 with open(f) as fp:
1568 for d in json.load(fp):
1569 artifactId = d['artifactId']
1570 groupId = d['groupId']
1571 deps = d['deps']
1572 makejmhdep(artifactId, groupId, deps)
1573 except ValueError as e:
1574 mx.abort('Error parsing {0}:\n{1}'.format(f, e))
1575
1576 def buildjmh(args):
1577 """build the JMH benchmarks"""
1578
1579 parser = ArgumentParser(prog='mx buildjmh')
1580 parser.add_argument('-s', '--settings', help='alternative path for Maven user settings file', metavar='<path>')
1581 parser.add_argument('-c', action='store_true', dest='clean', help='clean before building')
1582 args = parser.parse_args(args)
1583
1584 jmhPath = _get_jmh_path()
1585 mx.log('JMH benchmarks: ' + jmhPath)
1586
1587 # Ensure the mx injected dependencies are up to date
1588 makejmhdeps(['-p'] + (['-s', args.settings] if args.settings else []))
1589
1590 timestamp = mx.TimeStampFile(join(_suite.mxDir, 'jmh', jmhPath.replace(os.sep, '_') + '.timestamp'))
1591 mustBuild = args.clean
1592 if not mustBuild:
1593 try:
1594 hgfiles = [join(jmhPath, f) for f in subprocess.check_output(['hg', '-R', jmhPath, 'locate']).split('\n')]
1595 mustBuild = timestamp.isOlderThan(hgfiles)
1596 except:
1597 # not a Mercurial repository or hg commands are not available.
1598 mustBuild = True
1599
1600 if mustBuild:
1601 buildOutput = []
1602 def _redirect(x):
1603 if mx._opts.verbose:
1604 mx.log(x[:-1])
1605 else:
1606 buildOutput.append(x)
1607 env = os.environ.copy()
1608 env['JAVA_HOME'] = _jdk(vmToCheck='server')
1609 env['MAVEN_OPTS'] = '-server -XX:-UseJVMCIClassLoader'
1610 mx.log("Building benchmarks...")
1611 cmd = ['mvn']
1612 if args.settings:
1613 cmd = cmd + ['-s', args.settings]
1614 if args.clean:
1615 cmd.append('clean')
1616 cmd.append('package')
1617 retcode = mx.run(cmd, cwd=jmhPath, out=_redirect, env=env, nonZeroIsFatal=False)
1618 if retcode != 0:
1619 mx.log(''.join(buildOutput))
1620 mx.abort(retcode)
1621 timestamp.touch()
1622 else:
1623 mx.logv('[all Mercurial controlled files in ' + jmhPath + ' are older than ' + timestamp.path + ' - skipping build]')
1624
1625 def jmh(args):
1626 """run the JMH benchmarks
1627
1628 This command respects the standard --vm and --vmbuild options
1629 for choosing which VM to run the benchmarks with."""
1630 if '-h' in args:
1631 mx.help_(['jmh'])
1632 mx.abort(1)
1633
1634 vmArgs, benchmarksAndJsons = mx.extract_VM_args(args)
1635 if isJVMCIEnabled(get_vm()) and '-XX:-UseJVMCIClassLoader' not in vmArgs:
1636 vmArgs = ['-XX:-UseJVMCIClassLoader'] + vmArgs
1637
1638 benchmarks = [b for b in benchmarksAndJsons if not b.startswith('{')]
1639 jmhArgJsons = [b for b in benchmarksAndJsons if b.startswith('{')]
1640 jmhOutDir = join(_suite.mxDir, 'jmh')
1641 if not exists(jmhOutDir):
1642 os.makedirs(jmhOutDir)
1643 jmhOut = join(jmhOutDir, 'jmh.out')
1644 jmhArgs = {'-rff' : jmhOut, '-v' : 'EXTRA' if mx._opts.verbose else 'NORMAL'}
1645
1646 # e.g. '{"-wi" : 20}'
1647 for j in jmhArgJsons:
1648 try:
1649 for n, v in json.loads(j).iteritems():
1650 if v is None:
1651 del jmhArgs[n]
1652 else:
1653 jmhArgs[n] = v
1654 except ValueError as e:
1655 mx.abort('error parsing JSON input: {0}\n{1}'.format(j, e))
1656
1657 jmhPath = _get_jmh_path()
1658 mx.log('Using benchmarks in ' + jmhPath)
1659
1660 matchedSuites = set()
1661 numBench = [0]
1662 for micros in os.listdir(jmhPath):
1663 absoluteMicro = os.path.join(jmhPath, micros)
1664 if not os.path.isdir(absoluteMicro):
1665 continue
1666 if not micros.startswith("micros-"):
1667 mx.logv('JMH: ignored ' + absoluteMicro + " because it doesn't start with 'micros-'")
1668 continue
1669
1670 microJar = os.path.join(absoluteMicro, "target", "microbenchmarks.jar")
1671 if not exists(microJar):
1672 mx.log('Missing ' + microJar + ' - please run "mx buildjmh"')
1673 continue
1674 if benchmarks:
1675 def _addBenchmark(x):
1676 if x.startswith("Benchmark:"):
1677 return
1678 match = False
1679 for b in benchmarks:
1680 match = match or (b in x)
1681
1682 if match:
1683 numBench[0] += 1
1684 matchedSuites.add(micros)
1685
1686 mx.run_java(['-jar', microJar, "-l"], cwd=jmhPath, out=_addBenchmark, addDefaultArgs=False)
1687 else:
1688 matchedSuites.add(micros)
1689
1690 mx.logv("matchedSuites: " + str(matchedSuites))
1691 plural = 's' if not benchmarks or numBench[0] > 1 else ''
1692 number = str(numBench[0]) if benchmarks else "all"
1693 mx.log("Running " + number + " benchmark" + plural + '...')
1694
1695 regex = []
1696 if benchmarks:
1697 regex.append(r".*(" + "|".join(benchmarks) + ").*")
1698
1699 for suite in matchedSuites:
1700 absoluteMicro = os.path.join(jmhPath, suite)
1701 (pfx, exe, vm, forkedVmArgs, _) = parseVmArgs(vmArgs)
1702 if pfx:
1703 mx.log("JMH ignores prefix: \"" + ' '.join(pfx) + "\"")
1704 javaArgs = ['-jar', os.path.join(absoluteMicro, "target", "microbenchmarks.jar"),
1705 '--jvm', exe,
1706 '--jvmArgs', ' '.join(["-" + vm] + forkedVmArgs)]
1707 for k, v in jmhArgs.iteritems():
1708 javaArgs.append(k)
1709 if len(str(v)):
1710 javaArgs.append(str(v))
1711 mx.run_java(javaArgs + regex, addDefaultArgs=False, cwd=jmhPath)
1712
1713 def microbench(args):
1714 """run JMH microbenchmark projects"""
1715 vmArgs, jmhArgs = mx.extract_VM_args(args, useDoubleDash=True)
1716
1717 # look for -f in JMH arguments
1718 containsF = False
1719 forking = True
1720 for i in range(len(jmhArgs)):
1721 arg = jmhArgs[i]
1722 if arg.startswith('-f'):
1723 containsF = True
1724 if arg == '-f' and (i+1) < len(jmhArgs):
1725 arg += jmhArgs[i+1]
1726 try:
1727 if int(arg[2:]) == 0:
1728 forking = False
1729 except ValueError:
1730 pass
1731
1732 # default to -f1 if not specified otherwise
1733 if not containsF:
1734 jmhArgs += ['-f1']
1735
1736 # find all projects with a direct JMH dependency
1737 jmhProjects = []
1738 for p in mx.projects():
1739 if 'JMH' in p.deps:
1740 jmhProjects.append(p.name)
1741 cp = mx.classpath(jmhProjects)
1742
1743 # execute JMH runner
1744 args = ['-cp', cp]
1745 if not forking:
1746 args += vmArgs
1747 args += ['org.openjdk.jmh.Main']
1748 if forking:
1749 (_, _, jvm, _, _) = parseVmArgs(vmArgs)
1750 args += ['--jvmArgsPrepend', ' '.join(['-' + jvm] + vmArgs)]
1751 vm(args + jmhArgs)
1752
1753 def hsdis(args, copyToDir=None):
1754 """download the hsdis library
1755
1756 This is needed to support HotSpot's assembly dumping features.
1757 By default it downloads the Intel syntax version, use the 'att' argument to install AT&T syntax."""
1758 flavor = 'intel'
1759 if 'att' in args:
1760 flavor = 'att'
1761 if mx.get_arch() == "sparcv9":
1762 flavor = "sparcv9"
1763 lib = mx.add_lib_suffix('hsdis-' + mx.get_arch())
1764 path = join(_suite.dir, 'lib', lib)
1765
1766 sha1s = {
1767 'att/hsdis-amd64.dll' : 'bcbd535a9568b5075ab41e96205e26a2bac64f72',
1768 'att/hsdis-amd64.so' : '58919ba085d4ef7a513f25bae75e7e54ee73c049',
1769 'intel/hsdis-amd64.dll' : '6a388372cdd5fe905c1a26ced614334e405d1f30',
1770 'intel/hsdis-amd64.so' : '844ed9ffed64fe9599638f29a8450c50140e3192',
1771 'intel/hsdis-amd64.dylib' : 'fdb13ef0d7d23d93dacaae9c98837bea0d4fc5a2',
1772 'sparcv9/hsdis-sparcv9.so': '970640a9af0bd63641f9063c11275b371a59ee60',
1773 }
1774
1775 flavoredLib = flavor + "/" + lib
1776 if flavoredLib not in sha1s:
1777 mx.logv("hsdis not supported on this plattform or architecture")
1778 return
1779
1780 if not exists(path):
1781 sha1 = sha1s[flavoredLib]
1782 sha1path = path + '.sha1'
1783 mx.download_file_with_sha1('hsdis', path, ['http://lafo.ssw.uni-linz.ac.at/hsdis/' + flavoredLib], sha1, sha1path, True, True, sources=False)
1784 if copyToDir is not None and exists(copyToDir):
1785 shutil.copy(path, copyToDir)
1786
1787 def hcfdis(args):
1788 """disassemble HexCodeFiles embedded in text files
1789
1790 Run a tool over the input files to convert all embedded HexCodeFiles
1791 to a disassembled format."""
1792
1793 parser = ArgumentParser(prog='mx hcfdis')
1794 parser.add_argument('-m', '--map', help='address to symbol map applied to disassembler output')
1795 parser.add_argument('files', nargs=REMAINDER, metavar='files...')
1796
1797 args = parser.parse_args(args)
1798
1799 path = mx.library('HCFDIS').get_path(resolve=True)
1800 mx.run_java(['-cp', path, 'com.oracle.max.hcfdis.HexCodeFileDis'] + args.files)
1801
1802 if args.map is not None:
1803 addressRE = re.compile(r'0[xX]([A-Fa-f0-9]+)')
1804 with open(args.map) as fp:
1805 lines = fp.read().splitlines()
1806 symbols = dict()
1807 for l in lines:
1808 addressAndSymbol = l.split(' ', 1)
1809 if len(addressAndSymbol) == 2:
1810 address, symbol = addressAndSymbol
1811 if address.startswith('0x'):
1812 address = long(address, 16)
1813 symbols[address] = symbol
1814 for f in args.files:
1815 with open(f) as fp:
1816 lines = fp.read().splitlines()
1817 updated = False
1818 for i in range(0, len(lines)):
1819 l = lines[i]
1820 for m in addressRE.finditer(l):
1821 sval = m.group(0)
1822 val = long(sval, 16)
1823 sym = symbols.get(val)
1824 if sym:
1825 l = l.replace(sval, sym)
1826 updated = True
1827 lines[i] = l
1828 if updated:
1829 mx.log('updating ' + f)
1830 with open('new_' + f, "w") as fp:
1831 for l in lines:
1832 print >> fp, l
1833
1834 def jacocoreport(args):
1835 """create a JaCoCo coverage report
1836
1837 Creates the report from the 'jacoco.exec' file in the current directory.
1838 Default output directory is 'coverage', but an alternative can be provided as an argument."""
1839 jacocoreport = mx.library("JACOCOREPORT", True)
1840 out = 'coverage'
1841 if len(args) == 1:
1842 out = args[0]
1843 elif len(args) > 1:
1844 mx.abort('jacocoreport takes only one argument : an output directory')
1845
1846 includes = ['com.oracle.graal', 'jdk.internal.jvmci']
1847 for p in mx.projects():
1848 projsetting = getattr(p, 'jacoco', '')
1849 if projsetting == 'include':
1850 includes.append(p.name)
1851
1852 includedirs = set()
1853 for p in mx.projects():
1854 projsetting = getattr(p, 'jacoco', '')
1855 if projsetting == 'exclude':
1856 continue
1857 for include in includes:
1858 if include in p.dir:
1859 includedirs.add(p.dir)
1860
1861 for i in includedirs:
1862 bindir = i + '/bin'
1863 if not os.path.exists(bindir):
1864 os.makedirs(bindir)
1865
1866 mx.run_java(['-jar', jacocoreport.get_path(True), '--in', 'jacoco.exec', '--out', out] + sorted(includedirs))
1867
1868 def isJVMCIEnabled(vm):
1869 return vm != 'original' and not vm.endswith('nojvmci')
1870
1871 def jol(args):
1872 """Java Object Layout"""
1873 joljar = mx.library('JOL_INTERNALS').get_path(resolve=True)
1874 candidates = mx.findclass(args, logToConsole=False, matcher=lambda s, classname: s == classname or classname.endswith('.' + s) or classname.endswith('$' + s))
1875
1876 if len(candidates) > 0:
1877 candidates = mx.select_items(sorted(candidates))
1878 else:
1879 # mx.findclass can be mistaken, don't give up yet
1880 candidates = args
1881
1882 vm(['-javaagent:' + joljar, '-cp', os.pathsep.join([mx.classpath(), joljar]), "org.openjdk.jol.MainObjectInternals"] + candidates)
1883
1884 def checkheaders(args):
1885 """check Java source headers against any required pattern"""
1886 failures = {}
1887 for p in mx.projects():
1888 if p.native:
1889 continue
1890
1891 csConfig = join(mx.project(p.checkstyleProj).dir, '.checkstyle_checks.xml')
1892 if not exists(csConfig):
1893 mx.log('Cannot check headers for ' + p.name + ' - ' + csConfig + ' does not exist')
1894 continue
1895 dom = xml.dom.minidom.parse(csConfig)
1896 for module in dom.getElementsByTagName('module'):
1897 if module.getAttribute('name') == 'RegexpHeader':
1898 for prop in module.getElementsByTagName('property'):
1899 if prop.getAttribute('name') == 'header':
1900 value = prop.getAttribute('value')
1901 matcher = re.compile(value, re.MULTILINE)
1902 for sourceDir in p.source_dirs():
1903 for root, _, files in os.walk(sourceDir):
1904 for name in files:
1905 if name.endswith('.java') and name != 'package-info.java':
1906 f = join(root, name)
1907 with open(f) as fp:
1908 content = fp.read()
1909 if not matcher.match(content):
1910 failures[f] = csConfig
1911 for n, v in failures.iteritems():
1912 mx.log('{0}: header does not match RegexpHeader defined in {1}'.format(n, v))
1913 return len(failures)
1914
1915 def mx_init(suite):
1916 commands = {
1917 'build': [build, ''],
1918 'buildjmh': [buildjmh, '[-options]'],
1919 'buildvars': [buildvars, ''],
1920 'buildvms': [buildvms, '[-options]'],
1921 'c1visualizer' : [c1visualizer, ''],
1922 'checkheaders': [checkheaders, ''],
1923 'clean': [clean, ''],
1924 'ctw': [ctw, '[-vmoptions|noinline|nocomplex|full]'],
1925 'export': [export, '[-options] [zipfile]'],
1926 'hsdis': [hsdis, '[att]'],
1927 'hcfdis': [hcfdis, ''],
1928 'igv' : [igv, ''],
1929 'jdkhome': [print_jdkhome, ''],
1930 'jmh': [jmh, '[VM options] [filters|JMH-args-as-json...]'],
1931 'gate' : [gate, '[-options]'],
1932 'makejmhdeps' : [makejmhdeps, ''],
1933 'unittest' : [unittest, '[unittest options] [--] [VM options] [filters...]', mx_unittest.unittestHelpSuffix],
1934 'shortunittest' : [shortunittest, '[unittest options] [--] [VM options] [filters...]', mx_unittest.unittestHelpSuffix],
1935 'jacocoreport' : [jacocoreport, '[output directory]'],
1936 'vm': [vm, '[-options] class [args...]'],
1937 'deoptalot' : [deoptalot, '[n]'],
1938 'longtests' : [longtests, ''],
1939 'jol' : [jol, ''],
1940 'makefile' : [mx_jvmci_makefile.build_makefile, 'build makefiles for JDK build'],
1941 }
1942
1943 mx.add_argument('--jacoco', help='instruments com.oracle.* classes using JaCoCo', default='off', choices=['off', 'on', 'append'])
1944 mx.add_argument('--vmcwd', dest='vm_cwd', help='current directory will be changed to <path> before the VM is executed', default=None, metavar='<path>')
1945 mx.add_argument('--installed-jdks', help='the base directory in which the JDKs cloned from $JAVA_HOME exist. ' +
1946 'The VM selected by --vm and --vmbuild options is under this directory (i.e., ' +
1947 join('<path>', '<jdk-version>', '<vmbuild>', 'jre', 'lib', '<vm>', mx.add_lib_prefix(mx.add_lib_suffix('jvm'))) + ')', default=None, metavar='<path>')
1948
1949 mx.add_argument('--vm', action='store', dest='vm', choices=_vmChoices.keys(), help='the VM type to build/run')
1950 mx.add_argument('--vmbuild', action='store', dest='vmbuild', choices=_vmbuildChoices, help='the VM build to build/run (default: ' + _vmbuildChoices[0] + ')')
1951 mx.add_argument('--ecl', action='store_true', dest='make_eclipse_launch', help='create launch configuration for running VM execution(s) in Eclipse')
1952 mx.add_argument('--vmprefix', action='store', dest='vm_prefix', help='prefix for running the VM (e.g. "/usr/bin/gdb --args")', metavar='<prefix>')
1953 mx.add_argument('--gdb', action='store_const', const='/usr/bin/gdb --args', dest='vm_prefix', help='alias for --vmprefix "/usr/bin/gdb --args"')
1954 mx.add_argument('--lldb', action='store_const', const='lldb --', dest='vm_prefix', help='alias for --vmprefix "lldb --"')
1955
1956 mx.update_commands(suite, commands)
1957
1958 class JVMCIArchiveParticipant:
1959 def __init__(self, dist):
1960 self.dist = dist
1961 self.jvmciServices = {}
1962
1963 def __opened__(self, arc, srcArc, services):
1964 self.services = services
1965 self.arc = arc
1966 self.expectedOptionsProviders = set()
1967
1968 def __add__(self, arcname, contents):
1969 if arcname.startswith('META-INF/jvmci.services/'):
1970 service = arcname[len('META-INF/jvmci.services/'):]
1971 self.jvmciServices.setdefault(service, []).extend([provider for provider in contents.split('\n')])
1972 return True
1973 if arcname.startswith('META-INF/jvmci.providers/'):
1974 provider = arcname[len('META-INF/jvmci.providers/'):]
1975 for service in contents.split(os.linesep):
1976 self.jvmciServices.setdefault(service, []).append(provider)
1977 return True
1978 elif arcname.startswith('META-INF/jvmci.options/'):
1979 # Need to create service files for the providers of the
1980 # jdk.internal.jvmci.options.Options service created by
1981 # jdk.internal.jvmci.options.processor.OptionProcessor.
1982 optionsOwner = arcname[len('META-INF/jvmci.options/'):]
1983 provider = optionsOwner + '_Options'
1984 self.expectedOptionsProviders.add(provider.replace('.', '/') + '.class')
1985 self.services.setdefault('jdk.internal.jvmci.options.Options', []).append(provider)
1986 return False
1987
1988 def __addsrc__(self, arcname, contents):
1989 return False
1990
1991 def __closing__(self):
1992 self.expectedOptionsProviders -= set(self.arc.zf.namelist())
1993 assert len(self.expectedOptionsProviders) == 0, 'missing generated Options providers:\n ' + '\n '.join(self.expectedOptionsProviders)
1994 for service, providers in self.jvmciServices.iteritems():
1995 arcname = 'META-INF/jvmci.services/' + service
1996 # Convert providers to a set before printing to remove duplicates
1997 self.arc.zf.writestr(arcname, '\n'.join(frozenset(providers)))
1998
1999 def mx_post_parse_cmd_line(opts): #
2000 # TODO _minVersion check could probably be part of a Suite in mx?
2001 def _versionCheck(version):
2002 return version >= _minVersion and (not _untilVersion or version >= _untilVersion)
2003 versionDesc = ">=" + str(_minVersion)
2004 if _untilVersion:
2005 versionDesc += " and <=" + str(_untilVersion)
2006 mx.java(_versionCheck, versionDescription=versionDesc, defaultJdk=True)
2007
2008 if hasattr(opts, 'vm') and opts.vm is not None:
2009 global _vm
2010 _vm = opts.vm
2011 _vm = _vm.replace('graal', 'jvmci')
2012 if hasattr(opts, 'vmbuild') and opts.vmbuild is not None:
2013 global _vmbuild
2014 _vmbuild = opts.vmbuild
2015 global _make_eclipse_launch
2016 _make_eclipse_launch = getattr(opts, 'make_eclipse_launch', False)
2017 global _jacoco
2018 _jacoco = opts.jacoco
2019 global _vm_cwd
2020 _vm_cwd = opts.vm_cwd
2021 global _installed_jdks
2022 _installed_jdks = opts.installed_jdks
2023 global _vm_prefix
2024 _vm_prefix = opts.vm_prefix
2025
2026 for jdkDist in jdkDeployedDists:
2027 def _close(jdkDeployable):
2028 def _install(dist):
2029 assert dist.name == jdkDeployable.name, dist.name + "!=" + jdkDeployable.name
2030 if not jdkDist.partOfHotSpot:
2031 _installDistInJdks(jdkDeployable)
2032 return _install
2033 dist = mx.distribution(jdkDist.name)
2034 dist.add_update_listener(_close(jdkDist))
2035 if jdkDist.usesJVMCIClassLoader:
2036 dist.set_archiveparticipant(JVMCIArchiveParticipant(dist))
2037