Mercurial > hg > graal-compiler
comparison mx.graal/mx_graal.py @ 22017:66dd063eb6a0
renamed mx/ to mx.graal/ in preparation for working with mxtool2
author | Doug Simon <doug.simon@oracle.com> |
---|---|
date | Wed, 17 Jun 2015 13:56:55 +0200 |
parents | mx/mx_graal.py@2a98e51646c2 |
children | 11ed27b2abe8 |
comparison
equal
deleted
inserted
replaced
22016:f2cf8824040b | 22017:66dd063eb6a0 |
---|---|
1 # | |
2 # commands.py - the GraalVM specific commands | |
3 # | |
4 # ---------------------------------------------------------------------------------------------------- | |
5 # | |
6 # Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved. | |
7 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. | |
8 # | |
9 # This code is free software; you can redistribute it and/or modify it | |
10 # under the terms of the GNU General Public License version 2 only, as | |
11 # published by the Free Software Foundation. | |
12 # | |
13 # This code is distributed in the hope that it will be useful, but WITHOUT | |
14 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
15 # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
16 # version 2 for more details (a copy is included in the LICENSE file that | |
17 # accompanied this code). | |
18 # | |
19 # You should have received a copy of the GNU General Public License version | |
20 # 2 along with this work; if not, write to the Free Software Foundation, | |
21 # Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |
22 # | |
23 # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA | |
24 # or visit www.oracle.com if you need additional information or have any | |
25 # questions. | |
26 # | |
27 # ---------------------------------------------------------------------------------------------------- | |
28 | |
29 import os, stat, errno, sys, shutil, zipfile, tarfile, tempfile, re, time, datetime, platform, subprocess, StringIO, socket | |
30 from os.path import join, exists, dirname, basename | |
31 from argparse import ArgumentParser, RawDescriptionHelpFormatter, REMAINDER | |
32 from outputparser import OutputParser, ValuesMatcher | |
33 import mx | |
34 import xml.dom.minidom | |
35 import sanitycheck | |
36 import itertools | |
37 import json, textwrap | |
38 import fnmatch | |
39 import mx_graal_makefile | |
40 | |
41 # This works because when mx loads this file, it makes sure __file__ gets an absolute path | |
42 _graal_home = dirname(dirname(__file__)) | |
43 | |
44 """ Used to distinguish an exported GraalVM (see 'mx export'). """ | |
45 _vmSourcesAvailable = exists(join(_graal_home, 'make')) and exists(join(_graal_home, 'src')) | |
46 | |
47 """ The VMs that can be built and run along with an optional description. Only VMs with a | |
48 description are listed in the dialogue for setting the default VM (see _get_vm()). """ | |
49 _vmChoices = { | |
50 'jvmci' : 'Normal compilation is performed with a tiered system (C1 + Graal), Truffle compilation is performed with Graal.', | |
51 'server' : 'Normal compilation is performed with a tiered system (C1 + C2), Truffle compilation is performed with Graal. Use this for optimal Truffle performance.', | |
52 'client' : None, # normal compilation with client compiler, explicit compilation (e.g., by Truffle) with Graal | |
53 'server-nojvmci' : None, # all compilation with tiered system (i.e., client + server), JVMCI omitted | |
54 'client-nojvmci' : None, # all compilation with client compiler, JVMCI omitted | |
55 'original' : None, # default VM copied from bootstrap JDK | |
56 'graal' : None, # alias for jvmci | |
57 'server-nograal' : None, # alias for server-nojvmci | |
58 'client-nograal' : None, # alias for client-nojvmci | |
59 } | |
60 | |
61 """ The VM that will be run by the 'vm' command and built by default by the 'build' command. | |
62 This can be set via the global '--vm' option or the DEFAULT_VM environment variable. | |
63 It can also be temporarily set by using of a VM context manager object in a 'with' statement. """ | |
64 _vm = None | |
65 | |
66 """ The VM builds that will be run by the 'vm' command - default is first in list """ | |
67 _vmbuildChoices = ['product', 'fastdebug', 'debug', 'optimized'] | |
68 | |
69 """ The VM build that will be run by the 'vm' command. | |
70 This can be set via the global '--vmbuild' option. | |
71 It can also be temporarily set by using of a VM context manager object in a 'with' statement. """ | |
72 _vmbuild = _vmbuildChoices[0] | |
73 | |
74 _jacoco = 'off' | |
75 | |
76 """ The current working directory to switch to before running the VM. """ | |
77 _vm_cwd = None | |
78 | |
79 """ The base directory in which the JDKs cloned from $JAVA_HOME exist. """ | |
80 _installed_jdks = None | |
81 | |
82 """ Prefix for running the VM. """ | |
83 _vm_prefix = None | |
84 | |
85 _make_eclipse_launch = False | |
86 | |
87 _minVersion = mx.VersionSpec('1.8') | |
88 | |
89 # max version (first _unsupported_ version) | |
90 _untilVersion = None | |
91 | |
92 class JDKDeployedDist: | |
93 def __init__(self, name, isExtension=False, usesJVMCIClassLoader=False, partOfHotSpot=False): | |
94 self.name = name | |
95 self.isExtension = isExtension | |
96 self.usesJVMCIClassLoader = usesJVMCIClassLoader | |
97 self.partOfHotSpot = partOfHotSpot # true when this distribution is delivered with HotSpot | |
98 | |
99 _jdkDeployedDists = [ | |
100 JDKDeployedDist('TRUFFLE'), | |
101 JDKDeployedDist('JVMCI_SERVICE', partOfHotSpot=True), | |
102 JDKDeployedDist('JVMCI_API', usesJVMCIClassLoader=True, partOfHotSpot=True), | |
103 JDKDeployedDist('JVMCI_HOTSPOT', usesJVMCIClassLoader=True, partOfHotSpot=True), | |
104 JDKDeployedDist('GRAAL', usesJVMCIClassLoader=True), | |
105 JDKDeployedDist('GRAAL_TRUFFLE', usesJVMCIClassLoader=True) | |
106 ] | |
107 | |
108 JDK_UNIX_PERMISSIONS_DIR = 0755 | |
109 JDK_UNIX_PERMISSIONS_FILE = 0644 | |
110 JDK_UNIX_PERMISSIONS_EXEC = 0755 | |
111 | |
112 def isVMSupported(vm): | |
113 if 'client' == vm and len(platform.mac_ver()[0]) != 0: | |
114 # Client VM not supported: java launcher on Mac OS X translates '-client' to '-server' | |
115 return False | |
116 return True | |
117 | |
118 def _get_vm(): | |
119 """ | |
120 Gets the configured VM, presenting a dialogue if there is no currently configured VM. | |
121 """ | |
122 global _vm | |
123 if _vm: | |
124 return _vm | |
125 vm = mx.get_env('DEFAULT_VM') | |
126 envPath = join(_graal_home, 'mx', 'env') | |
127 if vm and 'graal' in vm: | |
128 if exists(envPath): | |
129 with open(envPath) as fp: | |
130 if 'DEFAULT_VM=' + vm in fp.read(): | |
131 mx.log('Please update the DEFAULT_VM value in ' + envPath + ' to replace "graal" with "jvmci"') | |
132 vm = vm.replace('graal', 'jvmci') | |
133 if vm is None: | |
134 if not mx.is_interactive(): | |
135 mx.abort('Need to specify VM with --vm option or DEFAULT_VM environment variable') | |
136 mx.log('Please select the VM to be executed from the following: ') | |
137 items = [k for k in _vmChoices.keys() if _vmChoices[k] is not None] | |
138 descriptions = [_vmChoices[k] for k in _vmChoices.keys() if _vmChoices[k] is not None] | |
139 vm = mx.select_items(items, descriptions, allowMultiple=False) | |
140 mx.ask_persist_env('DEFAULT_VM', vm) | |
141 _vm = vm | |
142 return vm | |
143 | |
144 """ | |
145 A context manager that can be used with the 'with' statement to set the VM | |
146 used by all VM executions within the scope of the 'with' statement. For example: | |
147 | |
148 with VM('server'): | |
149 dacapo(['pmd']) | |
150 """ | |
151 class VM: | |
152 def __init__(self, vm=None, build=None): | |
153 assert vm is None or vm in _vmChoices.keys() | |
154 assert build is None or build in _vmbuildChoices | |
155 self.vm = vm if vm else _vm | |
156 self.build = build if build else _vmbuild | |
157 self.previousVm = _vm | |
158 self.previousBuild = _vmbuild | |
159 | |
160 def __enter__(self): | |
161 global _vm, _vmbuild | |
162 _vm = self.vm | |
163 _vmbuild = self.build | |
164 | |
165 def __exit__(self, exc_type, exc_value, traceback): | |
166 global _vm, _vmbuild | |
167 _vm = self.previousVm | |
168 _vmbuild = self.previousBuild | |
169 | |
170 def chmodRecursive(dirname, chmodFlagsDir): | |
171 if mx.get_os() == 'windows': | |
172 return | |
173 | |
174 def _chmodDir(chmodFlags, dirname, fnames): | |
175 os.chmod(dirname, chmodFlagsDir) | |
176 | |
177 os.path.walk(dirname, _chmodDir, chmodFlagsDir) | |
178 | |
179 def clean(args): | |
180 """clean the source tree""" | |
181 opts = mx.clean(args, parser=ArgumentParser(prog='mx clean')) | |
182 | |
183 if opts.native: | |
184 def handleRemoveReadonly(func, path, exc): | |
185 excvalue = exc[1] | |
186 if mx.get_os() == 'windows' and func in (os.rmdir, os.remove) and excvalue.errno == errno.EACCES: | |
187 os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # 0777 | |
188 func(path) | |
189 else: | |
190 raise | |
191 | |
192 def rmIfExists(name): | |
193 if os.path.isdir(name): | |
194 shutil.rmtree(name, ignore_errors=False, onerror=handleRemoveReadonly) | |
195 elif os.path.isfile(name): | |
196 os.unlink(name) | |
197 | |
198 rmIfExists(join(_graal_home, 'build')) | |
199 rmIfExists(join(_graal_home, 'build-nojvmci')) | |
200 rmIfExists(_jdksDir()) | |
201 | |
202 def export(args): | |
203 """create archives of builds split by vmbuild and vm""" | |
204 | |
205 parser = ArgumentParser(prog='mx export') | |
206 args = parser.parse_args(args) | |
207 | |
208 # collect data about export | |
209 infos = dict() | |
210 infos['timestamp'] = time.time() | |
211 | |
212 hgcfg = mx.HgConfig() | |
213 hgcfg.check() | |
214 infos['revision'] = hgcfg.tip('.') + ('+' if hgcfg.isDirty('.') else '') | |
215 # TODO: infos['repository'] | |
216 | |
217 infos['jdkversion'] = str(mx.java().version) | |
218 | |
219 infos['architecture'] = mx.get_arch() | |
220 infos['platform'] = mx.get_os() | |
221 | |
222 if mx.get_os != 'windows': | |
223 pass | |
224 # infos['ccompiler'] | |
225 # infos['linker'] | |
226 | |
227 infos['hostname'] = socket.gethostname() | |
228 | |
229 def _writeJson(suffix, properties): | |
230 d = infos.copy() | |
231 for k, v in properties.iteritems(): | |
232 assert not d.has_key(k) | |
233 d[k] = v | |
234 | |
235 jsonFileName = 'export-' + suffix + '.json' | |
236 with open(jsonFileName, 'w') as f: | |
237 print >> f, json.dumps(d) | |
238 return jsonFileName | |
239 | |
240 | |
241 def _genFileName(archivtype, middle): | |
242 idPrefix = infos['revision'] + '_' | |
243 idSuffix = '.tar.gz' | |
244 return join(_graal_home, "graalvm_" + archivtype + "_" + idPrefix + middle + idSuffix) | |
245 | |
246 def _genFileArchPlatformName(archivtype, middle): | |
247 return _genFileName(archivtype, infos['platform'] + '_' + infos['architecture'] + '_' + middle) | |
248 | |
249 | |
250 # archive different build types of hotspot | |
251 for vmBuild in _vmbuildChoices: | |
252 jdkpath = join(_jdksDir(), vmBuild) | |
253 if not exists(jdkpath): | |
254 mx.logv("skipping " + vmBuild) | |
255 continue | |
256 | |
257 tarName = _genFileArchPlatformName('basejdk', vmBuild) | |
258 mx.logv("creating basejdk " + tarName) | |
259 vmSet = set() | |
260 with tarfile.open(tarName, 'w:gz') as tar: | |
261 for root, _, files in os.walk(jdkpath): | |
262 if basename(root) in _vmChoices.keys(): | |
263 # TODO: add some assert to check path assumption | |
264 vmSet.add(root) | |
265 continue | |
266 | |
267 for f in files: | |
268 name = join(root, f) | |
269 # print name | |
270 tar.add(name, name) | |
271 | |
272 n = _writeJson("basejdk-" + vmBuild, {'vmbuild' : vmBuild}) | |
273 tar.add(n, n) | |
274 | |
275 # create a separate archive for each VM | |
276 for vm in vmSet: | |
277 bVm = basename(vm) | |
278 vmTarName = _genFileArchPlatformName('vm', vmBuild + '_' + bVm) | |
279 mx.logv("creating vm " + vmTarName) | |
280 | |
281 debugFiles = set() | |
282 with tarfile.open(vmTarName, 'w:gz') as tar: | |
283 for root, _, files in os.walk(vm): | |
284 for f in files: | |
285 # TODO: mac, windows, solaris? | |
286 if any(map(f.endswith, [".debuginfo"])): | |
287 debugFiles.add(f) | |
288 else: | |
289 name = join(root, f) | |
290 # print name | |
291 tar.add(name, name) | |
292 | |
293 n = _writeJson("vm-" + vmBuild + "-" + bVm, {'vmbuild' : vmBuild, 'vm' : bVm}) | |
294 tar.add(n, n) | |
295 | |
296 if len(debugFiles) > 0: | |
297 debugTarName = _genFileArchPlatformName('debugfilesvm', vmBuild + '_' + bVm) | |
298 mx.logv("creating debugfilesvm " + debugTarName) | |
299 with tarfile.open(debugTarName, 'w:gz') as tar: | |
300 for f in debugFiles: | |
301 name = join(root, f) | |
302 # print name | |
303 tar.add(name, name) | |
304 | |
305 n = _writeJson("debugfilesvm-" + vmBuild + "-" + bVm, {'vmbuild' : vmBuild, 'vm' : bVm}) | |
306 tar.add(n, n) | |
307 | |
308 # graal directory | |
309 graalDirTarName = _genFileName('classfiles', 'javac') | |
310 mx.logv("creating graal " + graalDirTarName) | |
311 with tarfile.open(graalDirTarName, 'w:gz') as tar: | |
312 for root, _, files in os.walk("graal"): | |
313 for f in [f for f in files if not f.endswith('.java')]: | |
314 name = join(root, f) | |
315 # print name | |
316 tar.add(name, name) | |
317 | |
318 n = _writeJson("graal", {'javacompiler' : 'javac'}) | |
319 tar.add(n, n) | |
320 | |
321 | |
322 def _run_benchmark(args, availableBenchmarks, runBenchmark): | |
323 | |
324 vmOpts, benchmarksAndOptions = _extract_VM_args(args, useDoubleDash=availableBenchmarks is None) | |
325 | |
326 if availableBenchmarks is None: | |
327 harnessArgs = benchmarksAndOptions | |
328 return runBenchmark(None, harnessArgs, vmOpts) | |
329 | |
330 if len(benchmarksAndOptions) == 0: | |
331 mx.abort('at least one benchmark name or "all" must be specified') | |
332 benchmarks = list(itertools.takewhile(lambda x: not x.startswith('-'), benchmarksAndOptions)) | |
333 harnessArgs = benchmarksAndOptions[len(benchmarks):] | |
334 | |
335 if 'all' in benchmarks: | |
336 benchmarks = availableBenchmarks | |
337 else: | |
338 for bm in benchmarks: | |
339 if bm not in availableBenchmarks: | |
340 mx.abort('unknown benchmark: ' + bm + '\nselect one of: ' + str(availableBenchmarks)) | |
341 | |
342 failed = [] | |
343 for bm in benchmarks: | |
344 if not runBenchmark(bm, harnessArgs, vmOpts): | |
345 failed.append(bm) | |
346 | |
347 if len(failed) != 0: | |
348 mx.abort('Benchmark failures: ' + str(failed)) | |
349 | |
350 def dacapo(args): | |
351 """run one or more DaCapo benchmarks""" | |
352 | |
353 def launcher(bm, harnessArgs, extraVmOpts): | |
354 return sanitycheck.getDacapo(bm, harnessArgs).test(_get_vm(), extraVmOpts=extraVmOpts) | |
355 | |
356 _run_benchmark(args, sanitycheck.dacapoSanityWarmup.keys(), launcher) | |
357 | |
358 def scaladacapo(args): | |
359 """run one or more Scala DaCapo benchmarks""" | |
360 | |
361 def launcher(bm, harnessArgs, extraVmOpts): | |
362 return sanitycheck.getScalaDacapo(bm, harnessArgs).test(_get_vm(), extraVmOpts=extraVmOpts) | |
363 | |
364 _run_benchmark(args, sanitycheck.dacapoScalaSanityWarmup.keys(), launcher) | |
365 | |
366 def _vmLibDirInJdk(jdk): | |
367 """ | |
368 Get the directory within a JDK where the server and client | |
369 subdirectories are located. | |
370 """ | |
371 mxos = mx.get_os() | |
372 if mxos == 'darwin': | |
373 return join(jdk, 'jre', 'lib') | |
374 if mxos == 'windows' or mxos == 'cygwin': | |
375 return join(jdk, 'jre', 'bin') | |
376 return join(jdk, 'jre', 'lib', mx.get_arch()) | |
377 | |
378 def _vmJliLibDirs(jdk): | |
379 """ | |
380 Get the directories within a JDK where the jli library designates to. | |
381 """ | |
382 mxos = mx.get_os() | |
383 if mxos == 'darwin': | |
384 return [join(jdk, 'jre', 'lib', 'jli')] | |
385 if mxos == 'windows' or mxos == 'cygwin': | |
386 return [join(jdk, 'jre', 'bin'), join(jdk, 'bin')] | |
387 return [join(jdk, 'jre', 'lib', mx.get_arch(), 'jli'), join(jdk, 'lib', mx.get_arch(), 'jli')] | |
388 | |
389 def _vmCfgInJdk(jdk, jvmCfgFile='jvm.cfg'): | |
390 """ | |
391 Get the jvm.cfg file. | |
392 """ | |
393 mxos = mx.get_os() | |
394 if mxos == "windows" or mxos == "cygwin": | |
395 return join(jdk, 'jre', 'lib', mx.get_arch(), jvmCfgFile) | |
396 return join(_vmLibDirInJdk(jdk), jvmCfgFile) | |
397 | |
398 def _jdksDir(): | |
399 return os.path.abspath(join(_installed_jdks if _installed_jdks else _graal_home, 'jdk' + str(mx.java().version))) | |
400 | |
401 def _handle_missing_VM(bld, vm=None): | |
402 if not vm: | |
403 vm = _get_vm() | |
404 mx.log('The ' + bld + ' ' + vm + ' VM has not been created') | |
405 if mx.is_interactive(): | |
406 if mx.ask_yes_no('Build it now', 'y'): | |
407 with VM(vm, bld): | |
408 build([]) | |
409 return | |
410 mx.abort('You need to run "mx --vm ' + vm + ' --vmbuild ' + bld + ' build" to build the selected VM') | |
411 | |
412 def _jdk(build=None, vmToCheck=None, create=False, installJars=True): | |
413 """ | |
414 Get the JDK into which Graal is installed, creating it first if necessary. | |
415 """ | |
416 if not build: | |
417 build = _vmbuild if _vmSourcesAvailable else 'product' | |
418 jdk = join(_jdksDir(), build) | |
419 if create: | |
420 srcJdk = mx.java().jdk | |
421 if not exists(jdk): | |
422 mx.log('Creating ' + jdk + ' from ' + srcJdk) | |
423 shutil.copytree(srcJdk, jdk) | |
424 | |
425 # Make a copy of the default VM so that this JDK can be | |
426 # reliably used as the bootstrap for a HotSpot build. | |
427 jvmCfg = _vmCfgInJdk(jdk) | |
428 if not exists(jvmCfg): | |
429 mx.abort(jvmCfg + ' does not exist') | |
430 | |
431 defaultVM = None | |
432 jvmCfgLines = [] | |
433 with open(jvmCfg) as f: | |
434 for line in f: | |
435 if line.startswith('-') and defaultVM is None: | |
436 parts = line.split() | |
437 if len(parts) == 2: | |
438 assert parts[1] == 'KNOWN', parts[1] | |
439 defaultVM = parts[0][1:] | |
440 jvmCfgLines += ['# default VM is a copy of the unmodified ' + defaultVM + ' VM\n'] | |
441 jvmCfgLines += ['-original KNOWN\n'] | |
442 else: | |
443 # skip lines which we cannot parse (e.g. '-hotspot ALIASED_TO -client') | |
444 mx.log("WARNING: skipping not parsable line \"" + line + "\"") | |
445 else: | |
446 jvmCfgLines += [line] | |
447 | |
448 assert defaultVM is not None, 'Could not find default VM in ' + jvmCfg | |
449 chmodRecursive(jdk, JDK_UNIX_PERMISSIONS_DIR) | |
450 shutil.move(join(_vmLibDirInJdk(jdk), defaultVM), join(_vmLibDirInJdk(jdk), 'original')) | |
451 | |
452 if mx.get_os() != 'windows': | |
453 os.chmod(jvmCfg, JDK_UNIX_PERMISSIONS_FILE) | |
454 with open(jvmCfg, 'w') as fp: | |
455 for line in jvmCfgLines: | |
456 fp.write(line) | |
457 | |
458 # patch 'release' file (append graalvm revision) | |
459 releaseFile = join(jdk, 'release') | |
460 if exists(releaseFile): | |
461 releaseFileLines = [] | |
462 with open(releaseFile) as f: | |
463 for line in f: | |
464 releaseFileLines.append(line) | |
465 | |
466 if mx.get_os() != 'windows': | |
467 os.chmod(releaseFile, JDK_UNIX_PERMISSIONS_FILE) | |
468 with open(releaseFile, 'w') as fp: | |
469 for line in releaseFileLines: | |
470 if line.startswith("SOURCE="): | |
471 try: | |
472 sourceLine = line[0:-2] # remove last char | |
473 hgcfg = mx.HgConfig() | |
474 hgcfg.check() | |
475 revision = hgcfg.tip('.')[:12] # take first 12 chars | |
476 fp.write(sourceLine + ' graal:' + revision + '\"\n') | |
477 except: | |
478 fp.write(line) | |
479 else: | |
480 fp.write(line) | |
481 | |
482 # Install a copy of the disassembler library | |
483 try: | |
484 hsdis([], copyToDir=_vmLibDirInJdk(jdk)) | |
485 except SystemExit: | |
486 pass | |
487 else: | |
488 if not exists(jdk): | |
489 if _installed_jdks: | |
490 mx.log("The selected JDK directory does not (yet) exist: " + jdk) | |
491 _handle_missing_VM(build, vmToCheck) | |
492 | |
493 if installJars: | |
494 for jdkDist in _jdkDeployedDists: | |
495 dist = mx.distribution(jdkDist.name) | |
496 if exists(dist.path) and jdkDist.partOfHotSpot: | |
497 _installDistInJdks(jdkDist) | |
498 | |
499 if vmToCheck is not None: | |
500 jvmCfg = _vmCfgInJdk(jdk) | |
501 found = False | |
502 with open(jvmCfg) as f: | |
503 for line in f: | |
504 if line.strip() == '-' + vmToCheck + ' KNOWN': | |
505 found = True | |
506 break | |
507 if not found: | |
508 _handle_missing_VM(build, vmToCheck) | |
509 | |
510 return jdk | |
511 | |
512 def _updateInstalledJVMCIOptionsFile(jdk): | |
513 jvmciOptions = join(_graal_home, 'jvmci.options') | |
514 jreLibDir = join(jdk, 'jre', 'lib') | |
515 if exists(jvmciOptions): | |
516 shutil.copy(jvmciOptions, join(jreLibDir, 'jvmci.options')) | |
517 else: | |
518 toDelete = join(jreLibDir, 'jvmci.options') | |
519 if exists(toDelete): | |
520 os.unlink(toDelete) | |
521 | |
522 def _makeHotspotGeneratedSourcesDir(): | |
523 """ | |
524 Gets the directory containing all the HotSpot sources generated from | |
525 JVMCI Java sources. This directory will be created if it doesn't yet exist. | |
526 """ | |
527 hsSrcGenDir = join(mx.project('com.oracle.jvmci.hotspot').source_gen_dir(), 'hotspot') | |
528 if not exists(hsSrcGenDir): | |
529 os.makedirs(hsSrcGenDir) | |
530 return hsSrcGenDir | |
531 | |
532 def _copyToJdk(src, dst, permissions=JDK_UNIX_PERMISSIONS_FILE): | |
533 name = os.path.basename(src) | |
534 dstLib = join(dst, name) | |
535 if mx.get_env('SYMLINK_GRAAL_JAR', None) == 'true': | |
536 # Using symlinks is much faster than copying but may | |
537 # cause issues if the lib is being updated while | |
538 # the VM is running. | |
539 if not os.path.islink(dstLib) or not os.path.realpath(dstLib) == src: | |
540 if exists(dstLib): | |
541 os.remove(dstLib) | |
542 os.symlink(src, dstLib) | |
543 else: | |
544 # do a copy and then a move to get atomic updating (on Unix) | |
545 fd, tmp = tempfile.mkstemp(suffix='', prefix=name, dir=dst) | |
546 shutil.copyfile(src, tmp) | |
547 os.close(fd) | |
548 shutil.move(tmp, dstLib) | |
549 os.chmod(dstLib, permissions) | |
550 | |
551 def _extractJVMCIFiles(jdkJars, jvmciJars, servicesDir, optionsDir): | |
552 if exists(servicesDir): | |
553 shutil.rmtree(servicesDir) | |
554 if exists(optionsDir): | |
555 shutil.rmtree(optionsDir) | |
556 if not exists(servicesDir): | |
557 os.makedirs(servicesDir) | |
558 if not exists(optionsDir): | |
559 os.makedirs(optionsDir) | |
560 jvmciServices = {} | |
561 optionsFiles = [] | |
562 for jar in jvmciJars: | |
563 if os.path.isfile(jar): | |
564 with zipfile.ZipFile(jar) as zf: | |
565 for member in zf.namelist(): | |
566 if member.startswith('META-INF/jvmci.services') and member: | |
567 service = basename(member) | |
568 if service == "": | |
569 continue # Zip files may contain empty entries for directories (jar -cf ... creates such) | |
570 # we don't handle directories | |
571 assert service and member == 'META-INF/jvmci.services/' + service | |
572 with zf.open(member) as serviceFile: | |
573 providers = jvmciServices.setdefault(service, []) | |
574 for line in serviceFile.readlines(): | |
575 line = line.strip() | |
576 if line: | |
577 providers.append(line) | |
578 elif member.startswith('META-INF/jvmci.options'): | |
579 filename = basename(member) | |
580 if filename == "": | |
581 continue # Zip files may contain empty entries for directories (jar -cf ... creates such) | |
582 # we don't handle directories | |
583 assert filename and member == 'META-INF/jvmci.options/' + filename | |
584 targetpath = join(optionsDir, filename) | |
585 optionsFiles.append(filename) | |
586 with zf.open(member) as optionsFile, \ | |
587 file(targetpath, "wb") as target: | |
588 shutil.copyfileobj(optionsFile, target) | |
589 for service, providers in jvmciServices.iteritems(): | |
590 fd, tmp = tempfile.mkstemp(prefix=service) | |
591 f = os.fdopen(fd, 'w+') | |
592 for provider in providers: | |
593 f.write(provider + os.linesep) | |
594 target = join(servicesDir, service) | |
595 f.close() | |
596 shutil.move(tmp, target) | |
597 if mx.get_os() != 'windows': | |
598 os.chmod(target, JDK_UNIX_PERMISSIONS_FILE) | |
599 | |
600 def _updateJVMCIFiles(jdkDir): | |
601 jreJVMCIDir = join(jdkDir, 'jre', 'lib', 'jvmci') | |
602 jvmciJars = [join(jreJVMCIDir, e) for e in os.listdir(jreJVMCIDir) if e.endswith('.jar')] | |
603 jreJVMCIServicesDir = join(jreJVMCIDir, 'services') | |
604 jreJVMCIOptionsDir = join(jreJVMCIDir, 'options') | |
605 _extractJVMCIFiles(_getJdkDeployedJars(jdkDir), jvmciJars, jreJVMCIServicesDir, jreJVMCIOptionsDir) | |
606 | |
607 def _updateGraalPropertiesFile(jreLibDir): | |
608 """ | |
609 Updates (or creates) 'jreLibDir'/jvmci/graal.properties to set/modify the | |
610 graal.version property. | |
611 """ | |
612 version = graal_version() | |
613 graalProperties = join(jreLibDir, 'jvmci', 'graal.properties') | |
614 if not exists(graalProperties): | |
615 with open(graalProperties, 'w') as fp: | |
616 print >> fp, 'graal.version=' + version | |
617 else: | |
618 content = [] | |
619 with open(graalProperties) as fp: | |
620 for line in fp: | |
621 if line.startswith('graal.version='): | |
622 content.append('graal.version=' + version) | |
623 else: | |
624 content.append(line.rstrip(os.linesep)) | |
625 with open(graalProperties, 'w') as fp: | |
626 fp.write(os.linesep.join(content)) | |
627 | |
628 def _installDistInJdks(deployableDist): | |
629 """ | |
630 Installs the jar(s) for a given Distribution into all existing JVMCI JDKs | |
631 """ | |
632 dist = mx.distribution(deployableDist.name) | |
633 jdks = _jdksDir() | |
634 if exists(jdks): | |
635 for e in os.listdir(jdks): | |
636 jdkDir = join(jdks, e) | |
637 jreLibDir = join(jdkDir, 'jre', 'lib') | |
638 if exists(jreLibDir): | |
639 if deployableDist.isExtension: | |
640 targetDir = join(jreLibDir, 'ext') | |
641 elif deployableDist.usesJVMCIClassLoader: | |
642 targetDir = join(jreLibDir, 'jvmci') | |
643 else: | |
644 targetDir = jreLibDir | |
645 if not exists(targetDir): | |
646 os.makedirs(targetDir) | |
647 _copyToJdk(dist.path, targetDir) | |
648 if dist.sourcesPath: | |
649 _copyToJdk(dist.sourcesPath, jdkDir) | |
650 if deployableDist.usesJVMCIClassLoader: | |
651 # deploy service files | |
652 _updateJVMCIFiles(jdkDir) | |
653 if dist.name == 'GRAAL': | |
654 _updateGraalPropertiesFile(jreLibDir) | |
655 | |
656 def _getJdkDeployedJars(jdkDir): | |
657 """ | |
658 Gets jar paths for all deployed distributions in the context of | |
659 a given JDK directory. | |
660 """ | |
661 jreLibDir = join(jdkDir, 'jre', 'lib') | |
662 jars = [] | |
663 for dist in _jdkDeployedDists: | |
664 jar = basename(mx.distribution(dist.name).path) | |
665 if dist.isExtension: | |
666 jars.append(join(jreLibDir, 'ext', jar)) | |
667 elif dist.usesJVMCIClassLoader: | |
668 jars.append(join(jreLibDir, 'jvmci', jar)) | |
669 else: | |
670 jars.append(join(jreLibDir, jar)) | |
671 return jars | |
672 | |
673 | |
674 # run a command in the windows SDK Debug Shell | |
675 def _runInDebugShell(cmd, workingDir, logFile=None, findInOutput=None, respondTo=None): | |
676 if respondTo is None: | |
677 respondTo = {} | |
678 newLine = os.linesep | |
679 startToken = 'RUNINDEBUGSHELL_STARTSEQUENCE' | |
680 endToken = 'RUNINDEBUGSHELL_ENDSEQUENCE' | |
681 | |
682 winSDK = mx.get_env('WIN_SDK', 'C:\\Program Files\\Microsoft SDKs\\Windows\\v7.1\\') | |
683 | |
684 if not exists(mx._cygpathW2U(winSDK)): | |
685 mx.abort("Could not find Windows SDK : '" + winSDK + "' does not exist") | |
686 | |
687 winSDKSetEnv = mx._cygpathW2U(join(winSDK, 'Bin', 'SetEnv.cmd')) | |
688 if not exists(winSDKSetEnv): | |
689 mx.abort("Invalid Windows SDK path (" + winSDK + ") : could not find Bin/SetEnv.cmd (you can use the WIN_SDK environment variable to specify an other path)") | |
690 | |
691 wincmd = 'cmd.exe /E:ON /V:ON /K "' + mx._cygpathU2W(winSDKSetEnv) + '"' | |
692 p = subprocess.Popen(wincmd, shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT) | |
693 stdout = p.stdout | |
694 stdin = p.stdin | |
695 if logFile: | |
696 log = open(logFile, 'w') | |
697 ret = False | |
698 | |
699 def _writeProcess(s): | |
700 stdin.write(s + newLine) | |
701 | |
702 _writeProcess("echo " + startToken) | |
703 while True: | |
704 # encoding may be None on windows plattforms | |
705 if sys.stdout.encoding is None: | |
706 encoding = 'utf-8' | |
707 else: | |
708 encoding = sys.stdout.encoding | |
709 | |
710 line = stdout.readline().decode(encoding) | |
711 if logFile: | |
712 log.write(line.encode('utf-8')) | |
713 line = line.strip() | |
714 mx.log(line) | |
715 if line == startToken: | |
716 _writeProcess('cd /D ' + workingDir + ' & ' + cmd + ' & echo ' + endToken) | |
717 for regex in respondTo.keys(): | |
718 match = regex.search(line) | |
719 if match: | |
720 _writeProcess(respondTo[regex]) | |
721 if findInOutput: | |
722 match = findInOutput.search(line) | |
723 if match: | |
724 ret = True | |
725 if line == endToken: | |
726 if not findInOutput: | |
727 _writeProcess('echo ERRXXX%errorlevel%') | |
728 else: | |
729 break | |
730 if line.startswith('ERRXXX'): | |
731 if line == 'ERRXXX0': | |
732 ret = True | |
733 break | |
734 _writeProcess("exit") | |
735 if logFile: | |
736 log.close() | |
737 return ret | |
738 | |
739 def jdkhome(vm=None): | |
740 """return the JDK directory selected for the 'vm' command""" | |
741 return _jdk(installJars=False) | |
742 | |
743 def print_jdkhome(args, vm=None): | |
744 """print the JDK directory selected for the 'vm' command""" | |
745 print jdkhome(vm) | |
746 | |
747 def buildvars(args): | |
748 """describe the variables that can be set by the -D option to the 'mx build' commmand""" | |
749 | |
750 buildVars = { | |
751 'ALT_BOOTDIR' : 'The location of the bootstrap JDK installation (default: ' + mx.java().jdk + ')', | |
752 'ALT_OUTPUTDIR' : 'Build directory', | |
753 'HOTSPOT_BUILD_JOBS' : 'Number of CPUs used by make (default: ' + str(mx.cpu_count()) + ')', | |
754 'INSTALL' : 'Install the built VM into the JDK? (default: y)', | |
755 'ZIP_DEBUGINFO_FILES' : 'Install zipped debug symbols file? (default: 0)', | |
756 } | |
757 | |
758 mx.log('HotSpot build variables that can be set by the -D option to "mx build":') | |
759 mx.log('') | |
760 for n in sorted(buildVars.iterkeys()): | |
761 mx.log(n) | |
762 mx.log(textwrap.fill(buildVars[n], initial_indent=' ', subsequent_indent=' ', width=200)) | |
763 | |
764 mx.log('') | |
765 mx.log('Note that these variables can be given persistent values in the file ' + join(_graal_home, 'mx', 'env') + ' (see \'mx about\').') | |
766 | |
767 cached_graal_version = None | |
768 | |
769 def graal_version(dev_suffix='dev'): | |
770 global cached_graal_version | |
771 | |
772 if not cached_graal_version: | |
773 # extract latest release tag for graal | |
774 try: | |
775 tags = [x.split() for x in subprocess.check_output(['hg', '-R', _graal_home, 'tags']).split('\n') if x.startswith("graal-")] | |
776 current_id = subprocess.check_output(['hg', '-R', _graal_home, 'log', '--template', '{rev}\n', '--rev', 'tip']).strip() | |
777 except: | |
778 # not a mercurial repository or hg commands are not available. | |
779 tags = None | |
780 | |
781 if tags and current_id: | |
782 sorted_tags = sorted(tags, key=lambda e: [int(x) for x in e[0][len("graal-"):].split('.')], reverse=True) | |
783 most_recent_tag_name, most_recent_tag_revision = sorted_tags[0] | |
784 most_recent_tag_id = most_recent_tag_revision[:most_recent_tag_revision.index(":")] | |
785 most_recent_tag_version = most_recent_tag_name[len("graal-"):] | |
786 | |
787 # tagged commit is one-off with commit that tags it | |
788 if int(current_id) - int(most_recent_tag_id) <= 1: | |
789 cached_graal_version = most_recent_tag_version | |
790 else: | |
791 major, minor = map(int, most_recent_tag_version.split('.')) | |
792 cached_graal_version = str(major) + '.' + str(minor + 1) + '-' + dev_suffix | |
793 else: | |
794 cached_graal_version = 'unknown-{0}-{1}'.format(platform.node(), time.strftime('%Y-%m-%d_%H-%M-%S_%Z')) | |
795 | |
796 return cached_graal_version | |
797 | |
798 def build(args, vm=None): | |
799 """build the VM binary | |
800 | |
801 The global '--vm' and '--vmbuild' options select which VM type and build target to build.""" | |
802 | |
803 # Override to fail quickly if extra arguments are given | |
804 # at the end of the command line. This allows for a more | |
805 # helpful error message. | |
806 class AP(ArgumentParser): | |
807 def __init__(self): | |
808 ArgumentParser.__init__(self, prog='mx build') | |
809 def parse_args(self, args): | |
810 result = ArgumentParser.parse_args(self, args) | |
811 if len(result.remainder) != 0: | |
812 firstBuildTarget = result.remainder[0] | |
813 mx.abort('To specify the ' + firstBuildTarget + ' VM build target, you need to use the global "--vmbuild" option. For example:\n' + | |
814 ' mx --vmbuild ' + firstBuildTarget + ' build') | |
815 return result | |
816 | |
817 # Call mx.build to compile the Java sources | |
818 parser = AP() | |
819 parser.add_argument('-D', action='append', help='set a HotSpot build variable (run \'mx buildvars\' to list variables)', metavar='name=value') | |
820 | |
821 opts2 = mx.build(['--source', '1.7'] + args, parser=parser) | |
822 assert len(opts2.remainder) == 0 | |
823 | |
824 if not _vmSourcesAvailable or not opts2.native: | |
825 return | |
826 | |
827 builds = [_vmbuild] | |
828 | |
829 if os.environ.get('BUILDING_FROM_IDE', None) == 'true': | |
830 build = os.environ.get('IDE_BUILD_TARGET', None) | |
831 if build is None or len(build) == 0: | |
832 return | |
833 if build not in _vmbuildChoices: | |
834 mx.abort('VM build "' + build + '" specified by IDE_BUILD_TARGET environment variable is unknown (must be one of ' + | |
835 str(_vmbuildChoices) + ')') | |
836 builds = [build] | |
837 | |
838 if vm is None: | |
839 vm = _get_vm() | |
840 | |
841 if vm == 'original': | |
842 pass | |
843 elif vm.startswith('server'): | |
844 buildSuffix = '' | |
845 elif vm.startswith('client'): | |
846 buildSuffix = '1' | |
847 else: | |
848 assert vm == 'jvmci', vm | |
849 buildSuffix = 'jvmci' | |
850 | |
851 if _installed_jdks and _installed_jdks != _graal_home: | |
852 if not mx.ask_yes_no("Warning: building while --installed-jdks is set (" + _installed_jdks + ") is not recommanded - are you sure you want to continue", 'n'): | |
853 mx.abort(1) | |
854 | |
855 isWindows = platform.system() == 'Windows' or "CYGWIN" in platform.system() | |
856 for build in builds: | |
857 installJars = vm != 'original' and (isWindows or not opts2.java) | |
858 jdk = _jdk(build, create=True, installJars=installJars) | |
859 | |
860 if vm == 'original': | |
861 if build != 'product': | |
862 mx.log('only product build of original VM exists') | |
863 continue | |
864 | |
865 if not isVMSupported(vm): | |
866 mx.log('The ' + vm + ' VM is not supported on this platform - skipping') | |
867 continue | |
868 | |
869 vmDir = join(_vmLibDirInJdk(jdk), vm) | |
870 if not exists(vmDir): | |
871 chmodRecursive(jdk, JDK_UNIX_PERMISSIONS_DIR) | |
872 mx.log('Creating VM directory in JDK: ' + vmDir) | |
873 os.makedirs(vmDir) | |
874 | |
875 def filterXusage(line): | |
876 if not 'Xusage.txt' in line: | |
877 sys.stderr.write(line + os.linesep) | |
878 | |
879 # Check if a build really needs to be done | |
880 timestampFile = join(vmDir, '.build-timestamp') | |
881 if opts2.force or not exists(timestampFile): | |
882 mustBuild = True | |
883 else: | |
884 mustBuild = False | |
885 timestamp = os.path.getmtime(timestampFile) | |
886 sources = [] | |
887 for d in ['src', 'make', join('jvmci', 'com.oracle.jvmci.hotspot', 'src_gen', 'hotspot')]: | |
888 for root, dirnames, files in os.walk(join(_graal_home, d)): | |
889 # ignore <graal>/src/share/tools | |
890 if root == join(_graal_home, 'src', 'share'): | |
891 dirnames.remove('tools') | |
892 sources += [join(root, name) for name in files] | |
893 for f in sources: | |
894 if len(f) != 0 and os.path.getmtime(f) > timestamp: | |
895 mustBuild = True | |
896 break | |
897 | |
898 if not mustBuild: | |
899 mx.logv('[all files in src and make directories are older than ' + timestampFile[len(_graal_home) + 1:] + ' - skipping native build]') | |
900 continue | |
901 | |
902 if isWindows: | |
903 t_compilelogfile = mx._cygpathU2W(os.path.join(_graal_home, "graalCompile.log")) | |
904 mksHome = mx.get_env('MKS_HOME', 'C:\\cygwin\\bin') | |
905 | |
906 variant = {'client': 'compiler1', 'server': 'compiler2'}.get(vm, vm) | |
907 project_config = variant + '_' + build | |
908 t_graal_home = mx._cygpathU2W(_graal_home) | |
909 _runInDebugShell('msbuild ' + t_graal_home + r'\build\vs-amd64\jvm.vcproj /p:Configuration=' + project_config + ' /target:clean', t_graal_home) | |
910 winCompileCmd = r'set HotSpotMksHome=' + mksHome + r'& set OUT_DIR=' + mx._cygpathU2W(jdk) + r'& set JAVA_HOME=' + mx._cygpathU2W(jdk) + r'& set path=%JAVA_HOME%\bin;%path%;%HotSpotMksHome%& cd /D "' + t_graal_home + r'\make\windows"& call create.bat ' + t_graal_home | |
911 print winCompileCmd | |
912 winCompileSuccess = re.compile(r"^Writing \.vcxproj file:") | |
913 if not _runInDebugShell(winCompileCmd, t_graal_home, t_compilelogfile, winCompileSuccess): | |
914 mx.log('Error executing create command') | |
915 return | |
916 winBuildCmd = 'msbuild ' + t_graal_home + r'\build\vs-amd64\jvm.vcxproj /p:Configuration=' + project_config + ' /p:Platform=x64' | |
917 if not _runInDebugShell(winBuildCmd, t_graal_home, t_compilelogfile): | |
918 mx.log('Error building project') | |
919 return | |
920 else: | |
921 cpus = mx.cpu_count() | |
922 makeDir = join(_graal_home, 'make') | |
923 runCmd = [mx.gmake_cmd(), '-C', makeDir] | |
924 | |
925 env = os.environ.copy() | |
926 | |
927 # These must be passed as environment variables | |
928 env.setdefault('LANG', 'C') | |
929 env['JAVA_HOME'] = jdk | |
930 | |
931 def setMakeVar(name, default, env=None): | |
932 """Sets a make variable on the command line to the value | |
933 of the variable in 'env' with the same name if defined | |
934 and 'env' is not None otherwise to 'default' | |
935 """ | |
936 runCmd.append(name + '=' + (env.get(name, default) if env else default)) | |
937 | |
938 if opts2.D: | |
939 for nv in opts2.D: | |
940 name, value = nv.split('=', 1) | |
941 setMakeVar(name.strip(), value) | |
942 | |
943 setMakeVar('ARCH_DATA_MODEL', '64', env=env) | |
944 setMakeVar('HOTSPOT_BUILD_JOBS', str(cpus), env=env) | |
945 setMakeVar('ALT_BOOTDIR', mx.java().jdk, env=env) | |
946 setMakeVar("EXPORT_PATH", jdk) | |
947 | |
948 setMakeVar('MAKE_VERBOSE', 'y' if mx._opts.verbose else '') | |
949 if vm.endswith('nojvmci'): | |
950 setMakeVar('INCLUDE_JVMCI', 'false') | |
951 setMakeVar('ALT_OUTPUTDIR', join(_graal_home, 'build-nojvmci', mx.get_os()), env=env) | |
952 else: | |
953 version = graal_version() | |
954 setMakeVar('USER_RELEASE_SUFFIX', 'jvmci-' + version) | |
955 setMakeVar('INCLUDE_JVMCI', 'true') | |
956 setMakeVar('INSTALL', 'y', env=env) | |
957 if mx.get_os() == 'darwin' and platform.mac_ver()[0] != '': | |
958 # Force use of clang on MacOS | |
959 setMakeVar('USE_CLANG', 'true') | |
960 if mx.get_os() == 'solaris': | |
961 # If using sparcWorks, setup flags to avoid make complaining about CC version | |
962 cCompilerVersion = subprocess.Popen('CC -V', stdout=subprocess.PIPE, stderr=subprocess.PIPE, shell=True).stderr.readlines()[0] | |
963 if cCompilerVersion.startswith('CC: Sun C++'): | |
964 compilerRev = cCompilerVersion.split(' ')[3] | |
965 setMakeVar('ENFORCE_COMPILER_REV', compilerRev, env=env) | |
966 setMakeVar('ENFORCE_CC_COMPILER_REV', compilerRev, env=env) | |
967 if build == 'jvmg': | |
968 # We want ALL the symbols when debugging on Solaris | |
969 setMakeVar('STRIP_POLICY', 'no_strip') | |
970 # This removes the need to unzip the *.diz files before debugging in gdb | |
971 setMakeVar('ZIP_DEBUGINFO_FILES', '0', env=env) | |
972 | |
973 if buildSuffix == "1": | |
974 setMakeVar("BUILD_CLIENT_ONLY", "true") | |
975 | |
976 # Clear this variable as having it set can cause very confusing build problems | |
977 env.pop('CLASSPATH', None) | |
978 | |
979 # Issue an env prefix that can be used to run the make on the command line | |
980 if not mx._opts.verbose: | |
981 mx.log('--------------- make command line ----------------------') | |
982 | |
983 envPrefix = ' '.join([key + '=' + env[key] for key in env.iterkeys() if not os.environ.has_key(key) or env[key] != os.environ[key]]) | |
984 if len(envPrefix): | |
985 mx.log('env ' + envPrefix + ' \\') | |
986 | |
987 runCmd.append(build + buildSuffix) | |
988 runCmd.append("docs") | |
989 runCmd.append("export_" + build) | |
990 | |
991 if not mx._opts.verbose: | |
992 mx.log(' '.join(runCmd)) | |
993 mx.log('--------------------------------------------------------') | |
994 mx.run(runCmd, err=filterXusage, env=env) | |
995 | |
996 jvmCfg = _vmCfgInJdk(jdk) | |
997 if not exists(jvmCfg): | |
998 mx.abort(jvmCfg + ' does not exist') | |
999 | |
1000 prefix = '-' + vm + ' ' | |
1001 vmKnown = prefix + 'KNOWN\n' | |
1002 lines = [] | |
1003 found = False | |
1004 with open(jvmCfg) as f: | |
1005 for line in f: | |
1006 if line.strip() == vmKnown.strip(): | |
1007 found = True | |
1008 lines.append(line) | |
1009 | |
1010 if not found: | |
1011 mx.log('Prepending "' + prefix + 'KNOWN" to ' + jvmCfg) | |
1012 if mx.get_os() != 'windows': | |
1013 os.chmod(jvmCfg, JDK_UNIX_PERMISSIONS_FILE) | |
1014 with open(jvmCfg, 'w') as f: | |
1015 written = False | |
1016 for line in lines: | |
1017 if line.startswith('#'): | |
1018 f.write(line) | |
1019 continue | |
1020 if not written: | |
1021 f.write(vmKnown) | |
1022 if vm == 'jvmci': | |
1023 # Legacy support | |
1024 f.write('-graal ALIASED_TO -jvmci\n') | |
1025 written = True | |
1026 if line.startswith(prefix): | |
1027 line = vmKnown | |
1028 if written: | |
1029 continue | |
1030 f.write(line) | |
1031 | |
1032 for jdkDist in _jdkDeployedDists: # Install non HotSpot distribution | |
1033 if not jdkDist.partOfHotSpot: | |
1034 _installDistInJdks(jdkDist) | |
1035 if exists(timestampFile): | |
1036 os.utime(timestampFile, None) | |
1037 else: | |
1038 file(timestampFile, 'a') | |
1039 | |
1040 def vmg(args): | |
1041 """run the debug build of VM selected by the '--vm' option""" | |
1042 return vm(args, vmbuild='debug') | |
1043 | |
1044 def vmfg(args): | |
1045 """run the fastdebug build of VM selected by the '--vm' option""" | |
1046 return vm(args, vmbuild='fastdebug') | |
1047 | |
1048 def _parseVmArgs(args, vm=None, cwd=None, vmbuild=None): | |
1049 """run the VM selected by the '--vm' option""" | |
1050 | |
1051 if vm is None: | |
1052 vm = _get_vm() | |
1053 | |
1054 if not isVMSupported(vm): | |
1055 mx.abort('The ' + vm + ' is not supported on this platform') | |
1056 | |
1057 if cwd is None: | |
1058 cwd = _vm_cwd | |
1059 elif _vm_cwd is not None and _vm_cwd != cwd: | |
1060 mx.abort("conflicting working directories: do not set --vmcwd for this command") | |
1061 | |
1062 build = vmbuild if vmbuild else _vmbuild if _vmSourcesAvailable else 'product' | |
1063 jdk = _jdk(build, vmToCheck=vm, installJars=False) | |
1064 _updateInstalledJVMCIOptionsFile(jdk) | |
1065 mx.expand_project_in_args(args) | |
1066 if _make_eclipse_launch: | |
1067 mx.make_eclipse_launch(args, 'graal-' + build, name=None, deps=mx.project('com.oracle.graal.hotspot').all_deps([], True)) | |
1068 if _jacoco == 'on' or _jacoco == 'append': | |
1069 jacocoagent = mx.library("JACOCOAGENT", True) | |
1070 # Exclude all compiler tests and snippets | |
1071 | |
1072 includes = ['com.oracle.graal.*', 'com.oracle.jvmci.*'] | |
1073 baseExcludes = [] | |
1074 for p in mx.projects(): | |
1075 projsetting = getattr(p, 'jacoco', '') | |
1076 if projsetting == 'exclude': | |
1077 baseExcludes.append(p.name) | |
1078 if projsetting == 'include': | |
1079 includes.append(p.name + '.*') | |
1080 | |
1081 def _filter(l): | |
1082 # filter out specific classes which are already covered by a baseExclude package | |
1083 return [clazz for clazz in l if not any([clazz.startswith(package) for package in baseExcludes])] | |
1084 excludes = [] | |
1085 for p in mx.projects(): | |
1086 excludes += _filter(_find_classes_with_annotations(p, None, ['@Snippet', '@ClassSubstitution', '@Test'], includeInnerClasses=True).keys()) | |
1087 excludes += _filter(p.find_classes_with_matching_source_line(None, lambda line: 'JaCoCo Exclude' in line, includeInnerClasses=True).keys()) | |
1088 | |
1089 excludes += [package + '.*' for package in baseExcludes] | |
1090 agentOptions = { | |
1091 'append' : 'true' if _jacoco == 'append' else 'false', | |
1092 'bootclasspath' : 'true', | |
1093 'includes' : ':'.join(includes), | |
1094 'excludes' : ':'.join(excludes), | |
1095 'destfile' : 'jacoco.exec' | |
1096 } | |
1097 args = ['-javaagent:' + jacocoagent.get_path(True) + '=' + ','.join([k + '=' + v for k, v in agentOptions.items()])] + args | |
1098 exe = join(jdk, 'bin', mx.exe_suffix('java')) | |
1099 pfx = _vm_prefix.split() if _vm_prefix is not None else [] | |
1100 | |
1101 if '-version' in args: | |
1102 ignoredArgs = args[args.index('-version') + 1:] | |
1103 if len(ignoredArgs) > 0: | |
1104 mx.log("Warning: The following options will be ignored by the vm because they come after the '-version' argument: " + ' '.join(ignoredArgs)) | |
1105 | |
1106 # Unconditionally prepend Truffle to the boot class path. | |
1107 # This used to be done by the VM itself but was removed to | |
1108 # separate the VM from Truffle. | |
1109 truffle_jar = mx.archive(['@TRUFFLE'])[0] | |
1110 args = ['-Xbootclasspath/p:' + truffle_jar] + args | |
1111 | |
1112 args = mx.java().processArgs(args) | |
1113 return (pfx, exe, vm, args, cwd) | |
1114 | |
1115 def vm(args, vm=None, nonZeroIsFatal=True, out=None, err=None, cwd=None, timeout=None, vmbuild=None): | |
1116 (pfx_, exe_, vm_, args_, cwd) = _parseVmArgs(args, vm, cwd, vmbuild) | |
1117 return mx.run(pfx_ + [exe_, '-' + vm_] + args_, nonZeroIsFatal=nonZeroIsFatal, out=out, err=err, cwd=cwd, timeout=timeout) | |
1118 | |
1119 def _find_classes_with_annotations(p, pkgRoot, annotations, includeInnerClasses=False): | |
1120 """ | |
1121 Scan the sources of project 'p' for Java source files containing a line starting with 'annotation' | |
1122 (ignoring preceding whitespace) and return the fully qualified class name for each Java | |
1123 source file matched in a list. | |
1124 """ | |
1125 | |
1126 matches = lambda line: len([a for a in annotations if line == a or line.startswith(a + '(')]) != 0 | |
1127 return p.find_classes_with_matching_source_line(pkgRoot, matches, includeInnerClasses) | |
1128 | |
1129 def _extract_VM_args(args, allowClasspath=False, useDoubleDash=False, defaultAllVMArgs=True): | |
1130 """ | |
1131 Partitions a command line into a leading sequence of HotSpot VM options and the rest. | |
1132 """ | |
1133 for i in range(0, len(args)): | |
1134 if useDoubleDash: | |
1135 if args[i] == '--': | |
1136 vmArgs = args[:i] | |
1137 remainder = args[i + 1:] | |
1138 return vmArgs, remainder | |
1139 else: | |
1140 if not args[i].startswith('-'): | |
1141 if i != 0 and (args[i - 1] == '-cp' or args[i - 1] == '-classpath'): | |
1142 if not allowClasspath: | |
1143 mx.abort('Cannot supply explicit class path option') | |
1144 else: | |
1145 continue | |
1146 vmArgs = args[:i] | |
1147 remainder = args[i:] | |
1148 return vmArgs, remainder | |
1149 | |
1150 if defaultAllVMArgs: | |
1151 return args, [] | |
1152 else: | |
1153 return [], args | |
1154 | |
1155 def _run_tests(args, harness, annotations, testfile, blacklist, whitelist, regex): | |
1156 | |
1157 | |
1158 vmArgs, tests = _extract_VM_args(args) | |
1159 for t in tests: | |
1160 if t.startswith('-'): | |
1161 mx.abort('VM option ' + t + ' must precede ' + tests[0]) | |
1162 | |
1163 candidates = {} | |
1164 for p in mx.projects_opt_limit_to_suites(): | |
1165 if mx.java().javaCompliance < p.javaCompliance: | |
1166 continue | |
1167 for c in _find_classes_with_annotations(p, None, annotations).keys(): | |
1168 candidates[c] = p | |
1169 | |
1170 classes = [] | |
1171 if len(tests) == 0: | |
1172 classes = candidates.keys() | |
1173 projectsCp = mx.classpath([pcp.name for pcp in mx.projects_opt_limit_to_suites() if pcp.javaCompliance <= mx.java().javaCompliance]) | |
1174 else: | |
1175 projs = set() | |
1176 found = False | |
1177 if len(tests) == 1 and '#' in tests[0]: | |
1178 words = tests[0].split('#') | |
1179 if len(words) != 2: | |
1180 mx.abort("Method specification is class#method: " + tests[0]) | |
1181 t, method = words | |
1182 | |
1183 for c, p in candidates.iteritems(): | |
1184 # prefer exact matches first | |
1185 if t == c: | |
1186 found = True | |
1187 classes.append(c) | |
1188 projs.add(p.name) | |
1189 if not found: | |
1190 for c, p in candidates.iteritems(): | |
1191 if t in c: | |
1192 found = True | |
1193 classes.append(c) | |
1194 projs.add(p.name) | |
1195 if not found: | |
1196 mx.log('warning: no tests matched by substring "' + t) | |
1197 elif len(classes) != 1: | |
1198 mx.abort('More than one test matches substring {0} {1}'.format(t, classes)) | |
1199 | |
1200 classes = [c + "#" + method for c in classes] | |
1201 else: | |
1202 for t in tests: | |
1203 if '#' in t: | |
1204 mx.abort('Method specifications can only be used in a single test: ' + t) | |
1205 for c, p in candidates.iteritems(): | |
1206 if t in c: | |
1207 found = True | |
1208 classes.append(c) | |
1209 projs.add(p.name) | |
1210 if not found: | |
1211 mx.log('warning: no tests matched by substring "' + t) | |
1212 projectsCp = mx.classpath(projs) | |
1213 | |
1214 if blacklist: | |
1215 classes = [c for c in classes if not any((glob.match(c) for glob in blacklist))] | |
1216 | |
1217 if whitelist: | |
1218 classes = [c for c in classes if any((glob.match(c) for glob in whitelist))] | |
1219 | |
1220 if regex: | |
1221 classes = [c for c in classes if re.search(regex, c)] | |
1222 | |
1223 if len(classes) != 0: | |
1224 f_testfile = open(testfile, 'w') | |
1225 for c in classes: | |
1226 f_testfile.write(c + '\n') | |
1227 f_testfile.close() | |
1228 harness(projectsCp, vmArgs) | |
1229 | |
1230 def _unittest(args, annotations, prefixCp="", blacklist=None, whitelist=None, verbose=False, fail_fast=False, enable_timing=False, regex=None, color=False, eager_stacktrace=False, gc_after_test=False): | |
1231 testfile = os.environ.get('MX_TESTFILE', None) | |
1232 if testfile is None: | |
1233 (_, testfile) = tempfile.mkstemp(".testclasses", "graal") | |
1234 os.close(_) | |
1235 | |
1236 coreCp = mx.classpath(['com.oracle.graal.test', 'HCFDIS']) | |
1237 | |
1238 coreArgs = [] | |
1239 if verbose: | |
1240 coreArgs.append('-JUnitVerbose') | |
1241 if fail_fast: | |
1242 coreArgs.append('-JUnitFailFast') | |
1243 if enable_timing: | |
1244 coreArgs.append('-JUnitEnableTiming') | |
1245 if color: | |
1246 coreArgs.append('-JUnitColor') | |
1247 if eager_stacktrace: | |
1248 coreArgs.append('-JUnitEagerStackTrace') | |
1249 if gc_after_test: | |
1250 coreArgs.append('-JUnitGCAfterTest') | |
1251 | |
1252 | |
1253 def harness(projectsCp, vmArgs): | |
1254 if _get_vm() != 'jvmci': | |
1255 prefixArgs = ['-esa', '-ea'] | |
1256 else: | |
1257 prefixArgs = ['-XX:-BootstrapJVMCI', '-esa', '-ea'] | |
1258 if gc_after_test: | |
1259 prefixArgs.append('-XX:-DisableExplicitGC') | |
1260 with open(testfile) as fp: | |
1261 testclasses = [l.rstrip() for l in fp.readlines()] | |
1262 | |
1263 # Remove entries from class path that are in graal.jar and | |
1264 # run the VM in a mode where application/test classes can | |
1265 # access core Graal classes. | |
1266 cp = prefixCp + coreCp + os.pathsep + projectsCp | |
1267 if isJVMCIEnabled(_get_vm()): | |
1268 excluded = set() | |
1269 for jdkDist in _jdkDeployedDists: | |
1270 dist = mx.distribution(jdkDist.name) | |
1271 excluded.update([d.output_dir() for d in dist.sorted_deps()]) | |
1272 cp = os.pathsep.join([e for e in cp.split(os.pathsep) if e not in excluded]) | |
1273 vmArgs = ['-XX:-UseJVMCIClassLoader'] + vmArgs | |
1274 | |
1275 # suppress menubar and dock when running on Mac | |
1276 vmArgs = ['-Djava.awt.headless=true'] + vmArgs | |
1277 | |
1278 if len(testclasses) == 1: | |
1279 # Execute Junit directly when one test is being run. This simplifies | |
1280 # replaying the VM execution in a native debugger (e.g., gdb). | |
1281 vm(prefixArgs + vmArgs + ['-cp', mx._separatedCygpathU2W(cp), 'com.oracle.graal.test.GraalJUnitCore'] + coreArgs + testclasses) | |
1282 else: | |
1283 vm(prefixArgs + vmArgs + ['-cp', mx._separatedCygpathU2W(cp), 'com.oracle.graal.test.GraalJUnitCore'] + coreArgs + ['@' + mx._cygpathU2W(testfile)]) | |
1284 | |
1285 try: | |
1286 _run_tests(args, harness, annotations, testfile, blacklist, whitelist, regex) | |
1287 finally: | |
1288 if os.environ.get('MX_TESTFILE') is None: | |
1289 os.remove(testfile) | |
1290 | |
1291 _unittestHelpSuffix = """ | |
1292 Unittest options: | |
1293 | |
1294 --blacklist <file> run all testcases not specified in the blacklist | |
1295 --whitelist <file> run only testcases which are included | |
1296 in the given whitelist | |
1297 --verbose enable verbose JUnit output | |
1298 --fail-fast stop after first JUnit test class that has a failure | |
1299 --enable-timing enable JUnit test timing | |
1300 --regex <regex> run only testcases matching a regular expression | |
1301 --color enable colors output | |
1302 --eager-stacktrace print stacktrace eagerly | |
1303 --gc-after-test force a GC after each test | |
1304 | |
1305 To avoid conflicts with VM options '--' can be used as delimiter. | |
1306 | |
1307 If filters are supplied, only tests whose fully qualified name | |
1308 includes a filter as a substring are run. | |
1309 | |
1310 For example, this command line: | |
1311 | |
1312 mx unittest -G:Dump= -G:MethodFilter=BC_aload.* -G:+PrintCFG BC_aload | |
1313 | |
1314 will run all JUnit test classes that contain 'BC_aload' in their | |
1315 fully qualified name and will pass these options to the VM: | |
1316 | |
1317 -G:Dump= -G:MethodFilter=BC_aload.* -G:+PrintCFG | |
1318 | |
1319 To get around command line length limitations on some OSes, the | |
1320 JUnit class names to be executed are written to a file that a | |
1321 custom JUnit wrapper reads and passes onto JUnit proper. The | |
1322 MX_TESTFILE environment variable can be set to specify a | |
1323 file which will not be deleted once the unittests are done | |
1324 (unlike the temporary file otherwise used). | |
1325 | |
1326 As with all other commands, using the global '-v' before 'unittest' | |
1327 command will cause mx to show the complete command line | |
1328 it uses to run the VM. | |
1329 """ | |
1330 | |
1331 def unittest(args): | |
1332 """run the JUnit tests (all testcases){0}""" | |
1333 | |
1334 parser = ArgumentParser(prog='mx unittest', | |
1335 description='run the JUnit tests', | |
1336 add_help=False, | |
1337 formatter_class=RawDescriptionHelpFormatter, | |
1338 epilog=_unittestHelpSuffix, | |
1339 ) | |
1340 parser.add_argument('--blacklist', help='run all testcases not specified in the blacklist', metavar='<path>') | |
1341 parser.add_argument('--whitelist', help='run testcases specified in whitelist only', metavar='<path>') | |
1342 parser.add_argument('--verbose', help='enable verbose JUnit output', action='store_true') | |
1343 parser.add_argument('--fail-fast', help='stop after first JUnit test class that has a failure', action='store_true') | |
1344 parser.add_argument('--enable-timing', help='enable JUnit test timing', action='store_true') | |
1345 parser.add_argument('--regex', help='run only testcases matching a regular expression', metavar='<regex>') | |
1346 parser.add_argument('--color', help='enable color output', action='store_true') | |
1347 parser.add_argument('--eager-stacktrace', help='print stacktrace eagerly', action='store_true') | |
1348 parser.add_argument('--gc-after-test', help='force a GC after each test', action='store_true') | |
1349 | |
1350 ut_args = [] | |
1351 delimiter = False | |
1352 # check for delimiter | |
1353 while len(args) > 0: | |
1354 arg = args.pop(0) | |
1355 if arg == '--': | |
1356 delimiter = True | |
1357 break | |
1358 ut_args.append(arg) | |
1359 | |
1360 if delimiter: | |
1361 # all arguments before '--' must be recognized | |
1362 parsed_args = parser.parse_args(ut_args) | |
1363 else: | |
1364 # parse all know arguments | |
1365 parsed_args, args = parser.parse_known_args(ut_args) | |
1366 | |
1367 if parsed_args.whitelist: | |
1368 try: | |
1369 with open(join(_graal_home, parsed_args.whitelist)) as fp: | |
1370 parsed_args.whitelist = [re.compile(fnmatch.translate(l.rstrip())) for l in fp.readlines() if not l.startswith('#')] | |
1371 except IOError: | |
1372 mx.log('warning: could not read whitelist: ' + parsed_args.whitelist) | |
1373 if parsed_args.blacklist: | |
1374 try: | |
1375 with open(join(_graal_home, parsed_args.blacklist)) as fp: | |
1376 parsed_args.blacklist = [re.compile(fnmatch.translate(l.rstrip())) for l in fp.readlines() if not l.startswith('#')] | |
1377 except IOError: | |
1378 mx.log('warning: could not read blacklist: ' + parsed_args.blacklist) | |
1379 | |
1380 _unittest(args, ['@Test', '@Parameters'], **parsed_args.__dict__) | |
1381 | |
1382 def shortunittest(args): | |
1383 """alias for 'unittest --whitelist test/whitelist_shortunittest.txt'{0}""" | |
1384 | |
1385 args = ['--whitelist', 'test/whitelist_shortunittest.txt'] + args | |
1386 unittest(args) | |
1387 | |
1388 def microbench(args): | |
1389 """run JMH microbenchmark projects""" | |
1390 vmArgs, jmhArgs = _extract_VM_args(args, useDoubleDash=True) | |
1391 | |
1392 # look for -f in JMH arguments | |
1393 containsF = False | |
1394 forking = True | |
1395 for i in range(len(jmhArgs)): | |
1396 arg = jmhArgs[i] | |
1397 if arg.startswith('-f'): | |
1398 containsF = True | |
1399 if arg == '-f' and (i+1) < len(jmhArgs): | |
1400 arg += jmhArgs[i+1] | |
1401 try: | |
1402 if int(arg[2:]) == 0: | |
1403 forking = False | |
1404 except ValueError: | |
1405 pass | |
1406 | |
1407 # default to -f1 if not specified otherwise | |
1408 if not containsF: | |
1409 jmhArgs += ['-f1'] | |
1410 | |
1411 # find all projects with a direct JMH dependency | |
1412 jmhProjects = [] | |
1413 for p in mx.projects(): | |
1414 if 'JMH' in p.deps: | |
1415 jmhProjects.append(p.name) | |
1416 cp = mx.classpath(jmhProjects) | |
1417 | |
1418 # execute JMH runner | |
1419 args = ['-cp', cp] | |
1420 if not forking: | |
1421 args += vmArgs | |
1422 args += ['org.openjdk.jmh.Main'] | |
1423 if forking: | |
1424 (_, _, jvm, _, _) = _parseVmArgs(vmArgs) | |
1425 args += ['--jvmArgsPrepend', ' '.join(['-' + jvm] + vmArgs)] | |
1426 vm(args + jmhArgs) | |
1427 | |
1428 def buildvms(args): | |
1429 """build one or more VMs in various configurations""" | |
1430 | |
1431 vmsDefault = ','.join(_vmChoices.keys()) | |
1432 vmbuildsDefault = ','.join(_vmbuildChoices) | |
1433 | |
1434 parser = ArgumentParser(prog='mx buildvms') | |
1435 parser.add_argument('--vms', help='a comma separated list of VMs to build (default: ' + vmsDefault + ')', metavar='<args>', default=vmsDefault) | |
1436 parser.add_argument('--builds', help='a comma separated list of build types (default: ' + vmbuildsDefault + ')', metavar='<args>', default=vmbuildsDefault) | |
1437 parser.add_argument('--check-distributions', action='store_true', dest='check_distributions', help='check built distributions for overlap') | |
1438 parser.add_argument('-n', '--no-check', action='store_true', help='omit running "java -version" after each build') | |
1439 parser.add_argument('-c', '--console', action='store_true', help='send build output to console instead of log file') | |
1440 | |
1441 args = parser.parse_args(args) | |
1442 vms = args.vms.split(',') | |
1443 builds = args.builds.split(',') | |
1444 | |
1445 allStart = time.time() | |
1446 check_dists_args = ['--check-distributions'] if args.check_distributions else [] | |
1447 for v in vms: | |
1448 if not isVMSupported(v): | |
1449 mx.log('The ' + v + ' VM is not supported on this platform - skipping') | |
1450 continue | |
1451 | |
1452 for vmbuild in builds: | |
1453 if v == 'original' and vmbuild != 'product': | |
1454 continue | |
1455 if not args.console: | |
1456 logFile = join(v + '-' + vmbuild + '.log') | |
1457 log = open(join(_graal_home, logFile), 'wb') | |
1458 start = time.time() | |
1459 mx.log('BEGIN: ' + v + '-' + vmbuild + '\t(see: ' + logFile + ')') | |
1460 verbose = ['-v'] if mx._opts.verbose else [] | |
1461 # Run as subprocess so that output can be directed to a file | |
1462 cmd = [sys.executable, '-u', join('mxtool', 'mx.py')] + verbose + ['--vm', v, '--vmbuild', vmbuild, 'build'] + check_dists_args | |
1463 mx.logv("executing command: " + str(cmd)) | |
1464 subprocess.check_call(cmd, cwd=_graal_home, stdout=log, stderr=subprocess.STDOUT) | |
1465 duration = datetime.timedelta(seconds=time.time() - start) | |
1466 mx.log('END: ' + v + '-' + vmbuild + '\t[' + str(duration) + ']') | |
1467 else: | |
1468 with VM(v, vmbuild): | |
1469 build(check_dists_args) | |
1470 if not args.no_check: | |
1471 vmargs = ['-version'] | |
1472 if v == 'jvmci': | |
1473 vmargs.insert(0, '-XX:-BootstrapJVMCI') | |
1474 vm(vmargs, vm=v, vmbuild=vmbuild) | |
1475 allDuration = datetime.timedelta(seconds=time.time() - allStart) | |
1476 mx.log('TOTAL TIME: ' + '[' + str(allDuration) + ']') | |
1477 | |
1478 class Task: | |
1479 # None or a list of strings. If not None, only tasks whose title | |
1480 # matches at least one of the substrings in this list will return | |
1481 # a non-None value from __enter__. The body of a 'with Task(...) as t' | |
1482 # statement should check 't' and exit immediately if it is None. | |
1483 filters = None | |
1484 filtersExclude = False | |
1485 | |
1486 def __init__(self, title, tasks=None): | |
1487 self.tasks = tasks | |
1488 self.title = title | |
1489 if tasks is not None and Task.filters is not None: | |
1490 if Task.filtersExclude: | |
1491 self.skipped = any([f in title for f in Task.filters]) | |
1492 else: | |
1493 self.skipped = not any([f in title for f in Task.filters]) | |
1494 else: | |
1495 self.skipped = False | |
1496 if not self.skipped: | |
1497 self.start = time.time() | |
1498 self.end = None | |
1499 self.duration = None | |
1500 mx.log(time.strftime('gate: %d %b %Y %H:%M:%S: BEGIN: ') + title) | |
1501 def __enter__(self): | |
1502 assert self.tasks is not None, "using Task with 'with' statement requires to pass the tasks list in the constructor" | |
1503 if self.skipped: | |
1504 return None | |
1505 return self | |
1506 def __exit__(self, exc_type, exc_value, traceback): | |
1507 if not self.skipped: | |
1508 self.tasks.append(self.stop()) | |
1509 def stop(self): | |
1510 self.end = time.time() | |
1511 self.duration = datetime.timedelta(seconds=self.end - self.start) | |
1512 mx.log(time.strftime('gate: %d %b %Y %H:%M:%S: END: ') + self.title + ' [' + str(self.duration) + ']') | |
1513 return self | |
1514 def abort(self, codeOrMessage): | |
1515 self.end = time.time() | |
1516 self.duration = datetime.timedelta(seconds=self.end - self.start) | |
1517 mx.log(time.strftime('gate: %d %b %Y %H:%M:%S: ABORT: ') + self.title + ' [' + str(self.duration) + ']') | |
1518 mx.abort(codeOrMessage) | |
1519 return self | |
1520 | |
1521 def ctw(args): | |
1522 """run CompileTheWorld""" | |
1523 | |
1524 defaultCtwopts = '-Inline' | |
1525 | |
1526 parser = ArgumentParser(prog='mx ctw') | |
1527 parser.add_argument('--ctwopts', action='store', help='space separated JVMCI options used for CTW compilations (default: --ctwopts="' + defaultCtwopts + '")', default=defaultCtwopts, metavar='<options>') | |
1528 parser.add_argument('--jar', action='store', help='jar of classes to compiled instead of rt.jar', metavar='<path>') | |
1529 | |
1530 args, vmargs = parser.parse_known_args(args) | |
1531 | |
1532 if args.ctwopts: | |
1533 vmargs.append('-G:CompileTheWorldConfig=' + args.ctwopts) | |
1534 | |
1535 if args.jar: | |
1536 jar = os.path.abspath(args.jar) | |
1537 else: | |
1538 jar = join(_jdk(installJars=False), 'jre', 'lib', 'rt.jar') | |
1539 vmargs.append('-G:CompileTheWorldExcludeMethodFilter=sun.awt.X11.*.*') | |
1540 | |
1541 vmargs += ['-XX:+CompileTheWorld'] | |
1542 vm_ = _get_vm() | |
1543 if isJVMCIEnabled(vm_): | |
1544 if vm_ == 'jvmci': | |
1545 vmargs += ['-XX:+BootstrapJVMCI'] | |
1546 vmargs += ['-G:CompileTheWorldClasspath=' + jar] | |
1547 else: | |
1548 vmargs += ['-Xbootclasspath/p:' + jar] | |
1549 | |
1550 # suppress menubar and dock when running on Mac; exclude x11 classes as they may cause vm crashes (on Solaris) | |
1551 vmargs = ['-Djava.awt.headless=true'] + vmargs | |
1552 | |
1553 vm(vmargs) | |
1554 | |
1555 def _basic_gate_body(args, tasks): | |
1556 # Build server-hosted-jvmci now so we can run the unit tests | |
1557 with Task('BuildHotSpotJVMCIHosted: product', tasks) as t: | |
1558 if t: buildvms(['--vms', 'server', '--builds', 'product', '--check-distributions']) | |
1559 | |
1560 # Run unit tests on server-hosted-jvmci | |
1561 with VM('server', 'product'): | |
1562 with Task('UnitTests:hosted-product', tasks) as t: | |
1563 if t: unittest(['--enable-timing', '--verbose', '--fail-fast']) | |
1564 | |
1565 # Run unit tests on server-hosted-jvmci with -G:+SSA_LIR | |
1566 with VM('server', 'product'): | |
1567 with Task('UnitTestsSSA:hosted-product', tasks) as t: | |
1568 if t: unittest(['--enable-timing', '--verbose', '--fail-fast', '-G:+SSA_LIR']) | |
1569 # Run ctw against rt.jar on server-hosted-jvmci | |
1570 with VM('server', 'product'): | |
1571 with Task('CTW:hosted-product', tasks) as t: | |
1572 if t: ctw(['--ctwopts', '-Inline +ExitVMOnException', '-esa', '-G:+CompileTheWorldMultiThreaded', '-G:-CompileTheWorldVerbose']) | |
1573 | |
1574 # Build the other VM flavors | |
1575 with Task('BuildHotSpotGraalOthers: fastdebug,product', tasks) as t: | |
1576 if t: buildvms(['--vms', 'jvmci,server', '--builds', 'fastdebug,product', '--check-distributions']) | |
1577 | |
1578 with VM('jvmci', 'fastdebug'): | |
1579 with Task('BootstrapWithSystemAssertions:fastdebug', tasks) as t: | |
1580 if t: vm(['-esa', '-XX:-TieredCompilation', '-version']) | |
1581 | |
1582 with VM('jvmci', 'fastdebug'): | |
1583 with Task('BootstrapEconomyWithSystemAssertions:fastdebug', tasks) as t: | |
1584 if t: vm(['-esa', '-XX:-TieredCompilation', '-G:CompilerConfiguration=economy', '-version']) | |
1585 | |
1586 with VM('jvmci', 'fastdebug'): | |
1587 with Task('BootstrapWithSystemAssertionsNoCoop:fastdebug', tasks) as t: | |
1588 if t: vm(['-esa', '-XX:-TieredCompilation', '-XX:-UseCompressedOops', '-version']) | |
1589 | |
1590 with VM('jvmci', 'fastdebug'): | |
1591 with Task('BootstrapWithExceptionEdges:fastdebug', tasks) as t: | |
1592 if t: vm(['-esa', '-XX:-TieredCompilation', '-G:+StressInvokeWithExceptionNode', '-version']) | |
1593 | |
1594 with VM('jvmci', 'product'): | |
1595 with Task('BootstrapWithGCVerification:product', tasks) as t: | |
1596 if t: | |
1597 out = mx.DuplicateSuppressingStream(['VerifyAfterGC:', 'VerifyBeforeGC:']).write | |
1598 vm(['-XX:-TieredCompilation', '-XX:+UnlockDiagnosticVMOptions', '-XX:+VerifyBeforeGC', '-XX:+VerifyAfterGC', '-version'], out=out) | |
1599 | |
1600 with VM('jvmci', 'product'): | |
1601 with Task('BootstrapWithG1GCVerification:product', tasks) as t: | |
1602 if t: | |
1603 out = mx.DuplicateSuppressingStream(['VerifyAfterGC:', 'VerifyBeforeGC:']).write | |
1604 vm(['-XX:-TieredCompilation', '-XX:+UnlockDiagnosticVMOptions', '-XX:-UseSerialGC', '-XX:+UseG1GC', '-XX:+VerifyBeforeGC', '-XX:+VerifyAfterGC', '-version'], out=out) | |
1605 | |
1606 with VM('jvmci', 'product'): | |
1607 with Task('BootstrapWithRegisterPressure:product', tasks) as t: | |
1608 if t: | |
1609 registers = 'o0,o1,o2,o3,f8,f9,d32,d34' if platform.processor() == 'sparc' else 'rbx,r11,r10,r14,xmm3,xmm11,xmm14' | |
1610 vm(['-XX:-TieredCompilation', '-G:RegisterPressure=' + registers, '-esa', '-version']) | |
1611 | |
1612 with VM('jvmci', 'product'): | |
1613 with Task('BootstrapSSAWithRegisterPressure:product', tasks) as t: | |
1614 if t: | |
1615 registers = 'o0,o1,o2,o3,f8,f9,d32,d34' if platform.processor() == 'sparc' else 'rbx,r11,r10,r14,xmm3,xmm11,xmm14' | |
1616 vm(['-XX:-TieredCompilation', '-G:+SSA_LIR', '-G:RegisterPressure=' + registers, '-esa', '-version']) | |
1617 | |
1618 with VM('jvmci', 'product'): | |
1619 with Task('BootstrapWithImmutableCode:product', tasks) as t: | |
1620 if t: vm(['-XX:-TieredCompilation', '-G:+ImmutableCode', '-G:+VerifyPhases', '-esa', '-version']) | |
1621 | |
1622 for vmbuild in ['fastdebug', 'product']: | |
1623 for test in sanitycheck.getDacapos(level=sanitycheck.SanityCheckLevel.Gate, gateBuildLevel=vmbuild) + sanitycheck.getScalaDacapos(level=sanitycheck.SanityCheckLevel.Gate, gateBuildLevel=vmbuild): | |
1624 with Task(str(test) + ':' + vmbuild, tasks) as t: | |
1625 if t and not test.test('jvmci'): | |
1626 t.abort(test.name + ' Failed') | |
1627 | |
1628 # ensure -Xbatch still works | |
1629 with VM('jvmci', 'product'): | |
1630 with Task('DaCapo_pmd:BatchMode:product', tasks) as t: | |
1631 if t: dacapo(['-Xbatch', 'pmd']) | |
1632 | |
1633 # ensure -Xcomp still works | |
1634 with VM('jvmci', 'product'): | |
1635 with Task('XCompMode:product', tasks) as t: | |
1636 if t: vm(['-Xcomp', '-version']) | |
1637 | |
1638 if args.jacocout is not None: | |
1639 jacocoreport([args.jacocout]) | |
1640 | |
1641 global _jacoco | |
1642 _jacoco = 'off' | |
1643 | |
1644 with Task('CleanAndBuildIdealGraphVisualizer', tasks) as t: | |
1645 if t and platform.processor() != 'sparc': | |
1646 buildxml = mx._cygpathU2W(join(_graal_home, 'src', 'share', 'tools', 'IdealGraphVisualizer', 'build.xml')) | |
1647 mx.run(['ant', '-f', buildxml, '-q', 'clean', 'build'], env=_igvBuildEnv()) | |
1648 | |
1649 # Prevent JVMCI modifications from breaking the standard builds | |
1650 if args.buildNonJVMCI: | |
1651 with Task('BuildHotSpotVarieties', tasks) as t: | |
1652 if t: | |
1653 buildvms(['--vms', 'client,server', '--builds', 'fastdebug,product']) | |
1654 if mx.get_os() not in ['windows', 'cygwin']: | |
1655 buildvms(['--vms', 'server-nojvmci', '--builds', 'product,optimized']) | |
1656 | |
1657 for vmbuild in ['product', 'fastdebug']: | |
1658 for theVm in ['client', 'server']: | |
1659 if not isVMSupported(theVm): | |
1660 mx.log('The ' + theVm + ' VM is not supported on this platform') | |
1661 continue | |
1662 with VM(theVm, vmbuild): | |
1663 with Task('DaCapo_pmd:' + theVm + ':' + vmbuild, tasks) as t: | |
1664 if t: dacapo(['pmd']) | |
1665 | |
1666 with Task('UnitTests:' + theVm + ':' + vmbuild, tasks) as t: | |
1667 if t: unittest(['-XX:CompileCommand=exclude,*::run*', 'graal.api', 'java.test']) | |
1668 | |
1669 | |
1670 def gate(args, gate_body=_basic_gate_body): | |
1671 """run the tests used to validate a push | |
1672 | |
1673 If this command exits with a 0 exit code, then the source code is in | |
1674 a state that would be accepted for integration into the main repository.""" | |
1675 | |
1676 parser = ArgumentParser(prog='mx gate') | |
1677 parser.add_argument('-j', '--omit-java-clean', action='store_false', dest='cleanJava', help='omit cleaning Java native code') | |
1678 parser.add_argument('-n', '--omit-native-clean', action='store_false', dest='cleanNative', help='omit cleaning and building native code') | |
1679 parser.add_argument('-i', '--omit-ide-clean', action='store_false', dest='cleanIde', help='omit cleaning the ide project files') | |
1680 parser.add_argument('-g', '--only-build-jvmci', action='store_false', dest='buildNonJVMCI', help='only build the JVMCI VM') | |
1681 parser.add_argument('-t', '--task-filter', help='comma separated list of substrings to select subset of tasks to be run') | |
1682 parser.add_argument('-x', action='store_true', help='makes --task-filter an exclusion instead of inclusion filter') | |
1683 parser.add_argument('--jacocout', help='specify the output directory for jacoco report') | |
1684 | |
1685 args = parser.parse_args(args) | |
1686 | |
1687 global _jacoco | |
1688 if args.task_filter: | |
1689 Task.filters = args.task_filter.split(',') | |
1690 Task.filtersExclude = args.x | |
1691 elif args.x: | |
1692 mx.abort('-x option cannot be used without --task-filter option') | |
1693 | |
1694 # Force | |
1695 if not mx._opts.strict_compliance: | |
1696 mx.log("[gate] forcing strict compliance") | |
1697 mx._opts.strict_compliance = True | |
1698 | |
1699 tasks = [] | |
1700 total = Task('Gate') | |
1701 try: | |
1702 with Task('Check jvmci.make in sync with suite.py', tasks) as t: | |
1703 if t: | |
1704 jvmciMake = join('make', 'jvmci.make') | |
1705 if mx_graal_makefile.build_makefile(['-o', jvmciMake]) != 0: | |
1706 t.abort('Rerun "mx makefile -o ' + jvmciMake + ' and check-in the modified ' + jvmciMake) | |
1707 | |
1708 with Task('Pylint', tasks) as t: | |
1709 if t: mx.pylint([]) | |
1710 | |
1711 def _clean(name='Clean'): | |
1712 with Task(name, tasks) as t: | |
1713 if t: | |
1714 cleanArgs = [] | |
1715 if not args.cleanNative: | |
1716 cleanArgs.append('--no-native') | |
1717 if not args.cleanJava: | |
1718 cleanArgs.append('--no-java') | |
1719 clean(cleanArgs) | |
1720 _clean() | |
1721 | |
1722 with Task('IDEConfigCheck', tasks) as t: | |
1723 if t: | |
1724 if args.cleanIde: | |
1725 mx.ideclean([]) | |
1726 mx.ideinit([]) | |
1727 | |
1728 eclipse_exe = mx.get_env('ECLIPSE_EXE') | |
1729 if eclipse_exe is not None: | |
1730 with Task('CodeFormatCheck', tasks) as t: | |
1731 if t and mx.eclipseformat(['-e', eclipse_exe]) != 0: | |
1732 t.abort('Formatter modified files - run "mx eclipseformat", check in changes and repush') | |
1733 | |
1734 with Task('Canonicalization Check', tasks) as t: | |
1735 if t: | |
1736 mx.log(time.strftime('%d %b %Y %H:%M:%S - Ensuring mx/projects files are canonicalized...')) | |
1737 if mx.canonicalizeprojects([]) != 0: | |
1738 t.abort('Rerun "mx canonicalizeprojects" and check-in the modified mx/suite*.py files.') | |
1739 | |
1740 if mx.get_env('JDT'): | |
1741 with Task('BuildJavaWithEcj', tasks): | |
1742 if t: build(['-p', '--no-native', '--jdt-warning-as-error']) | |
1743 _clean('CleanAfterEcjBuild') | |
1744 | |
1745 with Task('BuildJavaWithJavac', tasks): | |
1746 if t: build(['-p', '--no-native', '--force-javac']) | |
1747 | |
1748 with Task('Checkstyle', tasks) as t: | |
1749 if t and mx.checkstyle([]) != 0: | |
1750 t.abort('Checkstyle warnings were found') | |
1751 | |
1752 with Task('Checkheaders', tasks) as t: | |
1753 if t and checkheaders([]) != 0: | |
1754 t.abort('Checkheaders warnings were found') | |
1755 | |
1756 with Task('FindBugs', tasks) as t: | |
1757 if t and findbugs([]) != 0: | |
1758 t.abort('FindBugs warnings were found') | |
1759 | |
1760 if exists('jacoco.exec'): | |
1761 os.unlink('jacoco.exec') | |
1762 | |
1763 if args.jacocout is not None: | |
1764 _jacoco = 'append' | |
1765 else: | |
1766 _jacoco = 'off' | |
1767 | |
1768 gate_body(args, tasks) | |
1769 | |
1770 except KeyboardInterrupt: | |
1771 total.abort(1) | |
1772 | |
1773 except BaseException as e: | |
1774 import traceback | |
1775 traceback.print_exc() | |
1776 total.abort(str(e)) | |
1777 | |
1778 total.stop() | |
1779 | |
1780 mx.log('Gate task times:') | |
1781 for t in tasks: | |
1782 mx.log(' ' + str(t.duration) + '\t' + t.title) | |
1783 mx.log(' =======') | |
1784 mx.log(' ' + str(total.duration)) | |
1785 | |
1786 if args.task_filter: | |
1787 Task.filters = None | |
1788 | |
1789 def deoptalot(args): | |
1790 """bootstrap a fastdebug JVMCI VM with DeoptimizeALot and VerifyOops on | |
1791 | |
1792 If the first argument is a number, the process will be repeated | |
1793 this number of times. All other arguments are passed to the VM.""" | |
1794 count = 1 | |
1795 if len(args) > 0 and args[0].isdigit(): | |
1796 count = int(args[0]) | |
1797 del args[0] | |
1798 | |
1799 for _ in range(count): | |
1800 if not vm(['-XX:+DeoptimizeALot', '-XX:+VerifyOops'] + args + ['-version'], vmbuild='fastdebug') == 0: | |
1801 mx.abort("Failed") | |
1802 | |
1803 def longtests(args): | |
1804 | |
1805 deoptalot(['15', '-Xmx48m']) | |
1806 | |
1807 dacapo(['100', 'eclipse', '-esa']) | |
1808 | |
1809 def _igvJdk(): | |
1810 v8u20 = mx.VersionSpec("1.8.0_20") | |
1811 v8u40 = mx.VersionSpec("1.8.0_40") | |
1812 v8 = mx.VersionSpec("1.8") | |
1813 def _igvJdkVersionCheck(version): | |
1814 return version >= v8 and (version < v8u20 or version >= v8u40) | |
1815 return mx.java(_igvJdkVersionCheck, versionDescription='>= 1.8 and < 1.8.0u20 or >= 1.8.0u40', purpose="building & running IGV").jdk | |
1816 | |
1817 def _igvBuildEnv(): | |
1818 # When the http_proxy environment variable is set, convert it to the proxy settings that ant needs | |
1819 env = dict(os.environ) | |
1820 proxy = os.environ.get('http_proxy') | |
1821 if not (proxy is None) and len(proxy) > 0: | |
1822 if '://' in proxy: | |
1823 # Remove the http:// prefix (or any other protocol prefix) | |
1824 proxy = proxy.split('://', 1)[1] | |
1825 # Separate proxy server name and port number | |
1826 proxyName, proxyPort = proxy.split(':', 1) | |
1827 proxyEnv = '-DproxyHost="' + proxyName + '" -DproxyPort=' + proxyPort | |
1828 env['ANT_OPTS'] = proxyEnv | |
1829 | |
1830 env['JAVA_HOME'] = _igvJdk() | |
1831 return env | |
1832 | |
1833 def igv(args): | |
1834 """run the Ideal Graph Visualizer""" | |
1835 logFile = '.ideal_graph_visualizer.log' | |
1836 with open(join(_graal_home, logFile), 'w') as fp: | |
1837 mx.logv('[Ideal Graph Visualizer log is in ' + fp.name + ']') | |
1838 nbplatform = join(_graal_home, 'src', 'share', 'tools', 'IdealGraphVisualizer', 'nbplatform') | |
1839 | |
1840 # Remove NetBeans platform if it is earlier than the current supported version | |
1841 if exists(nbplatform): | |
1842 updateTrackingFile = join(nbplatform, 'platform', 'update_tracking', 'org-netbeans-core.xml') | |
1843 if not exists(updateTrackingFile): | |
1844 mx.log('Could not find \'' + updateTrackingFile + '\', removing NetBeans platform') | |
1845 shutil.rmtree(nbplatform) | |
1846 else: | |
1847 dom = xml.dom.minidom.parse(updateTrackingFile) | |
1848 currentVersion = mx.VersionSpec(dom.getElementsByTagName('module_version')[0].getAttribute('specification_version')) | |
1849 supportedVersion = mx.VersionSpec('3.43.1') | |
1850 if currentVersion < supportedVersion: | |
1851 mx.log('Replacing NetBeans platform version ' + str(currentVersion) + ' with version ' + str(supportedVersion)) | |
1852 shutil.rmtree(nbplatform) | |
1853 elif supportedVersion < currentVersion: | |
1854 mx.log('Supported NetBeans version in igv command should be updated to ' + str(currentVersion)) | |
1855 | |
1856 if not exists(nbplatform): | |
1857 mx.logv('[This execution may take a while as the NetBeans platform needs to be downloaded]') | |
1858 | |
1859 env = _igvBuildEnv() | |
1860 # make the jar for Batik 1.7 available. | |
1861 env['IGV_BATIK_JAR'] = mx.library('BATIK').get_path(True) | |
1862 if mx.run(['ant', '-f', mx._cygpathU2W(join(_graal_home, 'src', 'share', 'tools', 'IdealGraphVisualizer', 'build.xml')), '-l', mx._cygpathU2W(fp.name), 'run'], env=env, nonZeroIsFatal=False): | |
1863 mx.abort("IGV ant build & launch failed. Check '" + logFile + "'. You can also try to delete 'src/share/tools/IdealGraphVisualizer/nbplatform'.") | |
1864 | |
1865 def maven_install_truffle(args): | |
1866 """install Truffle into your local Maven repository""" | |
1867 for name in ['TRUFFLE', 'TRUFFLE-DSL-PROCESSOR']: | |
1868 mx.archive(["@" + name]) | |
1869 path = mx._dists[name].path | |
1870 mx.run(['mvn', 'install:install-file', '-DgroupId=com.oracle', '-DartifactId=' + name.lower(), '-Dversion=' + graal_version('SNAPSHOT'), '-Dpackaging=jar', '-Dfile=' + path]) | |
1871 | |
1872 def c1visualizer(args): | |
1873 """run the Cl Compiler Visualizer""" | |
1874 libpath = join(_graal_home, 'lib') | |
1875 if mx.get_os() == 'windows': | |
1876 executable = join(libpath, 'c1visualizer', 'bin', 'c1visualizer.exe') | |
1877 else: | |
1878 executable = join(libpath, 'c1visualizer', 'bin', 'c1visualizer') | |
1879 | |
1880 # Check whether the current C1Visualizer installation is the up-to-date | |
1881 if exists(executable) and not exists(mx.library('C1VISUALIZER_DIST').get_path(resolve=False)): | |
1882 mx.log('Updating C1Visualizer') | |
1883 shutil.rmtree(join(libpath, 'c1visualizer')) | |
1884 | |
1885 archive = mx.library('C1VISUALIZER_DIST').get_path(resolve=True) | |
1886 | |
1887 if not exists(executable): | |
1888 zf = zipfile.ZipFile(archive, 'r') | |
1889 zf.extractall(libpath) | |
1890 | |
1891 if not exists(executable): | |
1892 mx.abort('C1Visualizer binary does not exist: ' + executable) | |
1893 | |
1894 if mx.get_os() != 'windows': | |
1895 # Make sure that execution is allowed. The zip file does not always specfiy that correctly | |
1896 os.chmod(executable, 0777) | |
1897 | |
1898 mx.run([executable]) | |
1899 | |
1900 def bench(args): | |
1901 """run benchmarks and parse their output for results | |
1902 | |
1903 Results are JSON formated : {group : {benchmark : score}}.""" | |
1904 resultFile = None | |
1905 if '-resultfile' in args: | |
1906 index = args.index('-resultfile') | |
1907 if index + 1 < len(args): | |
1908 resultFile = args[index + 1] | |
1909 del args[index] | |
1910 del args[index] | |
1911 else: | |
1912 mx.abort('-resultfile must be followed by a file name') | |
1913 vm = _get_vm() | |
1914 if len(args) is 0: | |
1915 args = ['all'] | |
1916 | |
1917 vmArgs = [arg for arg in args if arg.startswith('-')] | |
1918 | |
1919 def benchmarks_in_group(group): | |
1920 prefix = group + ':' | |
1921 return [a[len(prefix):] for a in args if a.startswith(prefix)] | |
1922 | |
1923 results = {} | |
1924 benchmarks = [] | |
1925 # DaCapo | |
1926 if 'dacapo' in args or 'all' in args: | |
1927 benchmarks += sanitycheck.getDacapos(level=sanitycheck.SanityCheckLevel.Benchmark) | |
1928 else: | |
1929 dacapos = benchmarks_in_group('dacapo') | |
1930 for dacapo in dacapos: | |
1931 if dacapo not in sanitycheck.dacapoSanityWarmup.keys(): | |
1932 mx.abort('Unknown DaCapo : ' + dacapo) | |
1933 iterations = sanitycheck.dacapoSanityWarmup[dacapo][sanitycheck.SanityCheckLevel.Benchmark] | |
1934 if iterations > 0: | |
1935 benchmarks += [sanitycheck.getDacapo(dacapo, iterations)] | |
1936 | |
1937 if 'scaladacapo' in args or 'all' in args: | |
1938 benchmarks += sanitycheck.getScalaDacapos(level=sanitycheck.SanityCheckLevel.Benchmark) | |
1939 else: | |
1940 scaladacapos = benchmarks_in_group('scaladacapo') | |
1941 for scaladacapo in scaladacapos: | |
1942 if scaladacapo not in sanitycheck.dacapoScalaSanityWarmup.keys(): | |
1943 mx.abort('Unknown Scala DaCapo : ' + scaladacapo) | |
1944 iterations = sanitycheck.dacapoScalaSanityWarmup[scaladacapo][sanitycheck.SanityCheckLevel.Benchmark] | |
1945 if iterations > 0: | |
1946 benchmarks += [sanitycheck.getScalaDacapo(scaladacapo, ['-n', str(iterations)])] | |
1947 | |
1948 # Bootstrap | |
1949 if 'bootstrap' in args or 'all' in args: | |
1950 benchmarks += sanitycheck.getBootstraps() | |
1951 # SPECjvm2008 | |
1952 if 'specjvm2008' in args or 'all' in args: | |
1953 benchmarks += [sanitycheck.getSPECjvm2008(['-ikv', '-wt', '120', '-it', '120'])] | |
1954 else: | |
1955 specjvms = benchmarks_in_group('specjvm2008') | |
1956 for specjvm in specjvms: | |
1957 benchmarks += [sanitycheck.getSPECjvm2008(['-ikv', '-wt', '120', '-it', '120', specjvm])] | |
1958 | |
1959 if 'specjbb2005' in args or 'all' in args: | |
1960 benchmarks += [sanitycheck.getSPECjbb2005()] | |
1961 | |
1962 if 'specjbb2013' in args: # or 'all' in args //currently not in default set | |
1963 benchmarks += [sanitycheck.getSPECjbb2013()] | |
1964 | |
1965 if 'ctw-full' in args: | |
1966 benchmarks.append(sanitycheck.getCTW(vm, sanitycheck.CTWMode.Full)) | |
1967 if 'ctw-noinline' in args: | |
1968 benchmarks.append(sanitycheck.getCTW(vm, sanitycheck.CTWMode.NoInline)) | |
1969 | |
1970 for test in benchmarks: | |
1971 for (groupName, res) in test.bench(vm, extraVmOpts=vmArgs).items(): | |
1972 group = results.setdefault(groupName, {}) | |
1973 group.update(res) | |
1974 mx.log(json.dumps(results)) | |
1975 if resultFile: | |
1976 with open(resultFile, 'w') as f: | |
1977 f.write(json.dumps(results)) | |
1978 | |
1979 def _get_jmh_path(): | |
1980 path = mx.get_env('JMH_BENCHMARKS', None) | |
1981 if not path: | |
1982 probe = join(dirname(_graal_home), 'java-benchmarks') | |
1983 if exists(probe): | |
1984 path = probe | |
1985 | |
1986 if not path: | |
1987 mx.abort("Please set the JMH_BENCHMARKS environment variable to point to the java-benchmarks workspace") | |
1988 if not exists(path): | |
1989 mx.abort("The directory denoted by the JMH_BENCHMARKS environment variable does not exist: " + path) | |
1990 return path | |
1991 | |
1992 def makejmhdeps(args): | |
1993 """creates and installs Maven dependencies required by the JMH benchmarks | |
1994 | |
1995 The dependencies are specified by files named pom.mxdeps in the | |
1996 JMH directory tree. Each such file contains a list of dependencies | |
1997 defined in JSON format. For example: | |
1998 | |
1999 '[{"artifactId" : "compiler.test", "groupId" : "com.oracle.graal", "deps" : ["com.oracle.graal.compiler.test"]}]' | |
2000 | |
2001 will result in a dependency being installed in the local Maven repository | |
2002 that can be referenced in a pom.xml file as follows: | |
2003 | |
2004 <dependency> | |
2005 <groupId>com.oracle.graal</groupId> | |
2006 <artifactId>compiler.test</artifactId> | |
2007 <version>1.0-SNAPSHOT</version> | |
2008 </dependency>""" | |
2009 | |
2010 parser = ArgumentParser(prog='mx makejmhdeps') | |
2011 parser.add_argument('-s', '--settings', help='alternative path for Maven user settings file', metavar='<path>') | |
2012 parser.add_argument('-p', '--permissive', action='store_true', help='issue note instead of error if a Maven dependency cannot be built due to missing projects/libraries') | |
2013 args = parser.parse_args(args) | |
2014 | |
2015 def makejmhdep(artifactId, groupId, deps): | |
2016 graalSuite = mx.suite("graal") | |
2017 path = artifactId + '.jar' | |
2018 if args.permissive: | |
2019 allDeps = [] | |
2020 for name in deps: | |
2021 dist = mx.distribution(name, fatalIfMissing=False) | |
2022 if dist: | |
2023 allDeps = allDeps + [d.name for d in dist.sorted_deps(transitive=True)] | |
2024 else: | |
2025 if not mx.project(name, fatalIfMissing=False): | |
2026 if not mx.library(name, fatalIfMissing=False): | |
2027 mx.log('Skipping dependency ' + groupId + '.' + artifactId + ' as ' + name + ' cannot be resolved') | |
2028 return | |
2029 allDeps.append(name) | |
2030 d = mx.Distribution(graalSuite, name=artifactId, path=path, sourcesPath=path, deps=allDeps, mainClass=None, excludedDependencies=[], distDependencies=[], javaCompliance=None) | |
2031 d.make_archive() | |
2032 cmd = ['mvn', 'install:install-file', '-DgroupId=' + groupId, '-DartifactId=' + artifactId, | |
2033 '-Dversion=1.0-SNAPSHOT', '-Dpackaging=jar', '-Dfile=' + d.path] | |
2034 if not mx._opts.verbose: | |
2035 cmd.append('-q') | |
2036 if args.settings: | |
2037 cmd = cmd + ['-s', args.settings] | |
2038 mx.run(cmd) | |
2039 os.unlink(d.path) | |
2040 | |
2041 jmhPath = _get_jmh_path() | |
2042 for root, _, filenames in os.walk(jmhPath): | |
2043 for f in [join(root, n) for n in filenames if n == 'pom.mxdeps']: | |
2044 mx.logv('[processing ' + f + ']') | |
2045 try: | |
2046 with open(f) as fp: | |
2047 for d in json.load(fp): | |
2048 artifactId = d['artifactId'] | |
2049 groupId = d['groupId'] | |
2050 deps = d['deps'] | |
2051 makejmhdep(artifactId, groupId, deps) | |
2052 except ValueError as e: | |
2053 mx.abort('Error parsing {0}:\n{1}'.format(f, e)) | |
2054 | |
2055 def buildjmh(args): | |
2056 """build the JMH benchmarks""" | |
2057 | |
2058 parser = ArgumentParser(prog='mx buildjmh') | |
2059 parser.add_argument('-s', '--settings', help='alternative path for Maven user settings file', metavar='<path>') | |
2060 parser.add_argument('-c', action='store_true', dest='clean', help='clean before building') | |
2061 args = parser.parse_args(args) | |
2062 | |
2063 jmhPath = _get_jmh_path() | |
2064 mx.log('JMH benchmarks: ' + jmhPath) | |
2065 | |
2066 # Ensure the mx injected dependencies are up to date | |
2067 makejmhdeps(['-p'] + (['-s', args.settings] if args.settings else [])) | |
2068 | |
2069 timestamp = mx.TimeStampFile(join(_graal_home, 'mx', 'jmh', jmhPath.replace(os.sep, '_') + '.timestamp')) | |
2070 mustBuild = args.clean | |
2071 if not mustBuild: | |
2072 try: | |
2073 hgfiles = [join(jmhPath, f) for f in subprocess.check_output(['hg', '-R', jmhPath, 'locate']).split('\n')] | |
2074 mustBuild = timestamp.isOlderThan(hgfiles) | |
2075 except: | |
2076 # not a Mercurial repository or hg commands are not available. | |
2077 mustBuild = True | |
2078 | |
2079 if mustBuild: | |
2080 buildOutput = [] | |
2081 def _redirect(x): | |
2082 if mx._opts.verbose: | |
2083 mx.log(x[:-1]) | |
2084 else: | |
2085 buildOutput.append(x) | |
2086 env = os.environ.copy() | |
2087 env['JAVA_HOME'] = _jdk(vmToCheck='server') | |
2088 env['MAVEN_OPTS'] = '-server -XX:-UseJVMCIClassLoader' | |
2089 mx.log("Building benchmarks...") | |
2090 cmd = ['mvn'] | |
2091 if args.settings: | |
2092 cmd = cmd + ['-s', args.settings] | |
2093 if args.clean: | |
2094 cmd.append('clean') | |
2095 cmd.append('package') | |
2096 retcode = mx.run(cmd, cwd=jmhPath, out=_redirect, env=env, nonZeroIsFatal=False) | |
2097 if retcode != 0: | |
2098 mx.log(''.join(buildOutput)) | |
2099 mx.abort(retcode) | |
2100 timestamp.touch() | |
2101 else: | |
2102 mx.logv('[all Mercurial controlled files in ' + jmhPath + ' are older than ' + timestamp.path + ' - skipping build]') | |
2103 | |
2104 def jmh(args): | |
2105 """run the JMH benchmarks | |
2106 | |
2107 This command respects the standard --vm and --vmbuild options | |
2108 for choosing which VM to run the benchmarks with.""" | |
2109 if '-h' in args: | |
2110 mx.help_(['jmh']) | |
2111 mx.abort(1) | |
2112 | |
2113 vmArgs, benchmarksAndJsons = _extract_VM_args(args) | |
2114 if isJVMCIEnabled(_get_vm()) and '-XX:-UseJVMCIClassLoader' not in vmArgs: | |
2115 vmArgs = ['-XX:-UseJVMCIClassLoader'] + vmArgs | |
2116 | |
2117 benchmarks = [b for b in benchmarksAndJsons if not b.startswith('{')] | |
2118 jmhArgJsons = [b for b in benchmarksAndJsons if b.startswith('{')] | |
2119 jmhOutDir = join(_graal_home, 'mx', 'jmh') | |
2120 if not exists(jmhOutDir): | |
2121 os.makedirs(jmhOutDir) | |
2122 jmhOut = join(jmhOutDir, 'jmh.out') | |
2123 jmhArgs = {'-rff' : jmhOut, '-v' : 'EXTRA' if mx._opts.verbose else 'NORMAL'} | |
2124 | |
2125 # e.g. '{"-wi" : 20}' | |
2126 for j in jmhArgJsons: | |
2127 try: | |
2128 for n, v in json.loads(j).iteritems(): | |
2129 if v is None: | |
2130 del jmhArgs[n] | |
2131 else: | |
2132 jmhArgs[n] = v | |
2133 except ValueError as e: | |
2134 mx.abort('error parsing JSON input: {0}\n{1}'.format(j, e)) | |
2135 | |
2136 jmhPath = _get_jmh_path() | |
2137 mx.log('Using benchmarks in ' + jmhPath) | |
2138 | |
2139 matchedSuites = set() | |
2140 numBench = [0] | |
2141 for micros in os.listdir(jmhPath): | |
2142 absoluteMicro = os.path.join(jmhPath, micros) | |
2143 if not os.path.isdir(absoluteMicro): | |
2144 continue | |
2145 if not micros.startswith("micros-"): | |
2146 mx.logv('JMH: ignored ' + absoluteMicro + " because it doesn't start with 'micros-'") | |
2147 continue | |
2148 | |
2149 microJar = os.path.join(absoluteMicro, "target", "microbenchmarks.jar") | |
2150 if not exists(microJar): | |
2151 mx.log('Missing ' + microJar + ' - please run "mx buildjmh"') | |
2152 continue | |
2153 if benchmarks: | |
2154 def _addBenchmark(x): | |
2155 if x.startswith("Benchmark:"): | |
2156 return | |
2157 match = False | |
2158 for b in benchmarks: | |
2159 match = match or (b in x) | |
2160 | |
2161 if match: | |
2162 numBench[0] += 1 | |
2163 matchedSuites.add(micros) | |
2164 | |
2165 mx.run_java(['-jar', microJar, "-l"], cwd=jmhPath, out=_addBenchmark, addDefaultArgs=False) | |
2166 else: | |
2167 matchedSuites.add(micros) | |
2168 | |
2169 mx.logv("matchedSuites: " + str(matchedSuites)) | |
2170 plural = 's' if not benchmarks or numBench[0] > 1 else '' | |
2171 number = str(numBench[0]) if benchmarks else "all" | |
2172 mx.log("Running " + number + " benchmark" + plural + '...') | |
2173 | |
2174 regex = [] | |
2175 if benchmarks: | |
2176 regex.append(r".*(" + "|".join(benchmarks) + ").*") | |
2177 | |
2178 for suite in matchedSuites: | |
2179 absoluteMicro = os.path.join(jmhPath, suite) | |
2180 (pfx, exe, vm, forkedVmArgs, _) = _parseVmArgs(vmArgs) | |
2181 if pfx: | |
2182 mx.log("JMH ignores prefix: \"" + ' '.join(pfx) + "\"") | |
2183 javaArgs = ['-jar', os.path.join(absoluteMicro, "target", "microbenchmarks.jar"), | |
2184 '--jvm', exe, | |
2185 '--jvmArgs', ' '.join(["-" + vm] + forkedVmArgs)] | |
2186 for k, v in jmhArgs.iteritems(): | |
2187 javaArgs.append(k) | |
2188 if len(str(v)): | |
2189 javaArgs.append(str(v)) | |
2190 mx.run_java(javaArgs + regex, addDefaultArgs=False, cwd=jmhPath) | |
2191 | |
2192 def specjvm2008(args): | |
2193 """run one or more SPECjvm2008 benchmarks""" | |
2194 | |
2195 def launcher(bm, harnessArgs, extraVmOpts): | |
2196 return sanitycheck.getSPECjvm2008(harnessArgs + [bm]).bench(_get_vm(), extraVmOpts=extraVmOpts) | |
2197 | |
2198 availableBenchmarks = set(sanitycheck.specjvm2008Names) | |
2199 for name in sanitycheck.specjvm2008Names: | |
2200 parts = name.rsplit('.', 1) | |
2201 if len(parts) > 1: | |
2202 assert len(parts) == 2 | |
2203 group = parts[0] | |
2204 availableBenchmarks.add(group) | |
2205 | |
2206 _run_benchmark(args, sorted(availableBenchmarks), launcher) | |
2207 | |
2208 def specjbb2013(args): | |
2209 """run the composite SPECjbb2013 benchmark""" | |
2210 | |
2211 def launcher(bm, harnessArgs, extraVmOpts): | |
2212 assert bm is None | |
2213 return sanitycheck.getSPECjbb2013(harnessArgs).bench(_get_vm(), extraVmOpts=extraVmOpts) | |
2214 | |
2215 _run_benchmark(args, None, launcher) | |
2216 | |
2217 def specjbb2005(args): | |
2218 """run the composite SPECjbb2005 benchmark""" | |
2219 | |
2220 def launcher(bm, harnessArgs, extraVmOpts): | |
2221 assert bm is None | |
2222 return sanitycheck.getSPECjbb2005(harnessArgs).bench(_get_vm(), extraVmOpts=extraVmOpts) | |
2223 | |
2224 _run_benchmark(args, None, launcher) | |
2225 | |
2226 def hsdis(args, copyToDir=None): | |
2227 """download the hsdis library | |
2228 | |
2229 This is needed to support HotSpot's assembly dumping features. | |
2230 By default it downloads the Intel syntax version, use the 'att' argument to install AT&T syntax.""" | |
2231 flavor = 'intel' | |
2232 if 'att' in args: | |
2233 flavor = 'att' | |
2234 if mx.get_arch() == "sparcv9": | |
2235 flavor = "sparcv9" | |
2236 lib = mx.add_lib_suffix('hsdis-' + mx.get_arch()) | |
2237 path = join(_graal_home, 'lib', lib) | |
2238 | |
2239 sha1s = { | |
2240 'att/hsdis-amd64.dll' : 'bcbd535a9568b5075ab41e96205e26a2bac64f72', | |
2241 'att/hsdis-amd64.so' : '58919ba085d4ef7a513f25bae75e7e54ee73c049', | |
2242 'intel/hsdis-amd64.dll' : '6a388372cdd5fe905c1a26ced614334e405d1f30', | |
2243 'intel/hsdis-amd64.so' : '844ed9ffed64fe9599638f29a8450c50140e3192', | |
2244 'intel/hsdis-amd64.dylib' : 'fdb13ef0d7d23d93dacaae9c98837bea0d4fc5a2', | |
2245 'sparcv9/hsdis-sparcv9.so': '970640a9af0bd63641f9063c11275b371a59ee60', | |
2246 } | |
2247 | |
2248 flavoredLib = flavor + "/" + lib | |
2249 if flavoredLib not in sha1s: | |
2250 mx.logv("hsdis not supported on this plattform or architecture") | |
2251 return | |
2252 | |
2253 if not exists(path): | |
2254 sha1 = sha1s[flavoredLib] | |
2255 sha1path = path + '.sha1' | |
2256 mx.download_file_with_sha1('hsdis', path, ['http://lafo.ssw.uni-linz.ac.at/hsdis/' + flavoredLib], sha1, sha1path, True, True, sources=False) | |
2257 if copyToDir is not None and exists(copyToDir): | |
2258 shutil.copy(path, copyToDir) | |
2259 | |
2260 def hcfdis(args): | |
2261 """disassemble HexCodeFiles embedded in text files | |
2262 | |
2263 Run a tool over the input files to convert all embedded HexCodeFiles | |
2264 to a disassembled format.""" | |
2265 | |
2266 parser = ArgumentParser(prog='mx hcfdis') | |
2267 parser.add_argument('-m', '--map', help='address to symbol map applied to disassembler output') | |
2268 parser.add_argument('files', nargs=REMAINDER, metavar='files...') | |
2269 | |
2270 args = parser.parse_args(args) | |
2271 | |
2272 path = mx.library('HCFDIS').get_path(resolve=True) | |
2273 mx.run_java(['-cp', path, 'com.oracle.max.hcfdis.HexCodeFileDis'] + args.files) | |
2274 | |
2275 if args.map is not None: | |
2276 addressRE = re.compile(r'0[xX]([A-Fa-f0-9]+)') | |
2277 with open(args.map) as fp: | |
2278 lines = fp.read().splitlines() | |
2279 symbols = dict() | |
2280 for l in lines: | |
2281 addressAndSymbol = l.split(' ', 1) | |
2282 if len(addressAndSymbol) == 2: | |
2283 address, symbol = addressAndSymbol | |
2284 if address.startswith('0x'): | |
2285 address = long(address, 16) | |
2286 symbols[address] = symbol | |
2287 for f in args.files: | |
2288 with open(f) as fp: | |
2289 lines = fp.read().splitlines() | |
2290 updated = False | |
2291 for i in range(0, len(lines)): | |
2292 l = lines[i] | |
2293 for m in addressRE.finditer(l): | |
2294 sval = m.group(0) | |
2295 val = long(sval, 16) | |
2296 sym = symbols.get(val) | |
2297 if sym: | |
2298 l = l.replace(sval, sym) | |
2299 updated = True | |
2300 lines[i] = l | |
2301 if updated: | |
2302 mx.log('updating ' + f) | |
2303 with open('new_' + f, "w") as fp: | |
2304 for l in lines: | |
2305 print >> fp, l | |
2306 | |
2307 def jacocoreport(args): | |
2308 """create a JaCoCo coverage report | |
2309 | |
2310 Creates the report from the 'jacoco.exec' file in the current directory. | |
2311 Default output directory is 'coverage', but an alternative can be provided as an argument.""" | |
2312 jacocoreport = mx.library("JACOCOREPORT", True) | |
2313 out = 'coverage' | |
2314 if len(args) == 1: | |
2315 out = args[0] | |
2316 elif len(args) > 1: | |
2317 mx.abort('jacocoreport takes only one argument : an output directory') | |
2318 | |
2319 includes = ['com.oracle.graal', 'com.oracle.jvmci'] | |
2320 for p in mx.projects(): | |
2321 projsetting = getattr(p, 'jacoco', '') | |
2322 if projsetting == 'include': | |
2323 includes.append(p.name) | |
2324 | |
2325 includedirs = set() | |
2326 for p in mx.projects(): | |
2327 projsetting = getattr(p, 'jacoco', '') | |
2328 if projsetting == 'exclude': | |
2329 continue | |
2330 for include in includes: | |
2331 if include in p.dir: | |
2332 includedirs.add(p.dir) | |
2333 | |
2334 for i in includedirs: | |
2335 bindir = i + '/bin' | |
2336 if not os.path.exists(bindir): | |
2337 os.makedirs(bindir) | |
2338 | |
2339 mx.run_java(['-jar', jacocoreport.get_path(True), '--in', 'jacoco.exec', '--out', out] + sorted(includedirs)) | |
2340 | |
2341 def sl(args): | |
2342 """run an SL program""" | |
2343 vmArgs, slArgs = _extract_VM_args(args) | |
2344 vm(vmArgs + ['-cp', mx.classpath(["TRUFFLE", "com.oracle.truffle.sl"]), "com.oracle.truffle.sl.SLLanguage"] + slArgs) | |
2345 | |
2346 def sldebug(args): | |
2347 """run a simple command line debugger for the Simple Language""" | |
2348 vmArgs, slArgs = _extract_VM_args(args, useDoubleDash=True) | |
2349 vm(vmArgs + ['-cp', mx.classpath("com.oracle.truffle.sl.tools"), "com.oracle.truffle.sl.tools.debug.SLREPLServer"] + slArgs) | |
2350 | |
2351 def isJVMCIEnabled(vm): | |
2352 return vm != 'original' and not vm.endswith('nojvmci') | |
2353 | |
2354 def jol(args): | |
2355 """Java Object Layout""" | |
2356 joljar = mx.library('JOL_INTERNALS').get_path(resolve=True) | |
2357 candidates = mx.findclass(args, logToConsole=False, matcher=lambda s, classname: s == classname or classname.endswith('.' + s) or classname.endswith('$' + s)) | |
2358 | |
2359 if len(candidates) > 0: | |
2360 candidates = mx.select_items(sorted(candidates)) | |
2361 else: | |
2362 # mx.findclass can be mistaken, don't give up yet | |
2363 candidates = args | |
2364 | |
2365 vm(['-javaagent:' + joljar, '-cp', os.pathsep.join([mx.classpath(), joljar]), "org.openjdk.jol.MainObjectInternals"] + candidates) | |
2366 | |
2367 def site(args): | |
2368 """create a website containing javadoc and the project dependency graph""" | |
2369 | |
2370 return mx.site(['--name', 'Graal', | |
2371 '--jd', '@-tag', '--jd', '@test:X', | |
2372 '--jd', '@-tag', '--jd', '@run:X', | |
2373 '--jd', '@-tag', '--jd', '@bug:X', | |
2374 '--jd', '@-tag', '--jd', '@summary:X', | |
2375 '--jd', '@-tag', '--jd', '@vmoption:X', | |
2376 '--overview', join(_graal_home, 'graal', 'overview.html'), | |
2377 '--title', 'Graal OpenJDK Project Documentation', | |
2378 '--dot-output-base', 'projects'] + args) | |
2379 | |
2380 def generateZshCompletion(args): | |
2381 """generate zsh completion for mx""" | |
2382 try: | |
2383 from genzshcomp import CompletionGenerator | |
2384 except ImportError: | |
2385 mx.abort("install genzshcomp (pip install genzshcomp)") | |
2386 | |
2387 # need to fake module for the custom mx arg parser, otherwise a check in genzshcomp fails | |
2388 originalModule = mx._argParser.__module__ | |
2389 mx._argParser.__module__ = "argparse" | |
2390 generator = CompletionGenerator("mx", mx._argParser) | |
2391 mx._argParser.__module__ = originalModule | |
2392 | |
2393 # strip last line and define local variable "ret" | |
2394 complt = "\n".join(generator.get().split('\n')[0:-1]).replace('context state line', 'context state line ret=1') | |
2395 | |
2396 # add array of possible subcommands (as they are not part of the argument parser) | |
2397 complt += '\n ": :->command" \\\n' | |
2398 complt += ' "*::args:->args" && ret=0\n' | |
2399 complt += '\n' | |
2400 complt += 'case $state in\n' | |
2401 complt += '\t(command)\n' | |
2402 complt += '\t\tlocal -a main_commands\n' | |
2403 complt += '\t\tmain_commands=(\n' | |
2404 for cmd in sorted(mx._commands.iterkeys()): | |
2405 c, _ = mx._commands[cmd][:2] | |
2406 doc = c.__doc__ | |
2407 complt += '\t\t\t"{0}'.format(cmd) | |
2408 if doc: | |
2409 complt += ':{0}'.format(_fixQuotes(doc.split('\n', 1)[0])) | |
2410 complt += '"\n' | |
2411 complt += '\t\t)\n' | |
2412 complt += '\t\t_describe -t main_commands command main_commands && ret=0\n' | |
2413 complt += '\t\t;;\n' | |
2414 | |
2415 complt += '\t(args)\n' | |
2416 # TODO: improve matcher: if mx args are given, this doesn't work | |
2417 complt += '\t\tcase $line[1] in\n' | |
2418 complt += '\t\t\t(vm | vmg | vmfg | unittest | jmh | dacapo | scaladacapo | specjvm2008 | specjbb2013 | specjbb2005)\n' | |
2419 complt += '\t\t\t\tnoglob \\\n' | |
2420 complt += '\t\t\t\t\t_arguments -s -S \\\n' | |
2421 complt += _appendOptions("jvmci", r"G\:") | |
2422 # TODO: fix -XX:{-,+}Use* flags | |
2423 complt += _appendOptions("hotspot", r"XX\:") | |
2424 complt += '\t\t\t\t\t"-version" && ret=0 \n' | |
2425 complt += '\t\t\t\t;;\n' | |
2426 complt += '\t\tesac\n' | |
2427 complt += '\t\t;;\n' | |
2428 complt += 'esac\n' | |
2429 complt += '\n' | |
2430 complt += 'return $ret' | |
2431 print complt | |
2432 | |
2433 def _fixQuotes(arg): | |
2434 return arg.replace('\"', '').replace('\'', '').replace('`', '').replace('{', '\\{').replace('}', '\\}').replace('[', '\\[').replace(']', '\\]') | |
2435 | |
2436 def _appendOptions(optionType, optionPrefix): | |
2437 def isBoolean(vmap, field): | |
2438 return vmap[field] == "Boolean" or vmap[field] == "bool" | |
2439 | |
2440 def hasDescription(vmap): | |
2441 return vmap['optDefault'] or vmap['optDoc'] | |
2442 | |
2443 complt = "" | |
2444 for vmap in _parseVMOptions(optionType): | |
2445 complt += '\t\t\t\t\t-"' | |
2446 complt += optionPrefix | |
2447 if isBoolean(vmap, 'optType'): | |
2448 complt += '"{-,+}"' | |
2449 complt += vmap['optName'] | |
2450 if not isBoolean(vmap, 'optType'): | |
2451 complt += '=' | |
2452 if hasDescription(vmap): | |
2453 complt += "[" | |
2454 if vmap['optDefault']: | |
2455 complt += r"(default\: " + vmap['optDefault'] + ")" | |
2456 if vmap['optDoc']: | |
2457 complt += _fixQuotes(vmap['optDoc']) | |
2458 if hasDescription(vmap): | |
2459 complt += "]" | |
2460 complt += '" \\\n' | |
2461 return complt | |
2462 | |
2463 def _parseVMOptions(optionType): | |
2464 parser = OutputParser() | |
2465 # TODO: the optDoc part can wrapped accross multiple lines, currently only the first line will be captured | |
2466 # TODO: fix matching for float literals | |
2467 jvmOptions = re.compile( | |
2468 r"^[ \t]*" | |
2469 r"(?P<optType>(Boolean|Integer|Float|Double|String|bool|intx|uintx|ccstr|double)) " | |
2470 r"(?P<optName>[a-zA-Z0-9]+)" | |
2471 r"[ \t]+=[ \t]*" | |
2472 r"(?P<optDefault>([\-0-9]+(\.[0-9]+(\.[0-9]+\.[0-9]+))?|false|true|null|Name|sun\.boot\.class\.path))?" | |
2473 r"[ \t]*" | |
2474 r"(?P<optDoc>.+)?", | |
2475 re.MULTILINE) | |
2476 parser.addMatcher(ValuesMatcher(jvmOptions, { | |
2477 'optType' : '<optType>', | |
2478 'optName' : '<optName>', | |
2479 'optDefault' : '<optDefault>', | |
2480 'optDoc' : '<optDoc>', | |
2481 })) | |
2482 | |
2483 # gather JVMCI options | |
2484 output = StringIO.StringIO() | |
2485 vm(['-XX:-BootstrapJVMCI', '-XX:+UnlockDiagnosticVMOptions', '-G:+PrintFlags' if optionType == "jvmci" else '-XX:+PrintFlagsWithComments'], | |
2486 vm="jvmci", | |
2487 vmbuild="optimized", | |
2488 nonZeroIsFatal=False, | |
2489 out=output.write, | |
2490 err=subprocess.STDOUT) | |
2491 | |
2492 valueMap = parser.parse(output.getvalue()) | |
2493 return valueMap | |
2494 | |
2495 def findbugs(args): | |
2496 '''run FindBugs against non-test Java projects''' | |
2497 findBugsHome = mx.get_env('FINDBUGS_HOME', None) | |
2498 if findBugsHome: | |
2499 findbugsJar = join(findBugsHome, 'lib', 'findbugs.jar') | |
2500 else: | |
2501 findbugsLib = join(_graal_home, 'lib', 'findbugs-3.0.0') | |
2502 if not exists(findbugsLib): | |
2503 tmp = tempfile.mkdtemp(prefix='findbugs-download-tmp', dir=_graal_home) | |
2504 try: | |
2505 findbugsDist = mx.library('FINDBUGS_DIST').get_path(resolve=True) | |
2506 with zipfile.ZipFile(findbugsDist) as zf: | |
2507 candidates = [e for e in zf.namelist() if e.endswith('/lib/findbugs.jar')] | |
2508 assert len(candidates) == 1, candidates | |
2509 libDirInZip = os.path.dirname(candidates[0]) | |
2510 zf.extractall(tmp) | |
2511 shutil.copytree(join(tmp, libDirInZip), findbugsLib) | |
2512 finally: | |
2513 shutil.rmtree(tmp) | |
2514 findbugsJar = join(findbugsLib, 'findbugs.jar') | |
2515 assert exists(findbugsJar) | |
2516 nonTestProjects = [p for p in mx.projects() if not p.name.endswith('.test') and not p.name.endswith('.jtt')] | |
2517 outputDirs = map(mx._cygpathU2W, [p.output_dir() for p in nonTestProjects]) | |
2518 javaCompliance = max([p.javaCompliance for p in nonTestProjects]) | |
2519 findbugsResults = join(_graal_home, 'findbugs.results') | |
2520 | |
2521 cmd = ['-jar', mx._cygpathU2W(findbugsJar), '-textui', '-low', '-maxRank', '15'] | |
2522 if mx.is_interactive(): | |
2523 cmd.append('-progress') | |
2524 cmd = cmd + ['-auxclasspath', mx._separatedCygpathU2W(mx.classpath([d.name for d in _jdkDeployedDists] + [p.name for p in nonTestProjects])), '-output', mx._cygpathU2W(findbugsResults), '-exitcode'] + args + outputDirs | |
2525 exitcode = mx.run_java(cmd, nonZeroIsFatal=False, javaConfig=mx.java(javaCompliance)) | |
2526 if exitcode != 0: | |
2527 with open(findbugsResults) as fp: | |
2528 mx.log(fp.read()) | |
2529 os.unlink(findbugsResults) | |
2530 return exitcode | |
2531 | |
2532 def checkheaders(args): | |
2533 """check Java source headers against any required pattern""" | |
2534 failures = {} | |
2535 for p in mx.projects(): | |
2536 if p.native: | |
2537 continue | |
2538 | |
2539 csConfig = join(mx.project(p.checkstyleProj).dir, '.checkstyle_checks.xml') | |
2540 dom = xml.dom.minidom.parse(csConfig) | |
2541 for module in dom.getElementsByTagName('module'): | |
2542 if module.getAttribute('name') == 'RegexpHeader': | |
2543 for prop in module.getElementsByTagName('property'): | |
2544 if prop.getAttribute('name') == 'header': | |
2545 value = prop.getAttribute('value') | |
2546 matcher = re.compile(value, re.MULTILINE) | |
2547 for sourceDir in p.source_dirs(): | |
2548 for root, _, files in os.walk(sourceDir): | |
2549 for name in files: | |
2550 if name.endswith('.java') and name != 'package-info.java': | |
2551 f = join(root, name) | |
2552 with open(f) as fp: | |
2553 content = fp.read() | |
2554 if not matcher.match(content): | |
2555 failures[f] = csConfig | |
2556 for n, v in failures.iteritems(): | |
2557 mx.log('{0}: header does not match RegexpHeader defined in {1}'.format(n, v)) | |
2558 return len(failures) | |
2559 | |
2560 def mx_init(suite): | |
2561 commands = { | |
2562 'build': [build, ''], | |
2563 'buildjmh': [buildjmh, '[-options]'], | |
2564 'buildvars': [buildvars, ''], | |
2565 'buildvms': [buildvms, '[-options]'], | |
2566 'c1visualizer' : [c1visualizer, ''], | |
2567 'checkheaders': [checkheaders, ''], | |
2568 'clean': [clean, ''], | |
2569 'ctw': [ctw, '[-vmoptions|noinline|nocomplex|full]'], | |
2570 'findbugs': [findbugs, ''], | |
2571 'generateZshCompletion' : [generateZshCompletion, ''], | |
2572 'hsdis': [hsdis, '[att]'], | |
2573 'hcfdis': [hcfdis, ''], | |
2574 'igv' : [igv, ''], | |
2575 'maven-install-truffle' : [maven_install_truffle, ''], | |
2576 'jdkhome': [print_jdkhome, ''], | |
2577 'jmh': [jmh, '[VM options] [filters|JMH-args-as-json...]'], | |
2578 'dacapo': [dacapo, '[VM options] benchmarks...|"all" [DaCapo options]'], | |
2579 'scaladacapo': [scaladacapo, '[VM options] benchmarks...|"all" [Scala DaCapo options]'], | |
2580 'specjvm2008': [specjvm2008, '[VM options] benchmarks...|"all" [SPECjvm2008 options]'], | |
2581 'specjbb2013': [specjbb2013, '[VM options] [-- [SPECjbb2013 options]]'], | |
2582 'specjbb2005': [specjbb2005, '[VM options] [-- [SPECjbb2005 options]]'], | |
2583 'gate' : [gate, '[-options]'], | |
2584 'bench' : [bench, '[-resultfile file] [all(default)|dacapo|specjvm2008|bootstrap]'], | |
2585 'microbench' : [microbench, '[VM options] [-- [JMH options]]'], | |
2586 'unittest' : [unittest, '[unittest options] [--] [VM options] [filters...]', _unittestHelpSuffix], | |
2587 'makejmhdeps' : [makejmhdeps, ''], | |
2588 'shortunittest' : [shortunittest, '[unittest options] [--] [VM options] [filters...]', _unittestHelpSuffix], | |
2589 'jacocoreport' : [jacocoreport, '[output directory]'], | |
2590 'site' : [site, '[-options]'], | |
2591 'vm': [vm, '[-options] class [args...]'], | |
2592 'vmg': [vmg, '[-options] class [args...]'], | |
2593 'vmfg': [vmfg, '[-options] class [args...]'], | |
2594 'deoptalot' : [deoptalot, '[n]'], | |
2595 'longtests' : [longtests, ''], | |
2596 'sl' : [sl, '[SL args|@VM options]'], | |
2597 'sldebug' : [sldebug, '[SL args|@VM options]'], | |
2598 'jol' : [jol, ''], | |
2599 'makefile' : [mx_graal_makefile.build_makefile, 'build makefiles for JDK build'], | |
2600 } | |
2601 | |
2602 mx.add_argument('--jacoco', help='instruments com.oracle.* classes using JaCoCo', default='off', choices=['off', 'on', 'append']) | |
2603 mx.add_argument('--vmcwd', dest='vm_cwd', help='current directory will be changed to <path> before the VM is executed', default=None, metavar='<path>') | |
2604 mx.add_argument('--installed-jdks', help='the base directory in which the JDKs cloned from $JAVA_HOME exist. ' + | |
2605 'The VM selected by --vm and --vmbuild options is under this directory (i.e., ' + | |
2606 join('<path>', '<jdk-version>', '<vmbuild>', 'jre', 'lib', '<vm>', mx.add_lib_prefix(mx.add_lib_suffix('jvm'))) + ')', default=None, metavar='<path>') | |
2607 | |
2608 if _vmSourcesAvailable: | |
2609 mx.add_argument('--vm', action='store', dest='vm', choices=_vmChoices.keys(), help='the VM type to build/run') | |
2610 mx.add_argument('--vmbuild', action='store', dest='vmbuild', choices=_vmbuildChoices, help='the VM build to build/run (default: ' + _vmbuildChoices[0] + ')') | |
2611 mx.add_argument('--ecl', action='store_true', dest='make_eclipse_launch', help='create launch configuration for running VM execution(s) in Eclipse') | |
2612 mx.add_argument('--vmprefix', action='store', dest='vm_prefix', help='prefix for running the VM (e.g. "/usr/bin/gdb --args")', metavar='<prefix>') | |
2613 mx.add_argument('--gdb', action='store_const', const='/usr/bin/gdb --args', dest='vm_prefix', help='alias for --vmprefix "/usr/bin/gdb --args"') | |
2614 mx.add_argument('--lldb', action='store_const', const='lldb --', dest='vm_prefix', help='alias for --vmprefix "lldb --"') | |
2615 | |
2616 commands.update({ | |
2617 'export': [export, '[-options] [zipfile]'], | |
2618 }) | |
2619 | |
2620 mx.update_commands(suite, commands) | |
2621 | |
2622 def mx_post_parse_cmd_line(opts): # | |
2623 # TODO _minVersion check could probably be part of a Suite in mx? | |
2624 def _versionCheck(version): | |
2625 return version >= _minVersion and (not _untilVersion or version >= _untilVersion) | |
2626 versionDesc = ">=" + str(_minVersion) | |
2627 if _untilVersion: | |
2628 versionDesc += " and <=" + str(_untilVersion) | |
2629 mx.java(_versionCheck, versionDescription=versionDesc, defaultJdk=True) | |
2630 | |
2631 if _vmSourcesAvailable: | |
2632 if hasattr(opts, 'vm') and opts.vm is not None: | |
2633 global _vm | |
2634 _vm = opts.vm | |
2635 _vm = _vm.replace('graal', 'jvmci') | |
2636 if hasattr(opts, 'vmbuild') and opts.vmbuild is not None: | |
2637 global _vmbuild | |
2638 _vmbuild = opts.vmbuild | |
2639 global _make_eclipse_launch | |
2640 _make_eclipse_launch = getattr(opts, 'make_eclipse_launch', False) | |
2641 global _jacoco | |
2642 _jacoco = opts.jacoco | |
2643 global _vm_cwd | |
2644 _vm_cwd = opts.vm_cwd | |
2645 global _installed_jdks | |
2646 _installed_jdks = opts.installed_jdks | |
2647 global _vm_prefix | |
2648 _vm_prefix = opts.vm_prefix | |
2649 | |
2650 for jdkDist in _jdkDeployedDists: | |
2651 def _close(jdkDeployable): | |
2652 def _install(dist): | |
2653 assert dist.name == jdkDeployable.name, dist.name + "!=" + jdkDeployable.name | |
2654 if not jdkDist.partOfHotSpot: | |
2655 _installDistInJdks(jdkDeployable) | |
2656 return _install | |
2657 mx.distribution(jdkDist.name).add_update_listener(_close(jdkDist)) |