Mercurial > hg > graal-compiler

comparison mx/mx_graal.py @ 16313:151fe6b1e511

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Merge
author Stefan Anzinger <stefan.anzinger@gmail.com>
date Fri, 30 May 2014 15:09:09 +0200
parents 21c3c233e81a d676c4beeab8
children f57cf459f5d3
comparison
equal deleted inserted replaced
16312:b5f89908b9ba 16313:151fe6b1e511
24 # or visit www.oracle.com if you need additional information or have any 24 # or visit www.oracle.com if you need additional information or have any
25 # questions. 25 # questions.
26 # 26 #
27 # ---------------------------------------------------------------------------------------------------- 27 # ----------------------------------------------------------------------------------------------------
28 28
29 import os, sys, shutil, zipfile, tempfile, re, time, datetime, platform, subprocess, multiprocessing, StringIO 29 import os, stat, errno, sys, shutil, zipfile, tarfile, tempfile, re, time, datetime, platform, subprocess, multiprocessing, StringIO, socket
30 from os.path import join, exists, dirname, basename, getmtime 30 from os.path import join, exists, dirname, basename, getmtime
31 from argparse import ArgumentParser, RawDescriptionHelpFormatter, REMAINDER 31 from argparse import ArgumentParser, RawDescriptionHelpFormatter, REMAINDER
32 from outputparser import OutputParser, ValuesMatcher 32 from outputparser import OutputParser, ValuesMatcher
33 from fnmatch import fnmatch
34 import mx 33 import mx
35 import xml.dom.minidom 34 import xml.dom.minidom
36 import sanitycheck 35 import sanitycheck
37 import itertools 36 import itertools
38 import json, textwrap 37 import json, textwrap
38 import fnmatch
39 39
40 # This works because when mx loads this file, it makes sure __file__ gets an absolute path 40 # This works because when mx loads this file, it makes sure __file__ gets an absolute path
41 _graal_home = dirname(dirname(__file__)) 41 _graal_home = dirname(dirname(__file__))
42 42
43 """ Used to distinguish an exported GraalVM (see 'mx export'). """ 43 """ Used to distinguish an exported GraalVM (see 'mx export'). """
44 _vmSourcesAvailable = exists(join(_graal_home, 'make')) and exists(join(_graal_home, 'src')) 44 _vmSourcesAvailable = exists(join(_graal_home, 'make')) and exists(join(_graal_home, 'src'))
45 45
46 """ The VMs that can be built and run along with an optional description. Only VMs with a 46 """ The VMs that can be built and run along with an optional description. Only VMs with a
47 description are listed in the dialogue for setting the default VM (see _get_vm()). """ 47 description are listed in the dialogue for setting the default VM (see _get_vm()). """
48 _vmChoices = { 48 _vmChoices = {
49 'graal' : 'All compilation is performed with Graal. This includes bootstrapping Graal itself unless -XX:-BootstrapGraal is used.', 49 'graal' : 'Normal compilation is performed with a tiered system (C1 + Graal), Truffle compilation is performed with Graal.',
50 'server' : 'Normal compilation is performed with the tiered system (i.e., client + server), Truffle compilation is performed with Graal. Use this for optimal Truffle performance.', 50 'server' : 'Normal compilation is performed with a tiered system (C1 + C2), Truffle compilation is performed with Graal. Use this for optimal Truffle performance.',
51 'client' : None, # normal compilation with client compiler, explicit compilation (e.g., by Truffle) with Graal 51 'client' : None, # normal compilation with client compiler, explicit compilation (e.g., by Truffle) with Graal
52 'server-nograal' : None, # all compilation with tiered system (i.e., client + server), Graal omitted 52 'server-nograal' : None, # all compilation with tiered system (i.e., client + server), Graal omitted
53 'client-nograal' : None, # all compilation with client compiler, Graal omitted 53 'client-nograal' : None, # all compilation with client compiler, Graal omitted
54 'original' : None, # default VM copied from bootstrap JDK 54 'original' : None, # default VM copied from bootstrap JDK
55 } 55 }
78 """ Prefix for running the VM. """ 78 """ Prefix for running the VM. """
79 _vm_prefix = None 79 _vm_prefix = None
80 80
81 _make_eclipse_launch = False 81 _make_eclipse_launch = False
82 82
83 # @CallerSensitive introduced in 7u25 83 _minVersion = mx.VersionSpec('1.8')
84 _minVersion = mx.VersionSpec('1.7.0_25')
85 84
86 JDK_UNIX_PERMISSIONS = 0755 85 JDK_UNIX_PERMISSIONS = 0755
87 86
88 def isVMSupported(vm): 87 def isVMSupported(vm):
89 if 'client' in vm and len(platform.mac_ver()[0]) != 0: 88 if 'client' in vm and len(platform.mac_ver()[0]) != 0:
148 os.path.walk(dirname, _chmodDir, chmodFlags) 147 os.path.walk(dirname, _chmodDir, chmodFlags)
149 148
150 def clean(args): 149 def clean(args):
151 """clean the GraalVM source tree""" 150 """clean the GraalVM source tree"""
152 opts = mx.clean(args, parser=ArgumentParser(prog='mx clean')) 151 opts = mx.clean(args, parser=ArgumentParser(prog='mx clean'))
152
153 if opts.native: 153 if opts.native:
154 def handleRemoveReadonly(func, path, exc):
155 excvalue = exc[1]
156 if mx.get_os() == 'windows' and func in (os.rmdir, os.remove) and excvalue.errno == errno.EACCES:
157 os.chmod(path, stat.S_IRWXU | stat.S_IRWXG | stat.S_IRWXO) # 0777
158 func(path)
159 else:
160 raise
161
154 def rmIfExists(name): 162 def rmIfExists(name):
155 if os.path.isdir(name): 163 if os.path.isdir(name):
156 shutil.rmtree(name) 164 shutil.rmtree(name, ignore_errors=False, onerror=handleRemoveReadonly)
157 elif os.path.isfile(name): 165 elif os.path.isfile(name):
158 os.unlink(name) 166 os.unlink(name)
159 167
160 rmIfExists(join(_graal_home, 'build')) 168 rmIfExists(join(_graal_home, 'build'))
161 rmIfExists(join(_graal_home, 'build-nograal')) 169 rmIfExists(join(_graal_home, 'build-nograal'))
162 rmIfExists(_jdksDir()) 170 rmIfExists(_jdksDir())
163 rmIfExists(mx.distribution('GRAAL').path) 171 rmIfExists(mx.distribution('GRAAL').path)
164 172
165 def export(args): 173 def export(args):
166 """create a GraalVM zip file for distribution""" 174 """create archives of builds split by vmbuild and vm"""
167 175
168 parser = ArgumentParser(prog='mx export') 176 parser = ArgumentParser(prog='mx export')
169 parser.add_argument('--omit-vm-build', action='store_false', dest='vmbuild', help='omit VM build step')
170 parser.add_argument('--omit-dist-init', action='store_false', dest='distInit', help='omit class files and IDE configurations from distribution')
171 parser.add_argument('zipfile', nargs=REMAINDER, metavar='zipfile')
172
173 args = parser.parse_args(args) 177 args = parser.parse_args(args)
174 178
175 tmp = tempfile.mkdtemp(prefix='tmp', dir=_graal_home) 179 # collect data about export
176 if args.vmbuild: 180 infos = dict()
177 # Make sure the product VM binary is up to date 181 infos['timestamp'] = time.time()
178 with VM(vmbuild='product'): 182
179 build([]) 183 hgcfg = mx.HgConfig()
180 184 hgcfg.check()
181 mx.log('Copying Java sources and mx files...') 185 infos['revision'] = hgcfg.tip('.') + ('+' if hgcfg.isDirty('.') else '')
182 mx.run(('hg archive -I graal -I mx -I mxtool -I mx.sh ' + tmp).split()) 186 # TODO: infos['repository']
183 187
184 # Copy the GraalVM JDK 188 infos['jdkversion'] = str(mx.java().version)
185 mx.log('Copying GraalVM JDK...') 189
186 src = _jdk() 190 infos['architecture'] = _arch()
187 dst = join(tmp, basename(src)) 191 infos['platform'] = mx.get_os()
188 shutil.copytree(src, dst) 192
189 zfName = join(_graal_home, 'graalvm-' + mx.get_os() + '.zip') 193 if mx.get_os != 'windows':
190 zf = zipfile.ZipFile(zfName, 'w') 194 pass
191 for root, _, files in os.walk(tmp): 195 # infos['ccompiler']
192 for f in files: 196 # infos['linker']
193 name = join(root, f) 197
194 arcname = name[len(tmp) + 1:] 198 infos['hostname'] = socket.gethostname()
195 zf.write(join(tmp, name), arcname) 199
196 200 def _writeJson(suffix, properties):
197 # create class files and IDE configurations 201 d = infos.copy()
198 if args.distInit: 202 for k, v in properties.iteritems():
199 mx.log('Creating class files...') 203 assert not d.has_key(k)
200 mx.run('mx build'.split(), cwd=tmp) 204 d[k] = v
201 mx.log('Creating IDE configurations...') 205
202 mx.run('mx ideinit'.split(), cwd=tmp) 206 jsonFileName = 'export-' + suffix + '.json'
203 207 with open(jsonFileName, 'w') as f:
204 # clean up temp directory 208 print >> f, json.dumps(d)
205 mx.log('Cleaning up...') 209 return jsonFileName
206 shutil.rmtree(tmp) 210
207 211
208 mx.log('Created distribution in ' + zfName) 212 def _genFileName(archivtype, middle):
213 idPrefix = infos['revision'] + '_'
214 idSuffix = '.tar.gz'
215 return join(_graal_home, "graalvm_" + archivtype + "_" + idPrefix + middle + idSuffix)
216
217 def _genFileArchPlatformName(archivtype, middle):
218 return _genFileName(archivtype, infos['platform'] + '_' + infos['architecture'] + '_' + middle)
219
220
221 # archive different build types of hotspot
222 for vmBuild in _vmbuildChoices:
223 jdkpath = join(_jdksDir(), vmBuild)
224 if not exists(jdkpath):
225 mx.logv("skipping " + vmBuild)
226 continue
227
228 tarName = _genFileArchPlatformName('basejdk', vmBuild)
229 mx.logv("creating basejdk " + tarName)
230 vmSet = set()
231 with tarfile.open(tarName, 'w:gz') as tar:
232 for root, _, files in os.walk(jdkpath):
233 if basename(root) in _vmChoices.keys():
234 # TODO: add some assert to check path assumption
235 vmSet.add(root)
236 continue
237
238 for f in files:
239 name = join(root, f)
240 # print name
241 tar.add(name, name)
242
243 n = _writeJson("basejdk-" + vmBuild, {'vmbuild' : vmBuild})
244 tar.add(n, n)
245
246 # create a separate archive for each VM
247 for vm in vmSet:
248 bVm = basename(vm)
249 vmTarName = _genFileArchPlatformName('vm', vmBuild + '_' + bVm)
250 mx.logv("creating vm " + vmTarName)
251
252 debugFiles = set()
253 with tarfile.open(vmTarName, 'w:gz') as tar:
254 for root, _, files in os.walk(vm):
255 for f in files:
256 # TODO: mac, windows, solaris?
257 if any(map(f.endswith, [".debuginfo"])):
258 debugFiles.add(f)
259 else:
260 name = join(root, f)
261 # print name
262 tar.add(name, name)
263
264 n = _writeJson("vm-" + vmBuild + "-" + bVm, {'vmbuild' : vmBuild, 'vm' : bVm})
265 tar.add(n, n)
266
267 if len(debugFiles) > 0:
268 debugTarName = _genFileArchPlatformName('debugfilesvm', vmBuild + '_' + bVm)
269 mx.logv("creating debugfilesvm " + debugTarName)
270 with tarfile.open(debugTarName, 'w:gz') as tar:
271 for f in debugFiles:
272 name = join(root, f)
273 # print name
274 tar.add(name, name)
275
276 n = _writeJson("debugfilesvm-" + vmBuild + "-" + bVm, {'vmbuild' : vmBuild, 'vm' : bVm})
277 tar.add(n, n)
278
279 # graal directory
280 graalDirTarName = _genFileName('classfiles', 'javac')
281 mx.logv("creating graal " + graalDirTarName)
282 with tarfile.open(graalDirTarName, 'w:gz') as tar:
283 for root, _, files in os.walk("graal"):
284 for f in [f for f in files if not f.endswith('.java')]:
285 name = join(root, f)
286 # print name
287 tar.add(name, name)
288
289 n = _writeJson("graal", {'javacompiler' : 'javac'})
290 tar.add(n, n)
291
209 292
210 def _run_benchmark(args, availableBenchmarks, runBenchmark): 293 def _run_benchmark(args, availableBenchmarks, runBenchmark):
211 294
212 vmOpts, benchmarksAndOptions = _extract_VM_args(args, useDoubleDash=availableBenchmarks is None) 295 vmOpts, benchmarksAndOptions = _extract_VM_args(args, useDoubleDash=availableBenchmarks is None)
213 296
399 else: 482 else:
400 toDelete = join(jreLibDir, 'graal.options') 483 toDelete = join(jreLibDir, 'graal.options')
401 if exists(toDelete): 484 if exists(toDelete):
402 os.unlink(toDelete) 485 os.unlink(toDelete)
403 486
487 def _update_graalRuntime_inline_hpp(graalJar):
488 p = mx.project('com.oracle.graal.hotspot.sourcegen')
489 mainClass = 'com.oracle.graal.hotspot.sourcegen.GenGraalRuntimeInlineHpp'
490 if exists(join(p.output_dir(), mainClass.replace('.', os.sep) + '.class')):
491 hsSrcGenDir = join(mx.project('com.oracle.graal.hotspot').source_gen_dir(), 'hotspot')
492 if not exists(hsSrcGenDir):
493 os.makedirs(hsSrcGenDir)
494
495 tmp = StringIO.StringIO()
496 mx.run_java(['-cp', '{}{}{}'.format(graalJar, os.pathsep, p.output_dir()), mainClass], out=tmp.write)
497 mx.update_file(join(hsSrcGenDir, 'graalRuntime.inline.hpp'), tmp.getvalue())
498
404 def _installGraalJarInJdks(graalDist): 499 def _installGraalJarInJdks(graalDist):
405 graalJar = graalDist.path 500 graalJar = graalDist.path
501 _update_graalRuntime_inline_hpp(graalJar)
406 jdks = _jdksDir() 502 jdks = _jdksDir()
407 503
408 if exists(jdks): 504 if exists(jdks):
409 for e in os.listdir(jdks): 505 for e in os.listdir(jdks):
410 jreLibDir = join(jdks, e, 'jre', 'lib') 506 jreLibDir = join(jdks, e, 'jre', 'lib')
605 mustBuild = True 701 mustBuild = True
606 else: 702 else:
607 mustBuild = False 703 mustBuild = False
608 timestamp = os.path.getmtime(timestampFile) 704 timestamp = os.path.getmtime(timestampFile)
609 sources = [] 705 sources = []
610 for d in ['src', 'make']: 706 for d in ['src', 'make', join('graal', 'com.oracle.graal.hotspot', 'src_gen', 'hotspot')]:
611 for root, dirnames, files in os.walk(join(_graal_home, d)): 707 for root, dirnames, files in os.walk(join(_graal_home, d)):
612 # ignore <graal>/src/share/tools 708 # ignore <graal>/src/share/tools
613 if root == join(_graal_home, 'src', 'share'): 709 if root == join(_graal_home, 'src', 'share'):
614 dirnames.remove('tools') 710 dirnames.remove('tools')
615 sources += [join(root, name) for name in files] 711 sources += [join(root, name) for name in files]
827 if defaultAllVMArgs: 923 if defaultAllVMArgs:
828 return args, [] 924 return args, []
829 else: 925 else:
830 return [], args 926 return [], args
831 927
832 def _run_tests(args, harness, annotations, testfile, whitelist): 928 def _run_tests(args, harness, annotations, testfile, whitelist, regex):
833 929
834 930
835 vmArgs, tests = _extract_VM_args(args) 931 vmArgs, tests = _extract_VM_args(args)
836 for t in tests: 932 for t in tests:
837 if t.startswith('-'): 933 if t.startswith('-'):
851 else: 947 else:
852 projs = set() 948 projs = set()
853 for t in tests: 949 for t in tests:
854 found = False 950 found = False
855 for c, p in candidates.iteritems(): 951 for c, p in candidates.iteritems():
856 if fnmatch(c, t): 952 if fnmatch.fnmatch(c, t):
857 found = True 953 found = True
858 classes.append(c) 954 classes.append(c)
859 projs.add(p.name) 955 projs.add(p.name)
860 if not found: 956 if not found:
861 mx.log('warning: no tests matched by substring "' + t) 957 mx.log('warning: no tests matched by substring "' + t)
862 projectscp = mx.classpath(projs) 958 projectscp = mx.classpath(projs)
863 959
864 if whitelist: 960 if whitelist:
865 classes = list(set(classes) & set(whitelist)) 961 classes = [c for c in classes if any((glob.match(c) for glob in whitelist))]
962
963 if regex:
964 classes = [c for c in classes if re.search(regex, c)]
866 965
867 if len(classes) != 0: 966 if len(classes) != 0:
868 f_testfile = open(testfile, 'w') 967 f_testfile = open(testfile, 'w')
869 for c in classes: 968 for c in classes:
870 f_testfile.write(c + '\n') 969 f_testfile.write(c + '\n')
871 f_testfile.close() 970 f_testfile.close()
872 harness(projectscp, vmArgs) 971 harness(projectscp, vmArgs)
873 972
874 def _unittest(args, annotations, prefixcp="", whitelist=None): 973 def _unittest(args, annotations, prefixcp="", whitelist=None, verbose=False, enable_timing=False, regex=None, color=False, eager_stacktrace=False, gc_after_test=False):
875 mxdir = dirname(__file__) 974 mxdir = dirname(__file__)
876 name = 'JUnitWrapper' 975 name = 'JUnitWrapper'
877 javaSource = join(mxdir, name + '.java') 976 javaSource = join(mxdir, name + '.java')
878 javaClass = join(mxdir, name + '.class') 977 javaClass = join(mxdir, name + '.class')
879 testfile = os.environ.get('MX_TESTFILE', None) 978 testfile = os.environ.get('MX_TESTFILE', None)
880 if testfile is None: 979 if testfile is None:
881 (_, testfile) = tempfile.mkstemp(".testclasses", "graal") 980 (_, testfile) = tempfile.mkstemp(".testclasses", "graal")
882 os.close(_) 981 os.close(_)
982 corecp = mx.classpath(['com.oracle.graal.test', 'ant'])
983
984 if not exists(javaClass) or getmtime(javaClass) < getmtime(javaSource):
985 subprocess.check_call([mx.java().javac, '-cp', corecp, '-d', mxdir, javaSource])
986
987 coreArgs = []
988 if verbose:
989 coreArgs.append('-JUnitVerbose')
990 if enable_timing:
991 coreArgs.append('-JUnitEnableTiming')
992 if color:
993 coreArgs.append('-JUnitColor')
994 if eager_stacktrace:
995 coreArgs.append('-JUnitEagerStackTrace')
996 if gc_after_test:
997 coreArgs.append('-JUnitGCAfterTest')
998
883 999
884 def harness(projectscp, vmArgs): 1000 def harness(projectscp, vmArgs):
885 projectscp = mx.classpath("ant") + os.pathsep + projectscp
886 if not exists(javaClass) or getmtime(javaClass) < getmtime(javaSource):
887 subprocess.check_call([mx.java().javac, '-cp', projectscp, '-d', mxdir, javaSource])
888 if _get_vm() != 'graal': 1001 if _get_vm() != 'graal':
889 prefixArgs = ['-esa', '-ea'] 1002 prefixArgs = ['-esa', '-ea']
890 else: 1003 else:
891 prefixArgs = ['-XX:-BootstrapGraal', '-esa', '-ea'] 1004 prefixArgs = ['-XX:-BootstrapGraal', '-esa', '-ea']
892 with open(testfile) as fp: 1005 with open(testfile) as fp:
893 testclasses = [l.rstrip() for l in fp.readlines()] 1006 testclasses = [l.rstrip() for l in fp.readlines()]
894 if len(testclasses) == 1: 1007 if len(testclasses) == 1:
895 # Execute Junit directly when one test is being run. This simplifies 1008 # Execute Junit directly when one test is being run. This simplifies
896 # replaying the VM execution in a native debugger (e.g., gdb). 1009 # replaying the VM execution in a native debugger (e.g., gdb).
897 vm(prefixArgs + vmArgs + ['-cp', prefixcp + projectscp, 'org.junit.runner.JUnitCore'] + testclasses) 1010 vm(prefixArgs + vmArgs + ['-cp', prefixcp + corecp + ':' + projectscp, 'com.oracle.graal.test.GraalJUnitCore'] + coreArgs + testclasses)
898 else: 1011 else:
899 vm(prefixArgs + vmArgs + ['-cp', prefixcp + projectscp + os.pathsep + mxdir, name] + [testfile]) 1012 vm(prefixArgs + vmArgs + ['-cp', prefixcp + corecp + ':' + projectscp + os.pathsep + mxdir, name] + [testfile] + coreArgs)
900 1013
901 try: 1014 try:
902 _run_tests(args, harness, annotations, testfile, whitelist) 1015 _run_tests(args, harness, annotations, testfile, whitelist, regex)
903 finally: 1016 finally:
904 if os.environ.get('MX_TESTFILE') is None: 1017 if os.environ.get('MX_TESTFILE') is None:
905 os.remove(testfile) 1018 os.remove(testfile)
906 1019
907 _unittestHelpSuffix = """ 1020 _unittestHelpSuffix = """
908 Unittest options: 1021 Unittest options:
909 1022
910 --short-only run short testcases only 1023 --whitelist <file> run only testcases which are included
911 --long-only run long testcases only 1024 in the given whitelist
912 --baseline-whitelist run only testcases which are known to 1025 --verbose enable verbose JUnit output
913 work with the baseline compiler 1026 --enable-timing enable JUnit test timing
1027 --regex <regex> run only testcases matching a regular expression
1028 --color enable colors output
1029 --eager-stacktrace print stacktrace eagerly
1030 --gc-after-test force a GC after each test
914 1031
915 To avoid conflicts with VM options '--' can be used as delimiter. 1032 To avoid conflicts with VM options '--' can be used as delimiter.
916 1033
917 If filters are supplied, only tests whose fully qualified name 1034 If filters are supplied, only tests whose fully qualified name
918 includes a filter as a substring are run. 1035 includes a filter as a substring are run.
945 description='run the JUnit tests', 1062 description='run the JUnit tests',
946 add_help=False, 1063 add_help=False,
947 formatter_class=RawDescriptionHelpFormatter, 1064 formatter_class=RawDescriptionHelpFormatter,
948 epilog=_unittestHelpSuffix, 1065 epilog=_unittestHelpSuffix,
949 ) 1066 )
950 group = parser.add_mutually_exclusive_group() 1067 parser.add_argument('--whitelist', help='run testcases specified in whitelist only', metavar='<path>')
951 group.add_argument('--short-only', action='store_true', help='run short testcases only') 1068 parser.add_argument('--verbose', help='enable verbose JUnit output', action='store_true')
952 group.add_argument('--long-only', action='store_true', help='run long testcases only') 1069 parser.add_argument('--enable-timing', help='enable JUnit test timing', action='store_true')
953 parser.add_argument('--baseline-whitelist', action='store_true', help='run baseline testcases only') 1070 parser.add_argument('--regex', help='run only testcases matching a regular expression', metavar='<regex>')
1071 parser.add_argument('--color', help='enable color output', action='store_true')
1072 parser.add_argument('--eager-stacktrace', help='print stacktrace eagerly', action='store_true')
1073 parser.add_argument('--gc-after-test', help='force a GC after each test', action='store_true')
954 1074
955 ut_args = [] 1075 ut_args = []
956 delimiter = False 1076 delimiter = False
957 # check for delimiter 1077 # check for delimiter
958 while len(args) > 0: 1078 while len(args) > 0:
967 parsed_args = parser.parse_args(ut_args) 1087 parsed_args = parser.parse_args(ut_args)
968 else: 1088 else:
969 # parse all know arguments 1089 # parse all know arguments
970 parsed_args, args = parser.parse_known_args(ut_args) 1090 parsed_args, args = parser.parse_known_args(ut_args)
971 1091
972 whitelist = None 1092 if parsed_args.whitelist:
973 if parsed_args.baseline_whitelist:
974 baseline_whitelist_file = 'test/baseline_whitelist.txt'
975 try: 1093 try:
976 with open(join(_graal_home, baseline_whitelist_file)) as fp: 1094 with open(join(_graal_home, parsed_args.whitelist)) as fp:
977 whitelist = [l.rstrip() for l in fp.readlines()] 1095 parsed_args.whitelist = [re.compile(fnmatch.translate(l.rstrip())) for l in fp.readlines() if not l.startswith('#')]
978 except IOError: 1096 except IOError:
979 mx.log('warning: could not read baseline whitelist: ' + baseline_whitelist_file) 1097 mx.log('warning: could not read whitelist: ' + parsed_args.whitelist)
980 1098
981 if parsed_args.long_only: 1099 _unittest(args, ['@Test', '@Parameters'], **parsed_args.__dict__)
982 annotations = ['@LongTest', '@Parameters']
983 elif parsed_args.short_only:
984 annotations = ['@Test']
985 else:
986 annotations = ['@Test', '@LongTest', '@Parameters']
987
988 _unittest(args, annotations, whitelist=whitelist)
989 1100
990 def shortunittest(args): 1101 def shortunittest(args):
991 """alias for 'unittest --short-only'{0}""" 1102 """alias for 'unittest --whitelist test/whitelist_shortunittest.txt'{0}"""
992 1103
993 args.insert(0, '--short-only') 1104 args = ['--whitelist', 'test/whitelist_shortunittest.txt'] + args
994 unittest(args)
995
996 def longunittest(args):
997 """alias for 'unittest --long-only'{0}"""
998
999 args.insert(0, '--long-only')
1000 unittest(args) 1105 unittest(args)
1001 1106
1002 def buildvms(args): 1107 def buildvms(args):
1003 """build one or more VMs in various configurations""" 1108 """build one or more VMs in various configurations"""
1004 1109
1101 vm(['-XX:-TieredCompilation', '-G:+ImmutableCode', '-G:+VerifyPhases', '-esa', '-version']) 1206 vm(['-XX:-TieredCompilation', '-G:+ImmutableCode', '-G:+VerifyPhases', '-esa', '-version'])
1102 tasks.append(t.stop()) 1207 tasks.append(t.stop())
1103 1208
1104 with VM('server', 'product'): # hosted mode 1209 with VM('server', 'product'): # hosted mode
1105 t = Task('UnitTests:hosted-product') 1210 t = Task('UnitTests:hosted-product')
1106 unittest([]) 1211 unittest(['--enable-timing', '--verbose'])
1212 tasks.append(t.stop())
1213
1214 with VM('server', 'product'): # hosted mode
1215 t = Task('UnitTests-BaselineCompiler:hosted-product')
1216 unittest(['--enable-timing', '--verbose', '--whitelist', 'test/whitelist_baseline.txt', '-G:+UseBaselineCompiler'])
1107 tasks.append(t.stop()) 1217 tasks.append(t.stop())
1108 1218
1109 for vmbuild in ['fastdebug', 'product']: 1219 for vmbuild in ['fastdebug', 'product']:
1110 for test in sanitycheck.getDacapos(level=sanitycheck.SanityCheckLevel.Gate, gateBuildLevel=vmbuild) + sanitycheck.getScalaDacapos(level=sanitycheck.SanityCheckLevel.Gate, gateBuildLevel=vmbuild): 1220 for test in sanitycheck.getDacapos(level=sanitycheck.SanityCheckLevel.Gate, gateBuildLevel=vmbuild) + sanitycheck.getScalaDacapos(level=sanitycheck.SanityCheckLevel.Gate, gateBuildLevel=vmbuild):
1111 t = Task(str(test) + ':' + vmbuild) 1221 t = Task(str(test) + ':' + vmbuild)
1199 t.abort('Rerun "mx canonicalizeprojects" and check-in the modified mx/projects files.') 1309 t.abort('Rerun "mx canonicalizeprojects" and check-in the modified mx/projects files.')
1200 tasks.append(t.stop()) 1310 tasks.append(t.stop())
1201 1311
1202 if mx.get_env('JDT'): 1312 if mx.get_env('JDT'):
1203 t = Task('BuildJavaWithEcj') 1313 t = Task('BuildJavaWithEcj')
1204 build(['--no-native', '--jdt-warning-as-error']) 1314 build(['-p', '--no-native', '--jdt-warning-as-error'])
1205 tasks.append(t.stop()) 1315 tasks.append(t.stop())
1206 1316
1207 _clean('CleanAfterEcjBuild') 1317 _clean('CleanAfterEcjBuild')
1208 1318
1209 t = Task('BuildJavaWithJavac') 1319 t = Task('BuildJavaWithJavac')
1210 build(['--no-native', '--force-javac']) 1320 build(['-p', '--no-native', '--force-javac'])
1211 tasks.append(t.stop()) 1321 tasks.append(t.stop())
1212 1322
1213 t = Task('Checkheaders') 1323 t = Task('Checkheaders')
1214 if checkheaders([]) != 0: 1324 if checkheaders([]) != 0:
1215 t.abort('Checkheaders warnings were found') 1325 t.abort('Checkheaders warnings were found')
1305 if mx.get_os() == 'windows': 1415 if mx.get_os() == 'windows':
1306 executable = join(libpath, 'c1visualizer', 'bin', 'c1visualizer.exe') 1416 executable = join(libpath, 'c1visualizer', 'bin', 'c1visualizer.exe')
1307 else: 1417 else:
1308 executable = join(libpath, 'c1visualizer', 'bin', 'c1visualizer') 1418 executable = join(libpath, 'c1visualizer', 'bin', 'c1visualizer')
1309 1419
1310 archive = join(libpath, 'c1visualizer.zip') 1420 archive = join(libpath, 'c1visualizer_2014-04-22.zip')
1311 if not exists(executable): 1421 if not exists(executable) or not exists(archive):
1312 if not exists(archive): 1422 if not exists(archive):
1313 mx.download(archive, ['https://java.net/downloads/c1visualizer/c1visualizer.zip']) 1423 mx.download(archive, ['https://java.net/downloads/c1visualizer/c1visualizer_2014-04-22.zip'])
1314 zf = zipfile.ZipFile(archive, 'r') 1424 zf = zipfile.ZipFile(archive, 'r')
1315 zf.extractall(libpath) 1425 zf.extractall(libpath)
1316 1426
1317 if not exists(executable): 1427 if not exists(executable):
1318 mx.abort('C1Visualizer binary does not exist: ' + executable) 1428 mx.abort('C1Visualizer binary does not exist: ' + executable)
1447 for name in deps: 1557 for name in deps:
1448 if not mx.project(name, fatalIfMissing=False): 1558 if not mx.project(name, fatalIfMissing=False):
1449 if not mx.library(name, fatalIfMissing=False): 1559 if not mx.library(name, fatalIfMissing=False):
1450 mx.log('Skipping ' + groupId + '.' + artifactId + '.jar as ' + name + ' cannot be resolved') 1560 mx.log('Skipping ' + groupId + '.' + artifactId + '.jar as ' + name + ' cannot be resolved')
1451 return 1561 return
1452 d = mx.Distribution(graalSuite, name=artifactId, path=path, sourcesPath=path, deps=deps, excludedLibs=[]) 1562 d = mx.Distribution(graalSuite, name=artifactId, path=path, sourcesPath=path, deps=deps, mainClass=None, excludedDependencies=[], distDependencies=[])
1453 d.make_archive() 1563 d.make_archive()
1454 cmd = ['mvn', '-q', 'install:install-file', '-DgroupId=' + groupId, '-DartifactId=' + artifactId, 1564 cmd = ['mvn', 'install:install-file', '-DgroupId=' + groupId, '-DartifactId=' + artifactId,
1455 '-Dversion=1.0-SNAPSHOT', '-Dpackaging=jar', '-Dfile=' + d.path] 1565 '-Dversion=1.0-SNAPSHOT', '-Dpackaging=jar', '-Dfile=' + d.path]
1566 if not mx._opts.verbose:
1567 cmd.append('-q')
1456 if args.settings: 1568 if args.settings:
1457 cmd = cmd + ['-s', args.settings] 1569 cmd = cmd + ['-s', args.settings]
1458 mx.run(cmd) 1570 mx.run(cmd)
1459 os.unlink(d.path) 1571 os.unlink(d.path)
1460 1572
1481 args = parser.parse_args(args) 1593 args = parser.parse_args(args)
1482 1594
1483 jmhPath = _get_jmh_path() 1595 jmhPath = _get_jmh_path()
1484 mx.log('JMH benchmarks: ' + jmhPath) 1596 mx.log('JMH benchmarks: ' + jmhPath)
1485 1597
1598 # Ensure the mx injected dependencies are up to date
1599 makejmhdeps(['-p'] + (['-s', args.settings] if args.settings else []))
1600
1486 timestamp = mx.TimeStampFile(join(_graal_home, 'mx', 'jmh', jmhPath.replace(os.sep, '_') + '.timestamp')) 1601 timestamp = mx.TimeStampFile(join(_graal_home, 'mx', 'jmh', jmhPath.replace(os.sep, '_') + '.timestamp'))
1487 mustBuild = args.clean 1602 mustBuild = args.clean
1488 if not mustBuild: 1603 if not mustBuild:
1489 try: 1604 try:
1490 hgfiles = [join(jmhPath, f) for f in subprocess.check_output(['hg', '-R', jmhPath, 'locate']).split('\n')] 1605 hgfiles = [join(jmhPath, f) for f in subprocess.check_output(['hg', '-R', jmhPath, 'locate']).split('\n')]
1502 buildOutput.append(x) 1617 buildOutput.append(x)
1503 env = os.environ.copy() 1618 env = os.environ.copy()
1504 env['JAVA_HOME'] = _jdk(vmToCheck='server') 1619 env['JAVA_HOME'] = _jdk(vmToCheck='server')
1505 env['MAVEN_OPTS'] = '-server' 1620 env['MAVEN_OPTS'] = '-server'
1506 mx.log("Building benchmarks...") 1621 mx.log("Building benchmarks...")
1507 makejmhdeps(['-p'] + (['-s', args.settings] if args.settings else []))
1508 cmd = ['mvn'] 1622 cmd = ['mvn']
1509 if args.settings: 1623 if args.settings:
1510 cmd = cmd + ['-s', args.settings] 1624 cmd = cmd + ['-s', args.settings]
1511 if args.clean: 1625 if args.clean:
1512 cmd.append('clean') 1626 cmd.append('clean')
1530 1644
1531 vmArgs, benchmarksAndJsons = _extract_VM_args(args) 1645 vmArgs, benchmarksAndJsons = _extract_VM_args(args)
1532 1646
1533 benchmarks = [b for b in benchmarksAndJsons if not b.startswith('{')] 1647 benchmarks = [b for b in benchmarksAndJsons if not b.startswith('{')]
1534 jmhArgJsons = [b for b in benchmarksAndJsons if b.startswith('{')] 1648 jmhArgJsons = [b for b in benchmarksAndJsons if b.startswith('{')]
1535 1649 jmhOutDir = join(_graal_home, 'mx', 'jmh')
1536 jmhArgs = {'-rff' : join(_graal_home, 'mx', 'jmh', 'jmh.out'), '-v' : 'EXTRA' if mx._opts.verbose else 'NORMAL'} 1650 if not exists(jmhOutDir):
1651 os.makedirs(jmhOutDir)
1652 jmhOut = join(jmhOutDir, 'jmh.out')
1653 jmhArgs = {'-rff' : jmhOut, '-v' : 'EXTRA' if mx._opts.verbose else 'NORMAL'}
1537 1654
1538 # e.g. '{"-wi" : 20}' 1655 # e.g. '{"-wi" : 20}'
1539 for j in jmhArgJsons: 1656 for j in jmhArgJsons:
1540 try: 1657 try:
1541 for n, v in json.loads(j).iteritems(): 1658 for n, v in json.loads(j).iteritems():
1559 mx.logv('JMH: ignored ' + absoluteMicro + " because it doesn't start with 'micros-'") 1676 mx.logv('JMH: ignored ' + absoluteMicro + " because it doesn't start with 'micros-'")
1560 continue 1677 continue
1561 1678
1562 microJar = os.path.join(absoluteMicro, "target", "microbenchmarks.jar") 1679 microJar = os.path.join(absoluteMicro, "target", "microbenchmarks.jar")
1563 if not exists(microJar): 1680 if not exists(microJar):
1564 mx.abort('Missing ' + microJar + ' - please run "mx buildjmh"') 1681 mx.log('Missing ' + microJar + ' - please run "mx buildjmh"')
1682 continue
1565 if benchmarks: 1683 if benchmarks:
1566 def _addBenchmark(x): 1684 def _addBenchmark(x):
1567 if x.startswith("Benchmark:"): 1685 if x.startswith("Benchmark:"):
1568 return 1686 return
1569 match = False 1687 match = False
1708 out = 'coverage' 1826 out = 'coverage'
1709 if len(args) == 1: 1827 if len(args) == 1:
1710 out = args[0] 1828 out = args[0]
1711 elif len(args) > 1: 1829 elif len(args) > 1:
1712 mx.abort('jacocoreport takes only one argument : an output directory') 1830 mx.abort('jacocoreport takes only one argument : an output directory')
1713 mx.run_java(['-jar', jacocoreport.get_path(True), '-in', 'jacoco.exec', '-g', join(_graal_home, 'graal'), out]) 1831 mx.run_java(['-jar', jacocoreport.get_path(True), '--in', 'jacoco.exec', '--out', out] + [p.dir for p in mx.projects()])
1714 1832
1715 def sl(args): 1833 def sl(args):
1716 """run an SL program""" 1834 """run an SL program"""
1717 vmArgs, slArgs = _extract_VM_args(args) 1835 vmArgs, slArgs = _extract_VM_args(args)
1718 vm(vmArgs + ['-cp', mx.classpath("com.oracle.truffle.sl"), "com.oracle.truffle.sl.SLMain"] + slArgs) 1836 vm(vmArgs + ['-cp', mx.classpath("com.oracle.truffle.sl"), "com.oracle.truffle.sl.SLMain"] + slArgs)
1719
1720 def trufflejar(args=None):
1721 """make truffle.jar"""
1722
1723 # Test with the built classes
1724 _unittest(["com.oracle.truffle.api.test", "com.oracle.truffle.api.dsl.test"], ['@Test', '@LongTest', '@Parameters'])
1725
1726 # We use the DSL processor as the starting point for the classpath - this
1727 # therefore includes the DSL processor, the DSL and the API.
1728 packagejar(mx.classpath("com.oracle.truffle.dsl.processor").split(os.pathsep), "truffle.jar", None, "com.oracle.truffle.dsl.processor.TruffleProcessor")
1729
1730 # Test with the JAR
1731 _unittest(["com.oracle.truffle.api.test", "com.oracle.truffle.api.dsl.test"], ['@Test', '@LongTest', '@Parameters'], "truffle.jar:")
1732
1733 1837
1734 def isGraalEnabled(vm): 1838 def isGraalEnabled(vm):
1735 return vm != 'original' and not vm.endswith('nograal') 1839 return vm != 'original' and not vm.endswith('nograal')
1736 1840
1737 def site(args): 1841 def site(args):
1949 'specjbb2013': [specjbb2013, '[VM options] [-- [SPECjbb2013 options]]'], 2053 'specjbb2013': [specjbb2013, '[VM options] [-- [SPECjbb2013 options]]'],
1950 'specjbb2005': [specjbb2005, '[VM options] [-- [SPECjbb2005 options]]'], 2054 'specjbb2005': [specjbb2005, '[VM options] [-- [SPECjbb2005 options]]'],
1951 'gate' : [gate, '[-options]'], 2055 'gate' : [gate, '[-options]'],
1952 'bench' : [bench, '[-resultfile file] [all(default)|dacapo|specjvm2008|bootstrap]'], 2056 'bench' : [bench, '[-resultfile file] [all(default)|dacapo|specjvm2008|bootstrap]'],
1953 'unittest' : [unittest, '[unittest options] [--] [VM options] [filters...]', _unittestHelpSuffix], 2057 'unittest' : [unittest, '[unittest options] [--] [VM options] [filters...]', _unittestHelpSuffix],
1954 'longunittest' : [longunittest, '[unittest options] [--] [VM options] [filters...]', _unittestHelpSuffix],
1955 'makejmhdeps' : [makejmhdeps, ''], 2058 'makejmhdeps' : [makejmhdeps, ''],
1956 'shortunittest' : [shortunittest, '[unittest options] [--] [VM options] [filters...]', _unittestHelpSuffix], 2059 'shortunittest' : [shortunittest, '[unittest options] [--] [VM options] [filters...]', _unittestHelpSuffix],
1957 'jacocoreport' : [jacocoreport, '[output directory]'], 2060 'jacocoreport' : [jacocoreport, '[output directory]'],
1958 'site' : [site, '[-options]'], 2061 'site' : [site, '[-options]'],
1959 'vm': [vm, '[-options] class [args...]'], 2062 'vm': [vm, '[-options] class [args...]'],
1960 'vmg': [vmg, '[-options] class [args...]'], 2063 'vmg': [vmg, '[-options] class [args...]'],
1961 'vmfg': [vmfg, '[-options] class [args...]'], 2064 'vmfg': [vmfg, '[-options] class [args...]'],
1962 'deoptalot' : [deoptalot, '[n]'], 2065 'deoptalot' : [deoptalot, '[n]'],
1963 'longtests' : [longtests, ''], 2066 'longtests' : [longtests, ''],
1964 'sl' : [sl, '[SL args|@VM options]'], 2067 'sl' : [sl, '[SL args|@VM options]']
1965 'trufflejar' : [trufflejar, '']
1966 } 2068 }
1967 2069
1968 mx.add_argument('--jacoco', help='instruments com.oracle.* classes using JaCoCo', default='off', choices=['off', 'on', 'append']) 2070 mx.add_argument('--jacoco', help='instruments com.oracle.* classes using JaCoCo', default='off', choices=['off', 'on', 'append'])
1969 mx.add_argument('--vmcwd', dest='vm_cwd', help='current directory will be changed to <path> before the VM is executed', default=None, metavar='<path>') 2071 mx.add_argument('--vmcwd', dest='vm_cwd', help='current directory will be changed to <path> before the VM is executed', default=None, metavar='<path>')
1970 mx.add_argument('--installed-jdks', help='the base directory in which the JDKs cloned from $JAVA_HOME exist. ' + 2072 mx.add_argument('--installed-jdks', help='the base directory in which the JDKs cloned from $JAVA_HOME exist. ' +
2006 _installed_jdks = opts.installed_jdks 2108 _installed_jdks = opts.installed_jdks
2007 global _vm_prefix 2109 global _vm_prefix
2008 _vm_prefix = opts.vm_prefix 2110 _vm_prefix = opts.vm_prefix
2009 2111
2010 mx.distribution('GRAAL').add_update_listener(_installGraalJarInJdks) 2112 mx.distribution('GRAAL').add_update_listener(_installGraalJarInJdks)
2011
2012 def packagejar(classpath, outputFile, mainClass=None, annotationProcessor=None, stripDebug=False):
2013 prefix = '' if mx.get_os() != 'windows' else '\\??\\' # long file name hack
2014 print "creating", outputFile
2015 filecount, totalsize = 0, 0
2016 with zipfile.ZipFile(outputFile, 'w', zipfile.ZIP_DEFLATED) as zf:
2017 manifest = "Manifest-Version: 1.0\n"
2018 if mainClass != None:
2019 manifest += "Main-Class: %s\n\n" % (mainClass)
2020 zf.writestr("META-INF/MANIFEST.MF", manifest)
2021 if annotationProcessor != None:
2022 zf.writestr("META-INF/services/javax.annotation.processing.Processor", annotationProcessor)
2023 for cp in classpath:
2024 print "+", cp
2025 if cp.endswith(".jar"):
2026 with zipfile.ZipFile(cp, 'r') as jar:
2027 for arcname in jar.namelist():
2028 if arcname.endswith('/') or arcname == 'META-INF/MANIFEST.MF' or arcname.endswith('.java') or arcname.lower().startswith("license") or arcname in [".project", ".classpath"]:
2029 continue
2030 zf.writestr(arcname, jar.read(arcname))
2031 else:
2032 for root, _, files in os.walk(cp):
2033 for f in files:
2034 fullname = os.path.join(root, f)
2035 arcname = fullname[len(cp) + 1:].replace('\\', '/')
2036 if f.endswith(".class"):
2037 zf.write(prefix + fullname, arcname)
2038
2039 for zi in zf.infolist():
2040 filecount += 1
2041 totalsize += zi.file_size
2042 print "%d files (total size: %.2f kB, jar size: %.2f kB)" % (filecount, totalsize / 1e3, os.path.getsize(outputFile) / 1e3)
2043 mx.run([mx.exe_suffix(join(mx.java().jdk, 'bin', 'pack200')), '-r'] + (['-G'] if stripDebug else []) + [outputFile])
2044 print "repacked jar size: %.2f kB" % (os.path.getsize(outputFile) / 1e3)