truffle: mx/mx_graal.py comparison

comparison mx/mx_graal.py @ 15451:3774b6f4319b

Merge with 2f684eda1938cc92a72a35461c8d00f1871fe389

author	Michael Van De Vanter <michael.van.de.vanter@oracle.com>
date	Tue, 29 Apr 2014 12:43:27 -0700
parents	5947bb02474f
children	f73fc9309f12

comparison

equal deleted inserted replaced

-:be0c151d912b
+:3774b6f4319b
 # or visit www.oracle.com if you need additional information or have any
 # questions.
 #
 # ----------------------------------------------------------------------------------------------------
-import os, sys, shutil, zipfile, tempfile, re, time, datetime, platform, subprocess, multiprocessing, StringIO
+import os, sys, shutil, zipfile, tarfile, tempfile, re, time, datetime, platform, subprocess, multiprocessing, StringIO, socket
 from os.path import join, exists, dirname, basename, getmtime
 from argparse import ArgumentParser, RawDescriptionHelpFormatter, REMAINDER
 from outputparser import OutputParser, ValuesMatcher
 import mx
 import xml.dom.minidom
 import sanitycheck
 import itertools
 import json, textwrap
+import fnmatch
 # This works because when mx loads this file, it makes sure __file__ gets an absolute path
 _graal_home = dirname(dirname(__file__))
 """ Used to distinguish an exported GraalVM (see 'mx export'). """
 rmIfExists(join(_graal_home, 'build-nograal'))
 rmIfExists(_jdksDir())
 rmIfExists(mx.distribution('GRAAL').path)
 def export(args):
-"""create a GraalVM zip file for distribution"""
+"""create archives of builds split by vmbuild and vm"""
 parser = ArgumentParser(prog='mx export')
-parser.add_argument('--omit-vm-build', action='store_false', dest='vmbuild', help='omit VM build step')
-parser.add_argument('--omit-dist-init', action='store_false', dest='distInit', help='omit class files and IDE configurations from distribution')
-parser.add_argument('zipfile', nargs=REMAINDER, metavar='zipfile')
 args = parser.parse_args(args)
-tmp = tempfile.mkdtemp(prefix='tmp', dir=_graal_home)
+# collect data about export
-if args.vmbuild:
+infos = dict()
-# Make sure the product VM binary is up to date
+infos['timestamp'] = time.time()
-with VM(vmbuild='product'):
-build([])
+hgcfg = mx.HgConfig()
+hgcfg.check()
-mx.log('Copying Java sources and mx files...')
+infos['revision'] = hgcfg.tip('.') + ('+' if hgcfg.isDirty('.') else '')
-mx.run(('hg archive -I graal -I mx -I mxtool -I mx.sh ' + tmp).split())
+# TODO: infos['repository']
-# Copy the GraalVM JDK
+infos['jdkversion'] = str(mx.java().version)
-mx.log('Copying GraalVM JDK...')
-src = _jdk()
+infos['architecture'] = _arch()
-dst = join(tmp, basename(src))
+infos['platform'] = mx.get_os()
-shutil.copytree(src, dst)
-zfName = join(_graal_home, 'graalvm-' + mx.get_os() + '.zip')
+if mx.get_os != 'windows':
-zf = zipfile.ZipFile(zfName, 'w')
+pass
-for root, _, files in os.walk(tmp):
+# infos['ccompiler']
-for f in files:
+# infos['linker']
-name = join(root, f)
-arcname = name[len(tmp) + 1:]
+infos['hostname'] = socket.gethostname()
-zf.write(join(tmp, name), arcname)
+def _writeJson(suffix, properties):
-# create class files and IDE configurations
+d = infos.copy()
-if args.distInit:
+for k, v in properties.iteritems():
-mx.log('Creating class files...')
+assert not d.has_key(k)
-mx.run('mx build'.split(), cwd=tmp)
+d[k] = v
-mx.log('Creating IDE configurations...')
-mx.run('mx ideinit'.split(), cwd=tmp)
+jsonFileName = 'export-' + suffix + '.json'
+with open(jsonFileName, 'w') as f:
-# clean up temp directory
+print >> f, json.dumps(d)
-mx.log('Cleaning up...')
+return jsonFileName
-shutil.rmtree(tmp)
-mx.log('Created distribution in ' + zfName)
+def _genFileName(archivtype, middle):
+idPrefix = infos['revision'] + '_'
+idSuffix = '.tar.gz'
+return join(_graal_home, "graalvm_" + archivtype + "_"  + idPrefix + middle + idSuffix)
+def _genFileArchPlatformName(archivtype, middle):
+return _genFileName(archivtype, infos['platform'] + '_' + infos['architecture'] + '_' + middle)
+# archive different build types of hotspot
+for vmBuild in _vmbuildChoices:
+jdkpath = join(_jdksDir(), vmBuild)
+if not exists(jdkpath):
+mx.logv("skipping " + vmBuild)
+continue
+tarName = _genFileArchPlatformName('basejdk', vmBuild)
+mx.logv("creating basejdk " + tarName)
+vmSet = set()
+with tarfile.open(tarName, 'w:gz') as tar:
+for root, _, files in os.walk(jdkpath):
+if basename(root) in _vmChoices.keys():
+# TODO: add some assert to check path assumption
+vmSet.add(root)
+continue
+for f in files:
+name = join(root, f)
+# print name
+tar.add(name, name)
+n = _writeJson("basejdk-" + vmBuild, {'vmbuild' : vmBuild})
+tar.add(n, n)
+# create a separate archive for each VM
+for vm in vmSet:
+bVm = basename(vm)
+vmTarName = _genFileArchPlatformName('vm', vmBuild + '_' + bVm)
+mx.logv("creating vm " + vmTarName)
+debugFiles = set()
+with tarfile.open(vmTarName, 'w:gz') as tar:
+for root, _, files in os.walk(vm):
+for f in files:
+# TODO: mac, windows, solaris?
+if any(map(f.endswith, [".debuginfo"])):
+debugFiles.add(f)
+else:
+name = join(root, f)
+# print name
+tar.add(name, name)
+n = _writeJson("vm-" + vmBuild + "-" + bVm, {'vmbuild' : vmBuild, 'vm' : bVm})
+tar.add(n, n)
+if len(debugFiles) > 0:
+debugTarName = _genFileArchPlatformName('debugfilesvm', vmBuild + '_' + bVm)
+mx.logv("creating debugfilesvm " + debugTarName)
+with tarfile.open(debugTarName, 'w:gz') as tar:
+for f in debugFiles:
+name = join(root, f)
+# print name
+tar.add(name, name)
+n = _writeJson("debugfilesvm-" + vmBuild + "-" + bVm, {'vmbuild' : vmBuild, 'vm' : bVm})
+tar.add(n, n)
+# graal directory
+graalDirTarName = _genFileName('classfiles', 'javac')
+mx.logv("creating graal " + graalDirTarName)
+with tarfile.open(graalDirTarName, 'w:gz') as tar:
+for root, _, files in os.walk("graal"):
+for f in [f for f in files if not f.endswith('.java')]:
+name = join(root, f)
+# print name
+tar.add(name, name)
+n = _writeJson("graal", {'javacompiler' : 'javac'})
+tar.add(n, n)
 def _run_benchmark(args, availableBenchmarks, runBenchmark):
 vmOpts, benchmarksAndOptions = _extract_VM_args(args, useDoubleDash=availableBenchmarks is None)
 if not found:
 mx.log('warning: no tests matched by substring "' + t)
 projectscp = mx.classpath(projs)
 if whitelist:
-classes = list(set(classes) & set(whitelist))
+classes = [c for c in classes if any((glob.match(c) for glob in whitelist))]
 if len(classes) != 0:
 f_testfile = open(testfile, 'w')
 for c in classes:
 f_testfile.write(c + '\n')
 os.remove(testfile)
 _unittestHelpSuffix = """
 Unittest options:
---short-only           run short testcases only
+--whitelist            run only testcases which are included
---long-only            run long testcases only
+in the given whitelist
---baseline-whitelist   run only testcases which are known to
-work with the baseline compiler
 To avoid conflicts with VM options '--' can be used as delimiter.
 If filters are supplied, only tests whose fully qualified name
 includes a filter as a substring are run.
 description='run the JUnit tests',
 add_help=False,
 formatter_class=RawDescriptionHelpFormatter,
 epilog=_unittestHelpSuffix,
 )
-group = parser.add_mutually_exclusive_group()
+parser.add_argument('--whitelist', help='run testcases specified in whitelist only', metavar='<path>')
-group.add_argument('--short-only', action='store_true', help='run short testcases only')
-group.add_argument('--long-only', action='store_true', help='run long testcases only')
-parser.add_argument('--baseline-whitelist', action='store_true', help='run baseline testcases only')
 ut_args = []
 delimiter = False
 # check for delimiter
 while len(args) > 0:
 else:
 # parse all know arguments
 parsed_args, args = parser.parse_known_args(ut_args)
 whitelist = None
-if parsed_args.baseline_whitelist:
+if parsed_args.whitelist:
-baseline_whitelist_file = 'test/baseline_whitelist.txt'
 try:
-with open(join(_graal_home, baseline_whitelist_file)) as fp:
+with open(join(_graal_home, parsed_args.whitelist)) as fp:
-whitelist = [l.rstrip() for l in fp.readlines()]
+whitelist = [re.compile(fnmatch.translate(l.rstrip())) for l in fp.readlines() if not l.startswith('#')]
 except IOError:
-mx.log('warning: could not read baseline whitelist: ' + baseline_whitelist_file)
+mx.log('warning: could not read whitelist: ' + parsed_args.whitelist)
-if parsed_args.long_only:
+_unittest(args, ['@Test', '@Parameters'], whitelist=whitelist)
-annotations = ['@LongTest', '@Parameters']
-elif parsed_args.short_only:
-annotations = ['@Test']
-else:
-annotations = ['@Test', '@LongTest', '@Parameters']
-_unittest(args, annotations, whitelist=whitelist)
 def shortunittest(args):
-"""alias for 'unittest --short-only'{0}"""
+"""alias for 'unittest --whitelist test/whitelist_shortunittest.txt'{0}"""
-args.insert(0, '--short-only')
+args = ['--whitelist', 'test/whitelist_shortunittest.txt'] + args
-unittest(args)
-def longunittest(args):
-"""alias for 'unittest --long-only'{0}"""
-args.insert(0, '--long-only')
 unittest(args)
 def buildvms(args):
 """build one or more VMs in various configurations"""
 with VM('server', 'product'):  # hosted mode
 t = Task('UnitTests:hosted-product')
 unittest([])
 tasks.append(t.stop())
+with VM('server', 'product'):  # hosted mode
+t = Task('UnitTests-BaselineCompiler:hosted-product')
+unittest(['--whitelist', 'test/whitelist_baseline.txt', '-G:+UseBaselineCompiler'])
+tasks.append(t.stop())
 for vmbuild in ['fastdebug', 'product']:
 for test in sanitycheck.getDacapos(level=sanitycheck.SanityCheckLevel.Gate, gateBuildLevel=vmbuild) + sanitycheck.getScalaDacapos(level=sanitycheck.SanityCheckLevel.Gate, gateBuildLevel=vmbuild):
 t = Task(str(test) + ':' + vmbuild)
 if not test.test('graal'):
 t.abort(test.name + ' Failed')
 executable = join(libpath, 'c1visualizer', 'bin', 'c1visualizer.exe')
 else:
 executable = join(libpath, 'c1visualizer', 'bin', 'c1visualizer')
 archive = join(libpath, 'c1visualizer_2014-04-22.zip')
-if not exists(executable):
+if not exists(executable) or not exists(archive):
 if not exists(archive):
 mx.download(archive, ['https://java.net/downloads/c1visualizer/c1visualizer_2014-04-22.zip'])
 zf = zipfile.ZipFile(archive, 'r')
 zf.extractall(libpath)
 def trufflejar(args=None):
 """make truffle.jar"""
 # Test with the built classes
-_unittest(["com.oracle.truffle.api.test", "com.oracle.truffle.api.dsl.test"], ['@Test', '@LongTest', '@Parameters'])
+_unittest(["com.oracle.truffle.api.test", "com.oracle.truffle.api.dsl.test"], ['@Test', '@Parameters'])
 # We use the DSL processor as the starting point for the classpath - this
 # therefore includes the DSL processor, the DSL and the API.
 packagejar(mx.classpath("com.oracle.truffle.dsl.processor").split(os.pathsep), "truffle.jar", None, "com.oracle.truffle.dsl.processor.TruffleProcessor")
 # Test with the JAR
-_unittest(["com.oracle.truffle.api.test", "com.oracle.truffle.api.dsl.test"], ['@Test', '@LongTest', '@Parameters'], "truffle.jar:")
+_unittest(["com.oracle.truffle.api.test", "com.oracle.truffle.api.dsl.test"], ['@Test', '@Parameters'], "truffle.jar:")
 def isGraalEnabled(vm):
 return vm != 'original' and not vm.endswith('nograal')
 'specjbb2013': [specjbb2013, '[VM options] [-- [SPECjbb2013 options]]'],
 'specjbb2005': [specjbb2005, '[VM options] [-- [SPECjbb2005 options]]'],
 'gate' : [gate, '[-options]'],
 'bench' : [bench, '[-resultfile file] [all(default)|dacapo|specjvm2008|bootstrap]'],
 'unittest' : [unittest, '[unittest options] [--] [VM options] [filters...]', _unittestHelpSuffix],
-'longunittest' : [longunittest, '[unittest options] [--] [VM options] [filters...]', _unittestHelpSuffix],
 'makejmhdeps' : [makejmhdeps, ''],
 'shortunittest' : [shortunittest, '[unittest options] [--] [VM options] [filters...]', _unittestHelpSuffix],
 'jacocoreport' : [jacocoreport, '[output directory]'],
 'site' : [site, '[-options]'],
 'vm': [vm, '[-options] class [args...]'],

Mercurial > hg > truffle

comparison mx/mx_graal.py @ 15451:3774b6f4319b