--- a/mx/mx_graal.py	Fri Apr 25 16:21:07 2014 +0200
+++ b/mx/mx_graal.py	Mon Apr 28 11:18:15 2014 +0200
@@ -28,13 +28,14 @@
 
 import os, sys, shutil, zipfile, tempfile, re, time, datetime, platform, subprocess, multiprocessing, StringIO
 from os.path import join, exists, dirname, basename, getmtime
-from argparse import ArgumentParser, REMAINDER
+from argparse import ArgumentParser, RawDescriptionHelpFormatter, REMAINDER
 from outputparser import OutputParser, ValuesMatcher
 import mx
 import xml.dom.minidom
 import sanitycheck
 import itertools
 import json, textwrap
+import fnmatch
 
 # This works because when mx loads this file, it makes sure __file__ gets an absolute path
 _graal_home = dirname(dirname(__file__))
@@ -828,7 +829,7 @@
     else:
         return [], args
 
-def _run_tests(args, harness, annotations, testfile):
+def _run_tests(args, harness, annotations, testfile, whitelist):
 
 
     vmArgs, tests = _extract_VM_args(args)
@@ -860,6 +861,9 @@
                 mx.log('warning: no tests matched by substring "' + t)
         projectscp = mx.classpath(projs)
 
+    if whitelist:
+        classes = [c for c in classes if any((glob.match(c) for glob in whitelist))]
+
     if len(classes) != 0:
         f_testfile = open(testfile, 'w')
         for c in classes:
@@ -867,7 +871,7 @@
         f_testfile.close()
         harness(projectscp, vmArgs)
 
-def _unittest(args, annotations, prefixcp=""):
+def _unittest(args, annotations, prefixcp="", whitelist=None):
     mxdir = dirname(__file__)
     name = 'JUnitWrapper'
     javaSource = join(mxdir, name + '.java')
@@ -894,12 +898,18 @@
             vm(prefixArgs + vmArgs + ['-cp', prefixcp + projectscp + os.pathsep + mxdir, name] + [testfile])
 
     try:
-        _run_tests(args, harness, annotations, testfile)
+        _run_tests(args, harness, annotations, testfile, whitelist)
     finally:
         if os.environ.get('MX_TESTFILE') is None:
             os.remove(testfile)
 
 _unittestHelpSuffix = """
+    Unittest options:
+
+      --whitelist            run only testcases which are included
+                             in the given whitelist
+
+    To avoid conflicts with VM options '--' can be used as delimiter.
 
     If filters are supplied, only tests whose fully qualified name
     includes a filter as a substring are run.
@@ -928,17 +938,46 @@
 def unittest(args):
     """run the JUnit tests (all testcases){0}"""
 
-    _unittest(args, ['@Test', '@LongTest', '@Parameters'])
+    parser = ArgumentParser(prog='mx unittest',
+          description='run the JUnit tests',
+          add_help=False,
+          formatter_class=RawDescriptionHelpFormatter,
+          epilog=_unittestHelpSuffix,
+        )
+    parser.add_argument('--whitelist', help='run testcases specified in whitelist only', metavar='<path>')
+
+    ut_args = []
+    delimiter = False
+    # check for delimiter
+    while len(args) > 0:
+        arg = args.pop(0)
+        if arg == '--':
+            delimiter = True
+            break
+        ut_args.append(arg)
+
+    if delimiter:
+        # all arguments before '--' must be recognized
+        parsed_args = parser.parse_args(ut_args)
+    else:
+        # parse all know arguments
+        parsed_args, args = parser.parse_known_args(ut_args)
+
+    whitelist = None
+    if parsed_args.whitelist:
+        try:
+            with open(join(_graal_home, parsed_args.whitelist)) as fp:
+                whitelist = [re.compile(fnmatch.translate(l.rstrip())) for l in fp.readlines() if not l.startswith('#')]
+        except IOError:
+            mx.log('warning: could not read whitelist: ' + parsed_args.whitelist)
+
+    _unittest(args, ['@Test', '@Parameters'], whitelist=whitelist)
 
 def shortunittest(args):
-    """run the JUnit tests (short testcases only){0}"""
-
-    _unittest(args, ['@Test'])
+    """alias for 'unittest --whitelist test/whitelist_shortunittest.txt'{0}"""
 
-def longunittest(args):
-    """run the JUnit tests (long testcases only){0}"""
-
-    _unittest(args, ['@LongTest', '@Parameters'])
+    args = ['--whitelist', 'test/whitelist_shortunittest.txt'] + args
+    unittest(args)
 
 def buildvms(args):
     """build one or more VMs in various configurations"""
@@ -1047,6 +1086,11 @@
         unittest([])
         tasks.append(t.stop())
 
+    with VM('server', 'product'):  # hosted mode
+        t = Task('UnitTests-BaselineCompiler:hosted-product')
+        unittest(['--whitelist', 'test/whitelist_baseline.txt', '-G:+UseBaselineCompiler'])
+        tasks.append(t.stop())
+
     for vmbuild in ['fastdebug', 'product']:
         for test in sanitycheck.getDacapos(level=sanitycheck.SanityCheckLevel.Gate, gateBuildLevel=vmbuild) + sanitycheck.getScalaDacapos(level=sanitycheck.SanityCheckLevel.Gate, gateBuildLevel=vmbuild):
             t = Task(str(test) + ':' + vmbuild)
@@ -1248,10 +1292,10 @@
     else:
         executable = join(libpath, 'c1visualizer', 'bin', 'c1visualizer')
 
-    archive = join(libpath, 'c1visualizer.zip')
-    if not exists(executable):
+    archive = join(libpath, 'c1visualizer_2014-04-22.zip')
+    if not exists(executable) or not exists(archive):
         if not exists(archive):
-            mx.download(archive, ['https://java.net/downloads/c1visualizer/c1visualizer.zip'])
+            mx.download(archive, ['https://java.net/downloads/c1visualizer/c1visualizer_2014-04-22.zip'])
         zf = zipfile.ZipFile(archive, 'r')
         zf.extractall(libpath)
 
@@ -1345,17 +1389,140 @@
         with open(resultFile, 'w') as f:
             f.write(json.dumps(results))
 
+def _get_jmh_path():
+    path = mx.get_env('JMH_BENCHMARKS', None)
+    if not path:
+        probe = join(dirname(_graal_home), 'java-benchmarks')
+        if exists(probe):
+            path = probe
+
+    if not path:
+        mx.abort("Please set the JMH_BENCHMARKS environment variable to point to the java-benchmarks workspace")
+    if not exists(path):
+        mx.abort("The directory denoted by the JMH_BENCHMARKS environment variable does not exist: " + path)
+    return path
+
+def makejmhdeps(args):
+    """creates and installs Maven dependencies required by the JMH benchmarks
+
+    The dependencies are specified by files named pom.mxdeps in the
+    JMH directory tree. Each such file contains a list of dependencies
+    defined in JSON format. For example:
+
+    '[{"artifactId" : "compiler.test", "groupId" : "com.oracle.graal", "deps" : ["com.oracle.graal.compiler.test"]}]'
+
+    will result in a dependency being installed in the local Maven repository
+    that can be referenced in a pom.xml file as follows:
+
+          <dependency>
+            <groupId>com.oracle.graal</groupId>
+            <artifactId>compiler.test</artifactId>
+            <version>1.0-SNAPSHOT</version>
+          </dependency>"""
+
+    parser = ArgumentParser(prog='mx makejmhdeps')
+    parser.add_argument('-s', '--settings', help='alternative path for Maven user settings file', metavar='<path>')
+    parser.add_argument('-p', '--permissive', action='store_true', help='issue note instead of error if a Maven dependency cannot be built due to missing projects/libraries')
+    args = parser.parse_args(args)
+
+    def makejmhdep(artifactId, groupId, deps):
+        graalSuite = mx.suite("graal")
+        path = artifactId + '.jar'
+        if args.permissive:
+            for name in deps:
+                if not mx.project(name, fatalIfMissing=False):
+                    if not mx.library(name, fatalIfMissing=False):
+                        mx.log('Skipping ' + groupId + '.' + artifactId + '.jar as ' + name + ' cannot be resolved')
+                        return
+        d = mx.Distribution(graalSuite, name=artifactId, path=path, sourcesPath=path, deps=deps, excludedLibs=[])
+        d.make_archive()
+        cmd = ['mvn', 'install:install-file', '-DgroupId=' + groupId, '-DartifactId=' + artifactId,
+               '-Dversion=1.0-SNAPSHOT', '-Dpackaging=jar', '-Dfile=' + d.path]
+        if not mx._opts.verbose:
+            cmd.append('-q')
+        if args.settings:
+            cmd = cmd + ['-s', args.settings]
+        mx.run(cmd)
+        os.unlink(d.path)
+
+    jmhPath = _get_jmh_path()
+    for root, _, filenames in os.walk(jmhPath):
+        for f in [join(root, n) for n in filenames if n == 'pom.mxdeps']:
+            mx.logv('[processing ' + f + ']')
+            try:
+                with open(f) as fp:
+                    for d in json.load(fp):
+                        artifactId = d['artifactId']
+                        groupId = d['groupId']
+                        deps = d['deps']
+                        makejmhdep(artifactId, groupId, deps)
+            except ValueError as e:
+                mx.abort('Error parsing {}:\n{}'.format(f, e))
+
+def buildjmh(args):
+    """build the JMH benchmarks"""
+
+    parser = ArgumentParser(prog='mx buildjmh')
+    parser.add_argument('-s', '--settings', help='alternative path for Maven user settings file', metavar='<path>')
+    parser.add_argument('-c', action='store_true', dest='clean', help='clean before building')
+    args = parser.parse_args(args)
+
+    jmhPath = _get_jmh_path()
+    mx.log('JMH benchmarks: ' + jmhPath)
+
+    # Ensure the mx injected dependencies are up to date
+    makejmhdeps(['-p'] + (['-s', args.settings] if args.settings else []))
+
+    timestamp = mx.TimeStampFile(join(_graal_home, 'mx', 'jmh', jmhPath.replace(os.sep, '_') + '.timestamp'))
+    mustBuild = args.clean
+    if not mustBuild:
+        try:
+            hgfiles = [join(jmhPath, f) for f in subprocess.check_output(['hg', '-R', jmhPath, 'locate']).split('\n')]
+            mustBuild = timestamp.isOlderThan(hgfiles)
+        except:
+            # not a Mercurial repository or hg commands are not available.
+            mustBuild = True
+
+    if mustBuild:
+        buildOutput = []
+        def _redirect(x):
+            if mx._opts.verbose:
+                mx.log(x[:-1])
+            else:
+                buildOutput.append(x)
+        env = os.environ.copy()
+        env['JAVA_HOME'] = _jdk(vmToCheck='server')
+        env['MAVEN_OPTS'] = '-server'
+        mx.log("Building benchmarks...")
+        cmd = ['mvn']
+        if args.settings:
+            cmd = cmd + ['-s', args.settings]
+        if args.clean:
+            cmd.append('clean')
+        cmd.append('package')
+        retcode = mx.run(cmd, cwd=jmhPath, out=_redirect, env=env, nonZeroIsFatal=False)
+        if retcode != 0:
+            mx.log(''.join(buildOutput))
+            mx.abort(retcode)
+        timestamp.touch()
+    else:
+        mx.logv('[all Mercurial controlled files in ' + jmhPath + ' are older than ' + timestamp.path + ' - skipping build]')
+
 def jmh(args):
-    """run the JMH_BENCHMARKS"""
+    """run the JMH benchmarks
 
-    # TODO: add option for `mvn clean package'
+    This command respects the standard --vm and --vmbuild options
+    for choosing which VM to run the benchmarks with."""
+    if '-h' in args:
+        mx.help_(['jmh'])
+        mx.abort(1)
 
     vmArgs, benchmarksAndJsons = _extract_VM_args(args)
 
     benchmarks = [b for b in benchmarksAndJsons if not b.startswith('{')]
     jmhArgJsons = [b for b in benchmarksAndJsons if b.startswith('{')]
 
-    jmhArgs = {'-v' : 'EXTRA' if mx._opts.verbose else 'NORMAL'}
+    jmhArgs = {'-rff' : join(_graal_home, 'mx', 'jmh', 'jmh.out'), '-v' : 'EXTRA' if mx._opts.verbose else 'NORMAL'}
 
     # e.g. '{"-wi" : 20}'
     for j in jmhArgJsons:
@@ -1366,21 +1533,10 @@
                 else:
                     jmhArgs[n] = v
         except ValueError as e:
-            mx.abort('error parsing JSON input: {}"\n{}'.format(j, e))
-
-    jmhPath = mx.get_env('JMH_BENCHMARKS', None)
-    if not jmhPath or not exists(jmhPath):
-        mx.abort("$JMH_BENCHMARKS not properly defined: " + str(jmhPath))
+            mx.abort('error parsing JSON input: {}\n{}'.format(j, e))
 
-    def _blackhole(x):
-        mx.logv(x[:-1])
-
-
-    env = os.environ.copy()
-    env['JAVA_HOME'] = _jdk(vmToCheck='graal')
-    env['MAVEN_OPTS'] = '-graal'
-    mx.log("Building benchmarks...")
-    mx.run(['mvn', 'package'], cwd=jmhPath, out=_blackhole, env=env)
+    jmhPath = _get_jmh_path()
+    mx.log('Using benchmarks in ' + jmhPath)
 
     matchedSuites = set()
     numBench = [0]
@@ -1394,8 +1550,7 @@
 
         microJar = os.path.join(absoluteMicro, "target", "microbenchmarks.jar")
         if not exists(microJar):
-            mx.logv('JMH: ignored ' + absoluteMicro + " because it doesn't contain the expected jar file ('" + microJar + "')")
-            continue
+            mx.abort('Missing ' + microJar + ' - please run "mx buildjmh"')
         if benchmarks:
             def _addBenchmark(x):
                 if x.startswith("Benchmark:"):
@@ -1435,7 +1590,6 @@
                 javaArgs.append(str(v))
         mx.run_java(javaArgs + regex, addDefaultArgs=False, cwd=jmhPath)
 
-
 def specjvm2008(args):
     """run one or more SPECjvm2008 benchmarks"""
 
@@ -1556,14 +1710,14 @@
     """make truffle.jar"""
 
     # Test with the built classes
-    _unittest(["com.oracle.truffle.api.test", "com.oracle.truffle.api.dsl.test"], ['@Test', '@LongTest', '@Parameters'])
+    _unittest(["com.oracle.truffle.api.test", "com.oracle.truffle.api.dsl.test"], ['@Test', '@Parameters'])
 
     # We use the DSL processor as the starting point for the classpath - this
     # therefore includes the DSL processor, the DSL and the API.
     packagejar(mx.classpath("com.oracle.truffle.dsl.processor").split(os.pathsep), "truffle.jar", None, "com.oracle.truffle.dsl.processor.TruffleProcessor")
 
     # Test with the JAR
-    _unittest(["com.oracle.truffle.api.test", "com.oracle.truffle.api.dsl.test"], ['@Test', '@LongTest', '@Parameters'], "truffle.jar:")
+    _unittest(["com.oracle.truffle.api.test", "com.oracle.truffle.api.dsl.test"], ['@Test', '@Parameters'], "truffle.jar:")
 
 
 def isGraalEnabled(vm):
@@ -1765,6 +1919,7 @@
 def mx_init(suite):
     commands = {
         'build': [build, ''],
+        'buildjmh': [buildjmh, '[-options]'],
         'buildvars': [buildvars, ''],
         'buildvms': [buildvms, '[-options]'],
         'c1visualizer' : [c1visualizer, ''],
@@ -1784,9 +1939,9 @@
         'specjbb2005': [specjbb2005, '[VM options] [-- [SPECjbb2005 options]]'],
         'gate' : [gate, '[-options]'],
         'bench' : [bench, '[-resultfile file] [all(default)|dacapo|specjvm2008|bootstrap]'],
-        'unittest' : [unittest, '[VM options] [filters...]', _unittestHelpSuffix],
-        'longunittest' : [longunittest, '[VM options] [filters...]', _unittestHelpSuffix],
-        'shortunittest' : [shortunittest, '[VM options] [filters...]', _unittestHelpSuffix],
+        'unittest' : [unittest, '[unittest options] [--] [VM options] [filters...]', _unittestHelpSuffix],
+        'makejmhdeps' : [makejmhdeps, ''],
+        'shortunittest' : [shortunittest, '[unittest options] [--] [VM options] [filters...]', _unittestHelpSuffix],
         'jacocoreport' : [jacocoreport, '[output directory]'],
         'site' : [site, '[-options]'],
         'vm': [vm, '[-options] class [args...]'],