Mercurial > hg > graal-jvmci-8
comparison mx/mx_graal.py @ 15250:4c68a0eb69ca
mx: refactored JMH benchmark building and dependency creation out of 'jmh' into 'buildjmh' and 'makejmhdeps' respectively
author | Doug Simon <doug.simon@oracle.com> |
---|---|
date | Sat, 19 Apr 2014 00:41:04 +0200 |
parents | 5c9185d42a3a |
children | 9ff9f6643986 12f2b3baa163 |
comparison
equal
deleted
inserted
replaced
15249:54dd156e5f09 | 15250:4c68a0eb69ca |
---|---|
1400 mx.log(json.dumps(results)) | 1400 mx.log(json.dumps(results)) |
1401 if resultFile: | 1401 if resultFile: |
1402 with open(resultFile, 'w') as f: | 1402 with open(resultFile, 'w') as f: |
1403 f.write(json.dumps(results)) | 1403 f.write(json.dumps(results)) |
1404 | 1404 |
1405 def jmh(args): | 1405 def _get_jmh_path(): |
1406 """run the JMH_BENCHMARKS | 1406 path = mx.get_env('JMH_BENCHMARKS', None) |
1407 | 1407 if not path: |
1408 The benchmarks are running with the default VM. | |
1409 You can override this with an explicit option. | |
1410 For example: | |
1411 | |
1412 mx jmh -server ... | |
1413 | |
1414 Will force the benchmarks to be run with the server VM. | |
1415 """ | |
1416 | |
1417 # TODO: add option for `mvn clean package' | |
1418 | |
1419 vmArgs, benchmarksAndJsons = _extract_VM_args(args) | |
1420 | |
1421 benchmarks = [b for b in benchmarksAndJsons if not b.startswith('{')] | |
1422 jmhArgJsons = [b for b in benchmarksAndJsons if b.startswith('{')] | |
1423 | |
1424 jmhArgs = {'-rff' : join(_graal_home, 'mx', 'jmh', 'jmh.out'), '-v' : 'EXTRA' if mx._opts.verbose else 'NORMAL'} | |
1425 | |
1426 # e.g. '{"-wi" : 20}' | |
1427 for j in jmhArgJsons: | |
1428 try: | |
1429 for n, v in json.loads(j).iteritems(): | |
1430 if v is None: | |
1431 del jmhArgs[n] | |
1432 else: | |
1433 jmhArgs[n] = v | |
1434 except ValueError as e: | |
1435 mx.abort('error parsing JSON input: {}"\n{}'.format(j, e)) | |
1436 | |
1437 jmhPath = mx.get_env('JMH_BENCHMARKS', None) | |
1438 if not jmhPath: | |
1439 probe = join(dirname(_graal_home), 'java-benchmarks') | 1408 probe = join(dirname(_graal_home), 'java-benchmarks') |
1440 if exists(probe): | 1409 if exists(probe): |
1441 jmhPath = probe | 1410 path = probe |
1442 | 1411 |
1443 if not jmhPath: | 1412 if not path: |
1444 mx.abort("Please set the JMH_BENCHMARKS environment variable to point to the java-benchmarks workspace") | 1413 mx.abort("Please set the JMH_BENCHMARKS environment variable to point to the java-benchmarks workspace") |
1445 if not exists(jmhPath): | 1414 if not exists(path): |
1446 mx.abort("The directory denoted by the JMH_BENCHMARKS environment variable does not exist: " + jmhPath) | 1415 mx.abort("The directory denoted by the JMH_BENCHMARKS environment variable does not exist: " + path) |
1447 mx.log('Using benchmarks in ' + jmhPath) | 1416 return path |
1417 | |
1418 def makejmhdeps(args): | |
1419 """creates and installs Maven dependencies required by the JMH benchmarks | |
1420 | |
1421 The dependencies are specified by files named pom.mxdeps in the | |
1422 JMH directory tree. Each such file contains a list of dependencies | |
1423 defined in JSON format. For example: | |
1424 | |
1425 '[{"artifactId" : "compiler.test", "groupId" : "com.oracle.graal", "deps" : ["com.oracle.graal.compiler.test"]}]' | |
1426 | |
1427 will result in a dependency being installed in the local Maven repository | |
1428 that can be referenced in a pom.xml file as follows: | |
1429 | |
1430 <dependency> | |
1431 <groupId>com.oracle.graal</groupId> | |
1432 <artifactId>compiler.test</artifactId> | |
1433 <version>1.0-SNAPSHOT</version> | |
1434 </dependency>""" | |
1435 | |
1436 parser = ArgumentParser(prog='mx makejmhdeps') | |
1437 parser.add_argument('-s', '--settings', help='alternative path for Maven user settings file', metavar='<path>') | |
1438 parser.add_argument('-p', '--permissive', action='store_true', help='issue note instead of error if a Maven dependency cannot be built due to missing projects/libraries') | |
1439 args = parser.parse_args(args) | |
1440 | |
1441 def makejmhdep(artifactId, groupId, deps): | |
1442 graalSuite = mx.suite("graal") | |
1443 path = artifactId + '.jar' | |
1444 if args.permissive: | |
1445 for name in deps: | |
1446 if not mx.project(name, fatalIfMissing=False): | |
1447 if not mx.library(name, fatalIfMissing=False): | |
1448 mx.log('Skipping ' + groupId + '.' + artifactId + '.jar as ' + name + ' cannot be resolved') | |
1449 return | |
1450 d = mx.Distribution(graalSuite, name=artifactId, path=path, sourcesPath=path, deps=deps, excludedLibs=[]) | |
1451 d.make_archive() | |
1452 cmd = ['mvn', '-q', 'install:install-file', '-DgroupId=' + groupId, '-DartifactId=' + artifactId, | |
1453 '-Dversion=1.0-SNAPSHOT', '-Dpackaging=jar', '-Dfile=' + d.path] | |
1454 if args.settings: | |
1455 cmd = cmd + ['-s', args.settings] | |
1456 mx.run(cmd) | |
1457 os.unlink(d.path) | |
1458 | |
1459 jmhPath = _get_jmh_path() | |
1460 for root, _, filenames in os.walk(jmhPath): | |
1461 for f in [join(root, n) for n in filenames if n == 'pom.mxdeps']: | |
1462 mx.logv('[processing ' + f + ']') | |
1463 try: | |
1464 with open(f) as fp: | |
1465 for d in json.load(fp): | |
1466 artifactId = d['artifactId'] | |
1467 groupId = d['groupId'] | |
1468 deps = d['deps'] | |
1469 makejmhdep(artifactId, groupId, deps) | |
1470 except ValueError as e: | |
1471 mx.abort('Error parsing {}:\n{}'.format(f, e)) | |
1472 | |
1473 def buildjmh(args): | |
1474 """build the JMH benchmarks""" | |
1475 | |
1476 parser = ArgumentParser(prog='mx buildjmh') | |
1477 parser.add_argument('-s', '--settings', help='alternative path for Maven user settings file', metavar='<path>') | |
1478 parser.add_argument('-c', action='store_true', dest='clean', help='clean before building') | |
1479 args = parser.parse_args(args) | |
1480 | |
1481 jmhPath = _get_jmh_path() | |
1482 mx.log('JMH benchmarks: ' + jmhPath) | |
1448 | 1483 |
1449 timestamp = mx.TimeStampFile(join(_graal_home, 'mx', 'jmh', jmhPath.replace(os.sep, '_') + '.timestamp')) | 1484 timestamp = mx.TimeStampFile(join(_graal_home, 'mx', 'jmh', jmhPath.replace(os.sep, '_') + '.timestamp')) |
1450 buildJmh = False | 1485 mustBuild = args.clean |
1451 jmhTree = [] | 1486 if not mustBuild: |
1452 for root, dirnames, filenames in os.walk(jmhPath): | 1487 try: |
1453 if root == jmhPath: | 1488 hgfiles = [join(jmhPath, f) for f in subprocess.check_output(['hg', '-R', jmhPath, 'locate']).split('\n')] |
1454 for n in ['.hg', '.metadata']: | 1489 mustBuild = timestamp.isOlderThan(hgfiles) |
1455 if n in dirnames: | 1490 except: |
1456 dirnames.remove(n) | 1491 # not a Mercurial repository or hg commands are not available. |
1457 jmhTree.append(os.path.relpath(root, jmhPath) + ':') | 1492 mustBuild = True |
1458 jmhTree = jmhTree + filenames | 1493 |
1459 jmhTree.append('') | 1494 if mustBuild: |
1460 | |
1461 files = [join(root, f) for f in filenames] | |
1462 if timestamp.isOlderThan(files): | |
1463 buildJmh = True | |
1464 | |
1465 if not buildJmh: | |
1466 with open(timestamp.path) as fp: | |
1467 oldJmhTree = fp.read().split('\n') | |
1468 if oldJmhTree != jmhTree: | |
1469 import difflib | |
1470 diff = difflib.unified_diff(oldJmhTree, jmhTree) | |
1471 mx.log("Need to rebuild JMH due to change in JMH directory tree indicated by this diff:") | |
1472 mx.log('\n'.join(diff)) | |
1473 buildJmh = True | |
1474 | |
1475 if buildJmh: | |
1476 buildOutput = [] | 1495 buildOutput = [] |
1477 def _redirect(x): | 1496 def _redirect(x): |
1478 if mx._opts.verbose: | 1497 if mx._opts.verbose: |
1479 mx.log(x[:-1]) | 1498 mx.log(x[:-1]) |
1480 else: | 1499 else: |
1481 buildOutput.append(x) | 1500 buildOutput.append(x) |
1482 env = os.environ.copy() | 1501 env = os.environ.copy() |
1483 env['JAVA_HOME'] = _jdk(vmToCheck='server') | 1502 env['JAVA_HOME'] = _jdk(vmToCheck='server') |
1484 env['MAVEN_OPTS'] = '-server' | 1503 env['MAVEN_OPTS'] = '-server' |
1485 mx.log("Building benchmarks...") | 1504 mx.log("Building benchmarks...") |
1486 retcode = mx.run(['mvn', 'package'], cwd=jmhPath, out=_redirect, env=env, nonZeroIsFatal=False) | 1505 makejmhdeps(['-p'] + (['-s', args.settings] if args.settings else [])) |
1506 cmd = ['mvn'] | |
1507 if args.settings: | |
1508 cmd = cmd + ['-s', args.settings] | |
1509 if args.clean: | |
1510 cmd.append('clean') | |
1511 cmd.append('package') | |
1512 retcode = mx.run(cmd, cwd=jmhPath, out=_redirect, env=env, nonZeroIsFatal=False) | |
1487 if retcode != 0: | 1513 if retcode != 0: |
1488 mx.log(''.join(buildOutput)) | 1514 mx.log(''.join(buildOutput)) |
1489 mx.abort(retcode) | 1515 mx.abort(retcode) |
1490 timestamp.touch() | 1516 timestamp.touch() |
1491 with open(timestamp.path, 'w') as fp: | 1517 else: |
1492 fp.write('\n'.join(jmhTree)) | 1518 mx.logv('[all Mercurial controlled files in ' + jmhPath + ' are older than ' + timestamp.path + ' - skipping build]') |
1519 | |
1520 def jmh(args): | |
1521 """run the JMH benchmarks | |
1522 | |
1523 This command respects the standard --vm and --vmbuild options | |
1524 for choosing which VM to run the benchmarks with.""" | |
1525 if '-h' in args: | |
1526 mx.help_(['jmh']) | |
1527 mx.abort(1) | |
1528 | |
1529 vmArgs, benchmarksAndJsons = _extract_VM_args(args) | |
1530 | |
1531 benchmarks = [b for b in benchmarksAndJsons if not b.startswith('{')] | |
1532 jmhArgJsons = [b for b in benchmarksAndJsons if b.startswith('{')] | |
1533 | |
1534 jmhArgs = {'-rff' : join(_graal_home, 'mx', 'jmh', 'jmh.out'), '-v' : 'EXTRA' if mx._opts.verbose else 'NORMAL'} | |
1535 | |
1536 # e.g. '{"-wi" : 20}' | |
1537 for j in jmhArgJsons: | |
1538 try: | |
1539 for n, v in json.loads(j).iteritems(): | |
1540 if v is None: | |
1541 del jmhArgs[n] | |
1542 else: | |
1543 jmhArgs[n] = v | |
1544 except ValueError as e: | |
1545 mx.abort('error parsing JSON input: {}\n{}'.format(j, e)) | |
1546 | |
1547 jmhPath = _get_jmh_path() | |
1548 mx.log('Using benchmarks in ' + jmhPath) | |
1493 | 1549 |
1494 matchedSuites = set() | 1550 matchedSuites = set() |
1495 numBench = [0] | 1551 numBench = [0] |
1496 for micros in os.listdir(jmhPath): | 1552 for micros in os.listdir(jmhPath): |
1497 absoluteMicro = os.path.join(jmhPath, micros) | 1553 absoluteMicro = os.path.join(jmhPath, micros) |
1501 mx.logv('JMH: ignored ' + absoluteMicro + " because it doesn't start with 'micros-'") | 1557 mx.logv('JMH: ignored ' + absoluteMicro + " because it doesn't start with 'micros-'") |
1502 continue | 1558 continue |
1503 | 1559 |
1504 microJar = os.path.join(absoluteMicro, "target", "microbenchmarks.jar") | 1560 microJar = os.path.join(absoluteMicro, "target", "microbenchmarks.jar") |
1505 if not exists(microJar): | 1561 if not exists(microJar): |
1506 mx.logv('JMH: ignored ' + absoluteMicro + " because it doesn't contain the expected jar file ('" + microJar + "')") | 1562 mx.abort('Missing ' + microJar + ' - please run "mx buildjmh"') |
1507 continue | |
1508 if benchmarks: | 1563 if benchmarks: |
1509 def _addBenchmark(x): | 1564 def _addBenchmark(x): |
1510 if x.startswith("Benchmark:"): | 1565 if x.startswith("Benchmark:"): |
1511 return | 1566 return |
1512 match = False | 1567 match = False |
1541 for k, v in jmhArgs.iteritems(): | 1596 for k, v in jmhArgs.iteritems(): |
1542 javaArgs.append(k) | 1597 javaArgs.append(k) |
1543 if len(str(v)): | 1598 if len(str(v)): |
1544 javaArgs.append(str(v)) | 1599 javaArgs.append(str(v)) |
1545 mx.run_java(javaArgs + regex, addDefaultArgs=False, cwd=jmhPath) | 1600 mx.run_java(javaArgs + regex, addDefaultArgs=False, cwd=jmhPath) |
1546 | |
1547 | 1601 |
1548 def specjvm2008(args): | 1602 def specjvm2008(args): |
1549 """run one or more SPECjvm2008 benchmarks""" | 1603 """run one or more SPECjvm2008 benchmarks""" |
1550 | 1604 |
1551 def launcher(bm, harnessArgs, extraVmOpts): | 1605 def launcher(bm, harnessArgs, extraVmOpts): |
1872 return len(failures) | 1926 return len(failures) |
1873 | 1927 |
1874 def mx_init(suite): | 1928 def mx_init(suite): |
1875 commands = { | 1929 commands = { |
1876 'build': [build, ''], | 1930 'build': [build, ''], |
1931 'buildjmh': [buildjmh, '[-options]'], | |
1877 'buildvars': [buildvars, ''], | 1932 'buildvars': [buildvars, ''], |
1878 'buildvms': [buildvms, '[-options]'], | 1933 'buildvms': [buildvms, '[-options]'], |
1879 'c1visualizer' : [c1visualizer, ''], | 1934 'c1visualizer' : [c1visualizer, ''], |
1880 'checkheaders': [checkheaders, ''], | 1935 'checkheaders': [checkheaders, ''], |
1881 'clean': [clean, ''], | 1936 'clean': [clean, ''], |
1893 'specjbb2005': [specjbb2005, '[VM options] [-- [SPECjbb2005 options]]'], | 1948 'specjbb2005': [specjbb2005, '[VM options] [-- [SPECjbb2005 options]]'], |
1894 'gate' : [gate, '[-options]'], | 1949 'gate' : [gate, '[-options]'], |
1895 'bench' : [bench, '[-resultfile file] [all(default)|dacapo|specjvm2008|bootstrap]'], | 1950 'bench' : [bench, '[-resultfile file] [all(default)|dacapo|specjvm2008|bootstrap]'], |
1896 'unittest' : [unittest, '[unittest options] [--] [VM options] [filters...]', _unittestHelpSuffix], | 1951 'unittest' : [unittest, '[unittest options] [--] [VM options] [filters...]', _unittestHelpSuffix], |
1897 'longunittest' : [longunittest, '[unittest options] [--] [VM options] [filters...]', _unittestHelpSuffix], | 1952 'longunittest' : [longunittest, '[unittest options] [--] [VM options] [filters...]', _unittestHelpSuffix], |
1953 'makejmhdeps' : [makejmhdeps, ''], | |
1898 'shortunittest' : [shortunittest, '[unittest options] [--] [VM options] [filters...]', _unittestHelpSuffix], | 1954 'shortunittest' : [shortunittest, '[unittest options] [--] [VM options] [filters...]', _unittestHelpSuffix], |
1899 'jacocoreport' : [jacocoreport, '[output directory]'], | 1955 'jacocoreport' : [jacocoreport, '[output directory]'], |
1900 'site' : [site, '[-options]'], | 1956 'site' : [site, '[-options]'], |
1901 'vm': [vm, '[-options] class [args...]'], | 1957 'vm': [vm, '[-options] class [args...]'], |
1902 'vmg': [vmg, '[-options] class [args...]'], | 1958 'vmg': [vmg, '[-options] class [args...]'], |