Mercurial > hg > graal-jvmci-8
comparison mx/mx_graal.py @ 21562:47bebae7454f
Merge.
author | Doug Simon <doug.simon@oracle.com> |
---|---|
date | Thu, 28 May 2015 21:58:33 +0200 |
parents | a9fba0dfb155 28cbfacd0518 |
children | 1de4d7e69f85 |
comparison
equal
deleted
inserted
replaced
21561:ce2113326bc8 | 21562:47bebae7454f |
---|---|
28 | 28 |
29 import os, stat, errno, sys, shutil, zipfile, tarfile, tempfile, re, time, datetime, platform, subprocess, StringIO, socket | 29 import os, stat, errno, sys, shutil, zipfile, tarfile, tempfile, re, time, datetime, platform, subprocess, StringIO, socket |
30 from os.path import join, exists, dirname, basename | 30 from os.path import join, exists, dirname, basename |
31 from argparse import ArgumentParser, RawDescriptionHelpFormatter, REMAINDER | 31 from argparse import ArgumentParser, RawDescriptionHelpFormatter, REMAINDER |
32 from outputparser import OutputParser, ValuesMatcher | 32 from outputparser import OutputParser, ValuesMatcher |
33 import hashlib | |
34 import mx | 33 import mx |
35 import xml.dom.minidom | 34 import xml.dom.minidom |
36 import sanitycheck | 35 import sanitycheck |
37 import itertools | 36 import itertools |
38 import json, textwrap | 37 import json, textwrap |
517 """ | 516 """ |
518 hsSrcGenDir = join(mx.project('com.oracle.jvmci.hotspot').source_gen_dir(), 'hotspot') | 517 hsSrcGenDir = join(mx.project('com.oracle.jvmci.hotspot').source_gen_dir(), 'hotspot') |
519 if not exists(hsSrcGenDir): | 518 if not exists(hsSrcGenDir): |
520 os.makedirs(hsSrcGenDir) | 519 os.makedirs(hsSrcGenDir) |
521 return hsSrcGenDir | 520 return hsSrcGenDir |
522 | |
523 def _update_graalRuntime_inline_hpp(dist): | |
524 """ | |
525 (Re)generates graalRuntime.inline.hpp based on a given distribution | |
526 that transitively represents all the input for the generation process. | |
527 | |
528 A SHA1 digest is computed for all generated content and is written to | |
529 graalRuntime.inline.hpp as well as stored in a generated class | |
530 that is appended to the dist.path jar. At runtime, these two digests | |
531 are checked for consistency. | |
532 """ | |
533 | |
534 p = mx.project('com.oracle.graal.hotspot.sourcegen') | |
535 mainClass = 'com.oracle.graal.hotspot.sourcegen.GenGraalRuntimeInlineHpp' | |
536 if exists(join(p.output_dir(), mainClass.replace('.', os.sep) + '.class')): | |
537 genSrcDir = _makeHotspotGeneratedSourcesDir() | |
538 graalRuntime_inline_hpp = join(genSrcDir, 'graalRuntime.inline.hpp') | |
539 cp = os.pathsep.join([mx.distribution(d).path for d in dist.distDependencies] + [dist.path, p.output_dir()]) | |
540 tmp = StringIO.StringIO() | |
541 mx.run_java(['-cp', mx._separatedCygpathU2W(cp), mainClass], out=tmp.write) | |
542 | |
543 # Compute SHA1 for currently generated graalRuntime.inline.hpp content | |
544 # and all other generated sources in genSrcDir | |
545 d = hashlib.sha1() | |
546 d.update(tmp.getvalue()) | |
547 for e in os.listdir(genSrcDir): | |
548 if e != 'graalRuntime.inline.hpp': | |
549 with open(join(genSrcDir, e)) as fp: | |
550 d.update(fp.read()) | |
551 sha1 = d.hexdigest() | |
552 | |
553 # Add SHA1 to end of graalRuntime.inline.hpp | |
554 print >> tmp, '' | |
555 print >> tmp, 'const char* JVMCIRuntime::_generated_sources_sha1 = "' + sha1 + '";' | |
556 | |
557 mx.update_file(graalRuntime_inline_hpp, tmp.getvalue()) | |
558 | |
559 # Store SHA1 in generated Java class and append class to specified jar | |
560 javaPackageName = 'com.oracle.jvmci.hotspot.sourcegen' | |
561 javaClassName = javaPackageName + '.GeneratedSourcesSha1' | |
562 javaSource = join(_graal_home, 'GeneratedSourcesSha1.java') | |
563 javaClass = join(_graal_home, javaClassName.replace('.', os.path.sep) + '.class') | |
564 with open(javaSource, 'w') as fp: | |
565 print >> fp, 'package ' + javaPackageName + ';' | |
566 print >> fp, 'class GeneratedSourcesSha1 { private static final String value = "' + sha1 + '"; }' | |
567 subprocess.check_call([mx.java().javac, '-d', mx._cygpathU2W(_graal_home), mx._cygpathU2W(javaSource)], stderr=subprocess.PIPE, stdout=subprocess.PIPE) | |
568 zf = zipfile.ZipFile(dist.path, 'a') | |
569 with open(javaClass, 'rb') as fp: | |
570 zf.writestr(javaClassName.replace('.', '/') + '.class', fp.read()) | |
571 zf.close() | |
572 os.unlink(javaSource) | |
573 os.unlink(javaClass) | |
574 javaClassParent = os.path.dirname(javaClass) | |
575 while len(os.listdir(javaClassParent)) == 0: | |
576 os.rmdir(javaClassParent) | |
577 javaClassParent = os.path.dirname(javaClassParent) | |
578 | 521 |
579 def _copyToJdk(src, dst, permissions=JDK_UNIX_PERMISSIONS_FILE): | 522 def _copyToJdk(src, dst, permissions=JDK_UNIX_PERMISSIONS_FILE): |
580 name = os.path.basename(src) | 523 name = os.path.basename(src) |
581 dstLib = join(dst, name) | 524 dstLib = join(dst, name) |
582 if mx.get_env('SYMLINK_GRAAL_JAR', None) == 'true': | 525 if mx.get_env('SYMLINK_GRAAL_JAR', None) == 'true': |
664 for className in classNames: | 607 for className in classNames: |
665 if classification[className] is True: | 608 if classification[className] is True: |
666 filtered.append(className) | 609 filtered.append(className) |
667 return filtered | 610 return filtered |
668 | 611 |
669 def _extractJVMCIServiceFiles(jvmciJars, destination, cleanDestination=True): | 612 def _extractJVMCIFiles(jvmciJars, servicesDir, optionsDir, cleanDestination=True): |
670 if cleanDestination: | 613 if cleanDestination: |
671 if exists(destination): | 614 if exists(servicesDir): |
672 shutil.rmtree(destination) | 615 shutil.rmtree(servicesDir) |
673 os.makedirs(destination) | 616 if exists(optionsDir): |
617 shutil.rmtree(optionsDir) | |
618 if not exists(servicesDir): | |
619 os.makedirs(servicesDir) | |
620 if not exists(optionsDir): | |
621 os.makedirs(optionsDir) | |
674 servicesMap = {} | 622 servicesMap = {} |
623 optionsFiles = [] | |
675 for jar in jvmciJars: | 624 for jar in jvmciJars: |
676 if os.path.isfile(jar): | 625 if os.path.isfile(jar): |
677 with zipfile.ZipFile(jar) as zf: | 626 with zipfile.ZipFile(jar) as zf: |
678 for member in zf.namelist(): | 627 for member in zf.namelist(): |
679 if not member.startswith('META-INF/services'): | 628 if member.startswith('META-INF/services'): |
680 continue | 629 serviceName = basename(member) |
681 serviceName = basename(member) | 630 # we don't handle directories |
682 # we don't handle directories | 631 assert serviceName and member == 'META-INF/services/' + serviceName |
683 assert serviceName and member == 'META-INF/services/' + serviceName | 632 with zf.open(member) as serviceFile: |
684 with zf.open(member) as serviceFile: | 633 serviceImpls = servicesMap.setdefault(serviceName, []) |
685 serviceImpls = servicesMap.setdefault(serviceName, []) | 634 for line in serviceFile.readlines(): |
686 for line in serviceFile.readlines(): | 635 line = line.strip() |
687 line = line.strip() | 636 if line: |
688 if line: | 637 serviceImpls.append(line) |
689 serviceImpls.append(line) | 638 elif member.startswith('META-INF/options'): |
639 filename = basename(member) | |
640 # we don't handle directories | |
641 assert filename and member == 'META-INF/options/' + filename | |
642 targetpath = join(optionsDir, filename) | |
643 optionsFiles.append(filename) | |
644 with zf.open(member) as optionsFile, \ | |
645 file(targetpath, "wb") as target: | |
646 shutil.copyfileobj(optionsFile, target) | |
690 jvmciServices = _filterJVMCIService(servicesMap.keys(), jvmciJars) | 647 jvmciServices = _filterJVMCIService(servicesMap.keys(), jvmciJars) |
691 for serviceName in jvmciServices: | 648 for serviceName in jvmciServices: |
692 serviceImpls = servicesMap[serviceName] | 649 serviceImpls = servicesMap[serviceName] |
693 fd, tmp = tempfile.mkstemp(prefix=serviceName) | 650 fd, tmp = tempfile.mkstemp(prefix=serviceName) |
694 f = os.fdopen(fd, 'w+') | 651 f = os.fdopen(fd, 'w+') |
695 for serviceImpl in serviceImpls: | 652 for serviceImpl in serviceImpls: |
696 f.write(serviceImpl + os.linesep) | 653 f.write(serviceImpl + os.linesep) |
697 target = join(destination, serviceName) | 654 target = join(servicesDir, serviceName) |
698 f.close() | 655 f.close() |
699 shutil.move(tmp, target) | 656 shutil.move(tmp, target) |
700 if mx.get_os() != 'windows': | 657 if mx.get_os() != 'windows': |
701 os.chmod(target, JDK_UNIX_PERMISSIONS_FILE) | 658 os.chmod(target, JDK_UNIX_PERMISSIONS_FILE) |
702 return jvmciServices | 659 return (jvmciServices, optionsFiles) |
703 | 660 |
704 def _updateJVMCIServiceFiles(jdkDir): | 661 def _updateJVMCIFiles(jdkDir): |
705 jreJVMCIDir = join(jdkDir, 'jre', 'lib', 'jvmci') | 662 jreJVMCIDir = join(jdkDir, 'jre', 'lib', 'jvmci') |
706 jvmciJars = [join(jreJVMCIDir, e) for e in os.listdir(jreJVMCIDir) if (e.startswith('jvmci') or e.startswith('graal')) and e.endswith('.jar')] | 663 graalJars = [join(jreJVMCIDir, e) for e in os.listdir(jreJVMCIDir) if e.startswith('graal') and e.endswith('.jar')] |
707 jreJVMCIServicesDir = join(jreJVMCIDir, 'services') | 664 jreGraalServicesDir = join(jreJVMCIDir, 'services') |
708 _extractJVMCIServiceFiles(jvmciJars, jreJVMCIServicesDir) | 665 jreGraalOptionsDir = join(jreJVMCIDir, 'options') |
666 _extractJVMCIFiles(graalJars, jreGraalServicesDir, jreGraalOptionsDir) | |
709 | 667 |
710 def _patchGraalVersionConstant(dist): | 668 def _patchGraalVersionConstant(dist): |
711 """ | 669 """ |
712 Patches the constant "@@graal.version@@" in the constant pool of Graal.class | 670 Patches the constant "@@graal.version@@" in the constant pool of Graal.class |
713 with the computed Graal version string. | 671 with the computed Graal version string. |
732 """ | 690 """ |
733 Installs the jar(s) for a given Distribution into all existing JVMCI JDKs | 691 Installs the jar(s) for a given Distribution into all existing JVMCI JDKs |
734 """ | 692 """ |
735 | 693 |
736 dist = mx.distribution(deployableDist.name) | 694 dist = mx.distribution(deployableDist.name) |
737 | |
738 if dist.name == 'GRAAL': | 695 if dist.name == 'GRAAL': |
739 _patchGraalVersionConstant(dist) | 696 _patchGraalVersionConstant(dist) |
740 | 697 |
741 elif dist.name == 'GRAAL_TRUFFLE': | |
742 # The content in graalRuntime.inline.hpp is generated from Graal | |
743 # classes that contain com.oracle.jvmci.options.Option annotated fields. | |
744 # Since GRAAL_TRUFFLE is the leaf most distribution containing | |
745 # such classes, the generation is triggered when GRAAL_TRUFFLE | |
746 # is (re)built. | |
747 _update_graalRuntime_inline_hpp(dist) | |
748 jdks = _jdksDir() | 698 jdks = _jdksDir() |
749 | |
750 if exists(jdks): | 699 if exists(jdks): |
751 for e in os.listdir(jdks): | 700 for e in os.listdir(jdks): |
752 jdkDir = join(jdks, e) | 701 jdkDir = join(jdks, e) |
753 jreLibDir = join(jdkDir, 'jre', 'lib') | 702 jreLibDir = join(jdkDir, 'jre', 'lib') |
754 if exists(jreLibDir): | 703 if exists(jreLibDir): |
763 _copyToJdk(dist.path, targetDir) | 712 _copyToJdk(dist.path, targetDir) |
764 if dist.sourcesPath: | 713 if dist.sourcesPath: |
765 _copyToJdk(dist.sourcesPath, jdkDir) | 714 _copyToJdk(dist.sourcesPath, jdkDir) |
766 if deployableDist.usesJVMCIClassLoader: | 715 if deployableDist.usesJVMCIClassLoader: |
767 # deploy service files | 716 # deploy service files |
768 _updateJVMCIServiceFiles(jdkDir) | 717 _updateJVMCIFiles(jdkDir) |
769 | 718 |
770 # run a command in the windows SDK Debug Shell | 719 # run a command in the windows SDK Debug Shell |
771 def _runInDebugShell(cmd, workingDir, logFile=None, findInOutput=None, respondTo=None): | 720 def _runInDebugShell(cmd, workingDir, logFile=None, findInOutput=None, respondTo=None): |
772 if respondTo is None: | 721 if respondTo is None: |
773 respondTo = {} | 722 respondTo = {} |
938 else: | 887 else: |
939 defLine = 'EXPORT_LIST += $(EXPORT_JRE_LIB_DIR)/' + basename(dist.path) | 888 defLine = 'EXPORT_LIST += $(EXPORT_JRE_LIB_DIR)/' + basename(dist.path) |
940 if defLine not in defs: | 889 if defLine not in defs: |
941 mx.abort('Missing following line in ' + defsPath + '\n' + defLine) | 890 mx.abort('Missing following line in ' + defsPath + '\n' + defLine) |
942 shutil.copy(dist.path, opts2.export_dir) | 891 shutil.copy(dist.path, opts2.export_dir) |
943 services = _extractJVMCIServiceFiles(jvmciJars, join(opts2.export_dir, 'services')) | 892 services, optionsFiles = _extractJVMCIFiles(jvmciJars, join(opts2.export_dir, 'services'), join(opts2.export_dir, 'options')) |
944 for service in services: | 893 for service in services: |
945 defLine = 'EXPORT_LIST += $(EXPORT_JRE_LIB_JVMCI_SERVICES_DIR)/' + service | 894 defLine = 'EXPORT_LIST += $(EXPORT_JRE_LIB_JVMCI_SERVICES_DIR)/' + service |
946 if defLine not in defs: | 895 if defLine not in defs: |
947 mx.abort('Missing following line in ' + defsPath + ' for service from ' + dist.name + '\n' + defLine) | 896 mx.abort('Missing following line in ' + defsPath + ' for service from ' + dist.name + '\n' + defLine) |
897 for optionsFile in optionsFiles: | |
898 defLine = 'EXPORT_LIST += $(EXPORT_JRE_LIB_JVMCI_OPTIONS_DIR)/' + optionsFile | |
899 if defLine not in defs: | |
900 mx.abort('Missing following line in ' + defsPath + ' for options from ' + dist.name + '\n' + defLine) | |
948 jvmciOptions = join(_graal_home, 'jvmci.options') | 901 jvmciOptions = join(_graal_home, 'jvmci.options') |
949 if exists(jvmciOptions): | 902 if exists(jvmciOptions): |
950 shutil.copy(jvmciOptions, opts2.export_dir) | 903 shutil.copy(jvmciOptions, opts2.export_dir) |
951 | 904 |
952 if not _vmSourcesAvailable or not opts2.native: | 905 if not _vmSourcesAvailable or not opts2.native: |
1784 if args.task_filter: | 1737 if args.task_filter: |
1785 Task.filters = args.task_filter.split(',') | 1738 Task.filters = args.task_filter.split(',') |
1786 | 1739 |
1787 # Force | 1740 # Force |
1788 if not mx._opts.strict_compliance: | 1741 if not mx._opts.strict_compliance: |
1789 mx.log("[gate] foring strict compliance") | 1742 mx.log("[gate] forcing strict compliance") |
1790 mx._opts.strict_compliance = True | 1743 mx._opts.strict_compliance = True |
1791 | 1744 |
1792 tasks = [] | 1745 tasks = [] |
1793 total = Task('Gate') | 1746 total = Task('Gate') |
1794 try: | 1747 try: |