comparison mxtool/mx.py @ 19033:74b144f7c54a

Merge.
author Josef Eisl <josef.eisl@jku.at>
date Wed, 28 Jan 2015 19:32:47 +0100
parents 9aa1cff041eb
children 258b3658845a
comparison
equal deleted inserted replaced
19032:b646e37bc989 19033:74b144f7c54a
1 #!/usr/bin/env python2.7 1 #!/usr/bin/env python2.7
2 # 2 #
3 # ---------------------------------------------------------------------------------------------------- 3 # ----------------------------------------------------------------------------------------------------
4 # 4 #
5 # Copyright (c) 2007, 2012, Oracle and/or its affiliates. All rights reserved. 5 # Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved.
6 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. 6 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
7 # 7 #
8 # This code is free software; you can redistribute it and/or modify it 8 # This code is free software; you can redistribute it and/or modify it
9 # under the terms of the GNU General Public License version 2 only, as 9 # under the terms of the GNU General Public License version 2 only, as
10 # published by the Free Software Foundation. 10 # published by the Free Software Foundation.
32 32
33 Full documentation can be found at https://wiki.openjdk.java.net/display/Graal/The+mx+Tool 33 Full documentation can be found at https://wiki.openjdk.java.net/display/Graal/The+mx+Tool
34 """ 34 """
35 35
36 import sys, os, errno, time, subprocess, shlex, types, StringIO, zipfile, signal, xml.sax.saxutils, tempfile, fnmatch, platform 36 import sys, os, errno, time, subprocess, shlex, types, StringIO, zipfile, signal, xml.sax.saxutils, tempfile, fnmatch, platform
37 import multiprocessing
38 import textwrap 37 import textwrap
39 import socket 38 import socket
40 import tarfile 39 import tarfile
41 import hashlib 40 import hashlib
42 import xml.parsers.expat 41 import xml.parsers.expat
59 cmd = popenargs[0] 58 cmd = popenargs[0]
60 error = subprocess.CalledProcessError(retcode, cmd) 59 error = subprocess.CalledProcessError(retcode, cmd)
61 error.output = output 60 error.output = output
62 raise error 61 raise error
63 return output 62 return output
63
64 # Support for jython
65 def is_jython():
66 return sys.platform.startswith('java')
67
68 if not is_jython():
69 import multiprocessing
70
71 def cpu_count():
72 if is_jython():
73 from java.lang import Runtime
74 runtime = Runtime.getRuntime()
75 return runtime.availableProcessors()
76 else:
77 return multiprocessing.cpu_count()
78
64 79
65 try: subprocess.check_output 80 try: subprocess.check_output
66 except: subprocess.check_output = check_output 81 except: subprocess.check_output = check_output
67 82
68 try: zipfile.ZipFile.__enter__ 83 try: zipfile.ZipFile.__enter__
249 class Dependency: 264 class Dependency:
250 def __init__(self, suite, name): 265 def __init__(self, suite, name):
251 self.name = name 266 self.name = name
252 self.suite = suite 267 self.suite = suite
253 268
269 def __cmp__(self, other):
270 return cmp(self.name, other.name)
271
254 def __str__(self): 272 def __str__(self):
255 return self.name 273 return self.name
256 274
257 def __eq__(self, other): 275 def __eq__(self, other):
258 return self.name == other.name 276 return self.name == other.name
305 def all_deps(self, deps, includeLibs, includeSelf=True, includeJreLibs=False, includeAnnotationProcessors=False): 323 def all_deps(self, deps, includeLibs, includeSelf=True, includeJreLibs=False, includeAnnotationProcessors=False):
306 """ 324 """
307 Add the transitive set of dependencies for this project, including 325 Add the transitive set of dependencies for this project, including
308 libraries if 'includeLibs' is true, to the 'deps' list. 326 libraries if 'includeLibs' is true, to the 'deps' list.
309 """ 327 """
310 return self._all_deps_helper(deps, [], includeLibs, includeSelf, includeJreLibs, includeAnnotationProcessors) 328 return sorted(self._all_deps_helper(deps, [], includeLibs, includeSelf, includeJreLibs, includeAnnotationProcessors))
311 329
312 def _all_deps_helper(self, deps, dependants, includeLibs, includeSelf=True, includeJreLibs=False, includeAnnotationProcessors=False): 330 def _all_deps_helper(self, deps, dependants, includeLibs, includeSelf=True, includeJreLibs=False, includeAnnotationProcessors=False):
313 if self in dependants: 331 if self in dependants:
314 abort(str(self) + 'Project dependency cycle found:\n ' + 332 abort(str(self) + 'Project dependency cycle found:\n ' +
315 '\n |\n V\n '.join(map(str, dependants[dependants.index(self):])) + 333 '\n |\n V\n '.join(map(str, dependants[dependants.index(self):])) +
525 aps.add(p.name) 543 aps.add(p.name)
526 544
527 # Inherit annotation processors from dependencies 545 # Inherit annotation processors from dependencies
528 aps.update(p.annotation_processors()) 546 aps.update(p.annotation_processors())
529 547
530 self._annotationProcessors = list(aps) 548 self._annotationProcessors = sorted(list(aps))
531 return self._annotationProcessors 549 return self._annotationProcessors
532 550
533 """ 551 """
534 Gets the class path composed of the distribution jars containing the 552 Gets the class path composed of the distribution jars containing the
535 annotation processors that will be applied when compiling this project. 553 annotation processors that will be applied when compiling this project.
536 """ 554 """
537 def annotation_processors_path(self): 555 def annotation_processors_path(self):
538 aps = [project(ap) for ap in self.annotation_processors()] 556 aps = [project(ap) for ap in self.annotation_processors()]
539 if len(aps): 557 libAps = [dep for dep in self.all_deps([], includeLibs=True, includeSelf=False) if dep.isLibrary() and hasattr(dep, 'annotationProcessor') and getattr(dep, 'annotationProcessor').lower() == 'true']
540 return os.pathsep.join([ap.definedAnnotationProcessorsDist.path for ap in aps if ap.definedAnnotationProcessorsDist]) 558 if len(aps) + len(libAps):
559 return os.pathsep.join([ap.definedAnnotationProcessorsDist.path for ap in aps if ap.definedAnnotationProcessorsDist] + [lib.get_path(False) for lib in libAps])
541 return None 560 return None
561
562 def uses_annotation_processor_library(self):
563 for dep in self.all_deps([], includeLibs=True, includeSelf=False):
564 if dep.isLibrary() and hasattr(dep, 'annotationProcessor'):
565 return True
566 return False
542 567
543 def update_current_annotation_processors_file(self): 568 def update_current_annotation_processors_file(self):
544 aps = self.annotation_processors() 569 aps = self.annotation_processors()
545 outOfDate = False 570 outOfDate = False
546 currentApsFile = join(self.suite.mxDir, 'currentAnnotationProcessors', self.name) 571 currentApsFile = join(self.suite.mxDir, 'currentAnnotationProcessors', self.name)
587 break 612 break
588 d.update(buf) 613 d.update(buf)
589 return d.hexdigest() 614 return d.hexdigest()
590 615
591 def download_file_with_sha1(name, path, urls, sha1, sha1path, resolve, mustExist, sources=False, canSymlink=True): 616 def download_file_with_sha1(name, path, urls, sha1, sha1path, resolve, mustExist, sources=False, canSymlink=True):
617 canSymlink = canSymlink and not (get_os() == 'windows' or get_os() == 'cygwin')
592 def _download_lib(): 618 def _download_lib():
593 cacheDir = get_env('MX_CACHE_DIR', join(_opts.user_home, '.mx', 'cache')) 619 cacheDir = _cygpathW2U(get_env('MX_CACHE_DIR', join(_opts.user_home, '.mx', 'cache')))
594 if not exists(cacheDir): 620 if not exists(cacheDir):
595 os.makedirs(cacheDir) 621 os.makedirs(cacheDir)
596 base = basename(path) 622 base = basename(path)
597 cachePath = join(cacheDir, base + '_' + sha1) 623 cachePath = join(cacheDir, base + '_' + sha1)
598 624
678 """ 704 """
679 Add the transitive set of dependencies for this JRE library to the 'deps' list. 705 Add the transitive set of dependencies for this JRE library to the 'deps' list.
680 """ 706 """
681 if includeJreLibs and includeSelf and not self in deps: 707 if includeJreLibs and includeSelf and not self in deps:
682 deps.append(self) 708 deps.append(self)
683 return deps 709 return sorted(deps)
684 710
685 class Library(BaseLibrary): 711 class Library(BaseLibrary):
686 def __init__(self, suite, name, path, optional, urls, sha1, sourcePath, sourceUrls, sourceSha1, deps): 712 def __init__(self, suite, name, path, optional, urls, sha1, sourcePath, sourceUrls, sourceSha1, deps):
687 BaseLibrary.__init__(self, suite, name, optional) 713 BaseLibrary.__init__(self, suite, name, optional)
688 self.path = path.replace('/', os.sep) 714 self.path = path.replace('/', os.sep)
751 def all_deps(self, deps, includeLibs, includeSelf=True, includeJreLibs=False, includeAnnotationProcessors=False): 777 def all_deps(self, deps, includeLibs, includeSelf=True, includeJreLibs=False, includeAnnotationProcessors=False):
752 """ 778 """
753 Add the transitive set of dependencies for this library to the 'deps' list. 779 Add the transitive set of dependencies for this library to the 'deps' list.
754 """ 780 """
755 if not includeLibs: 781 if not includeLibs:
756 return deps 782 return sorted(deps)
757 childDeps = list(self.deps) 783 childDeps = list(self.deps)
758 if self in deps: 784 if self in deps:
759 return deps 785 return sorted(deps)
760 for name in childDeps: 786 for name in childDeps:
761 assert name != self.name 787 assert name != self.name
762 dep = library(name) 788 dep = library(name)
763 if not dep in deps: 789 if not dep in deps:
764 dep.all_deps(deps, includeLibs=includeLibs, includeJreLibs=includeJreLibs, includeAnnotationProcessors=includeAnnotationProcessors) 790 dep.all_deps(deps, includeLibs=includeLibs, includeJreLibs=includeJreLibs, includeAnnotationProcessors=includeAnnotationProcessors)
765 if not self in deps and includeSelf: 791 if not self in deps and includeSelf:
766 deps.append(self) 792 deps.append(self)
767 return deps 793 return sorted(deps)
768 794
769 class HgConfig: 795 class HgConfig:
770 """ 796 """
771 Encapsulates access to Mercurial (hg) 797 Encapsulates access to Mercurial (hg)
772 """ 798 """
959 return [] 985 return []
960 if not isinstance(v, list): 986 if not isinstance(v, list):
961 abort('Attribute "' + name + '" for ' + context + ' must be a list') 987 abort('Attribute "' + name + '" for ' + context + ' must be a list')
962 return v 988 return v
963 989
964 for name, attrs in projsMap.iteritems(): 990 for name, attrs in sorted(projsMap.iteritems()):
965 context = 'project ' + name 991 context = 'project ' + name
966 srcDirs = pop_list(attrs, 'sourceDirs', context) 992 srcDirs = pop_list(attrs, 'sourceDirs', context)
967 deps = pop_list(attrs, 'dependencies', context) 993 deps = pop_list(attrs, 'dependencies', context)
968 ap = pop_list(attrs, 'annotationProcessors', context) 994 ap = pop_list(attrs, 'annotationProcessors', context)
969 javaCompliance = attrs.pop('javaCompliance', None) 995 javaCompliance = attrs.pop('javaCompliance', None)
981 if len(ap) > 0: 1007 if len(ap) > 0:
982 p._declaredAnnotationProcessors = ap 1008 p._declaredAnnotationProcessors = ap
983 p.__dict__.update(attrs) 1009 p.__dict__.update(attrs)
984 self.projects.append(p) 1010 self.projects.append(p)
985 1011
986 for name, attrs in jreLibsMap.iteritems(): 1012 for name, attrs in sorted(jreLibsMap.iteritems()):
987 jar = attrs.pop('jar') 1013 jar = attrs.pop('jar')
988 # JRE libraries are optional by default 1014 # JRE libraries are optional by default
989 optional = attrs.pop('optional', 'true') != 'false' 1015 optional = attrs.pop('optional', 'true') != 'false'
990 l = JreLibrary(self, name, jar, optional) 1016 l = JreLibrary(self, name, jar, optional)
991 self.jreLibs.append(l) 1017 self.jreLibs.append(l)
992 1018
993 for name, attrs in libsMap.iteritems(): 1019 for name, attrs in sorted(libsMap.iteritems()):
994 context = 'library ' + name 1020 context = 'library ' + name
995 if "|" in name: 1021 if "|" in name:
996 if name.count('|') != 2: 1022 if name.count('|') != 2:
997 abort("Format error in library name: " + name + "\nsyntax: libname|os-platform|architecture") 1023 abort("Format error in library name: " + name + "\nsyntax: libname|os-platform|architecture")
998 name, platform, architecture = name.split("|") 1024 name, platform, architecture = name.split("|")
1009 optional = False 1035 optional = False
1010 l = Library(self, name, path, optional, urls, sha1, sourcePath, sourceUrls, sourceSha1, deps) 1036 l = Library(self, name, path, optional, urls, sha1, sourcePath, sourceUrls, sourceSha1, deps)
1011 l.__dict__.update(attrs) 1037 l.__dict__.update(attrs)
1012 self.libs.append(l) 1038 self.libs.append(l)
1013 1039
1014 for name, attrs in distsMap.iteritems(): 1040 for name, attrs in sorted(distsMap.iteritems()):
1015 context = 'distribution ' + name 1041 context = 'distribution ' + name
1016 path = attrs.pop('path') 1042 path = attrs.pop('path')
1017 sourcesPath = attrs.pop('sourcesPath', None) 1043 sourcesPath = attrs.pop('sourcesPath', None)
1018 deps = pop_list(attrs, 'dependencies', context) 1044 deps = pop_list(attrs, 'dependencies', context)
1019 mainClass = attrs.pop('mainClass', None) 1045 mainClass = attrs.pop('mainClass', None)
1287 result = xml.sax.saxutils.escape(result, entities) 1313 result = xml.sax.saxutils.escape(result, entities)
1288 if standalone is not None: 1314 if standalone is not None:
1289 result = result.replace('encoding="UTF-8"?>', 'encoding="UTF-8" standalone="' + str(standalone) + '"?>') 1315 result = result.replace('encoding="UTF-8"?>', 'encoding="UTF-8" standalone="' + str(standalone) + '"?>')
1290 return result 1316 return result
1291 1317
1318 def get_jython_os():
1319 from java.lang import System as System
1320 os_name = System.getProperty('os.name').lower()
1321 if System.getProperty('isCygwin'):
1322 return 'cygwin'
1323 elif os_name.startswith('mac'):
1324 return 'darwin'
1325 elif os_name.startswith('linux'):
1326 return 'linux'
1327 elif os_name.startswith('sunos'):
1328 return 'solaris'
1329 elif os_name.startswith('win'):
1330 return 'windows'
1331 else:
1332 abort('Unknown operating system ' + os_name)
1333
1292 def get_os(): 1334 def get_os():
1293 """ 1335 """
1294 Get a canonical form of sys.platform. 1336 Get a canonical form of sys.platform.
1295 """ 1337 """
1296 if sys.platform.startswith('darwin'): 1338 if is_jython():
1339 return get_jython_os()
1340 elif sys.platform.startswith('darwin'):
1297 return 'darwin' 1341 return 'darwin'
1298 elif sys.platform.startswith('linux'): 1342 elif sys.platform.startswith('linux'):
1299 return 'linux' 1343 return 'linux'
1300 elif sys.platform.startswith('sunos'): 1344 elif sys.platform.startswith('sunos'):
1301 return 'solaris' 1345 return 'solaris'
1311 Translate a path from unix-style to windows-style. 1355 Translate a path from unix-style to windows-style.
1312 This method has no effects on other platforms than cygwin. 1356 This method has no effects on other platforms than cygwin.
1313 """ 1357 """
1314 if p is None or get_os() != "cygwin": 1358 if p is None or get_os() != "cygwin":
1315 return p 1359 return p
1316 return subprocess.check_output(['cygpath', '-w', p]).strip() 1360 return subprocess.check_output(['cygpath', '-a', '-w', p]).strip()
1317 1361
1318 def _cygpathW2U(p): 1362 def _cygpathW2U(p):
1319 """ 1363 """
1320 Translate a path from windows-style to unix-style. 1364 Translate a path from windows-style to unix-style.
1321 This method has no effects on other platforms than cygwin. 1365 This method has no effects on other platforms than cygwin.
1322 """ 1366 """
1323 if p is None or get_os() != "cygwin": 1367 if p is None or get_os() != "cygwin":
1324 return p 1368 return p
1325 return subprocess.check_output(['cygpath', '-u', p]).strip() 1369 return subprocess.check_output(['cygpath', '-a', '-u', p]).strip()
1326 1370
1327 def _separatedCygpathU2W(p): 1371 def _separatedCygpathU2W(p):
1328 """ 1372 """
1329 Translate a group of paths, separated by a path separator. 1373 Translate a group of paths, separated by a path separator.
1330 unix-style to windows-style. 1374 unix-style to windows-style.
1591 deps = [] 1635 deps = []
1592 for p in projects: 1636 for p in projects:
1593 p.all_deps(deps, includeLibs=includeLibs, includeJreLibs=includeJreLibs, includeAnnotationProcessors=includeAnnotationProcessors) 1637 p.all_deps(deps, includeLibs=includeLibs, includeJreLibs=includeJreLibs, includeAnnotationProcessors=includeAnnotationProcessors)
1594 return deps 1638 return deps
1595 1639
1596 def _handle_missing_java_home(): 1640 def _handle_lookup_java_home(jdk):
1641 return _handle_lookup_jdk(jdk, 'JAVA_HOME', '--java-home', False)
1642
1643 def _handle_lookup_extra_java_homes(jdk):
1644 return _handle_lookup_jdk(jdk, 'EXTRA_JAVA_HOMES', '--extra-java-homes', True)
1645
1646 def _handle_lookup_jdk(jdk, varName, flagName, allowMultiple):
1647 if jdk != None and jdk != '':
1648 return jdk
1649 jdk = os.environ.get(varName)
1650 if jdk != None and jdk != '':
1651 return jdk
1652
1597 if not sys.stdout.isatty(): 1653 if not sys.stdout.isatty():
1598 abort('Could not find bootstrap JDK. Use --java-home option or ensure JAVA_HOME environment variable is set.') 1654 abort('Could not find bootstrap {0}. Use {1} option or ensure {2} environment variable is set.'.format(varName, flagName, varName))
1599 1655
1600 candidateJdks = [] 1656 candidateJdks = []
1601 if get_os() == 'darwin': 1657 if get_os() == 'darwin':
1602 base = '/Library/Java/JavaVirtualMachines' 1658 base = '/Library/Java/JavaVirtualMachines'
1603 candidateJdks = [join(base, n, 'Contents/Home') for n in os.listdir(base) if exists(join(base, n, 'Contents/Home'))] 1659 candidateJdks = [join(base, n, 'Contents/Home') for n in os.listdir(base) if exists(join(base, n, 'Contents/Home'))]
1611 base = r'C:\Program Files\Java' 1667 base = r'C:\Program Files\Java'
1612 candidateJdks = [join(base, n) for n in os.listdir(base) if exists(join(base, n, r'jre\lib\rt.jar'))] 1668 candidateJdks = [join(base, n) for n in os.listdir(base) if exists(join(base, n, r'jre\lib\rt.jar'))]
1613 1669
1614 javaHome = None 1670 javaHome = None
1615 if len(candidateJdks) != 0: 1671 if len(candidateJdks) != 0:
1616 javaHome = select_items(candidateJdks + ['<other>'], allowMultiple=False) 1672 log('Missing value for {0}.'.format(varName))
1673 javaHome = select_items(candidateJdks + ['<other>'], allowMultiple=allowMultiple)
1617 if javaHome == '<other>': 1674 if javaHome == '<other>':
1618 javaHome = None 1675 javaHome = None
1676 if javaHome != None and allowMultiple:
1677 javaHome = os.pathsep.join(javaHome)
1619 1678
1620 while javaHome is None: 1679 while javaHome is None:
1621 javaHome = raw_input('Enter path of bootstrap JDK: ') 1680 javaHome = raw_input('Enter path of JDK for {0}: '.format(varName))
1622 rtJarPath = join(javaHome, 'jre', 'lib', 'rt.jar') 1681 rtJarPath = join(javaHome, 'jre', 'lib', 'rt.jar')
1623 if not exists(rtJarPath): 1682 if not exists(rtJarPath):
1624 log('Does not appear to be a valid JDK as ' + rtJarPath + ' does not exist') 1683 log('Does not appear to be a valid JDK as ' + rtJarPath + ' does not exist')
1625 javaHome = None 1684 javaHome = None
1626 else: 1685 else:
1627 break 1686 break
1628 1687
1629 envPath = join(_primary_suite.mxDir, 'env') 1688 envPath = join(_primary_suite.mxDir, 'env')
1630 if ask_yes_no('Persist this setting by adding "JAVA_HOME=' + javaHome + '" to ' + envPath, 'y'): 1689 if ask_yes_no('Persist this setting by adding "{0}={1}" to {2}'.format(varName, javaHome, envPath), 'y'):
1631 with open(envPath, 'a') as fp: 1690 with open(envPath, 'a') as fp:
1632 print >> fp, 'JAVA_HOME=' + javaHome 1691 print >> fp, varName + '=' + javaHome
1633 1692
1634 return javaHome 1693 return javaHome
1635 1694
1636 class ArgParser(ArgumentParser): 1695 class ArgParser(ArgumentParser):
1637 # Override parent to append the list of available commands 1696 # Override parent to append the list of available commands
1648 self.add_argument('-V', action='store_true', dest='very_verbose', help='enable very verbose output') 1707 self.add_argument('-V', action='store_true', dest='very_verbose', help='enable very verbose output')
1649 self.add_argument('-w', action='store_true', dest='warn', help='enable warning messages') 1708 self.add_argument('-w', action='store_true', dest='warn', help='enable warning messages')
1650 self.add_argument('-p', '--primary-suite-path', help='set the primary suite directory', metavar='<path>') 1709 self.add_argument('-p', '--primary-suite-path', help='set the primary suite directory', metavar='<path>')
1651 self.add_argument('--dbg', type=int, dest='java_dbg_port', help='make Java processes wait on <port> for a debugger', metavar='<port>') 1710 self.add_argument('--dbg', type=int, dest='java_dbg_port', help='make Java processes wait on <port> for a debugger', metavar='<port>')
1652 self.add_argument('-d', action='store_const', const=8000, dest='java_dbg_port', help='alias for "-dbg 8000"') 1711 self.add_argument('-d', action='store_const', const=8000, dest='java_dbg_port', help='alias for "-dbg 8000"')
1712 self.add_argument('--backup-modified', action='store_true', help='backup generated files if they pre-existed and are modified')
1653 self.add_argument('--cp-pfx', dest='cp_prefix', help='class path prefix', metavar='<arg>') 1713 self.add_argument('--cp-pfx', dest='cp_prefix', help='class path prefix', metavar='<arg>')
1654 self.add_argument('--cp-sfx', dest='cp_suffix', help='class path suffix', metavar='<arg>') 1714 self.add_argument('--cp-sfx', dest='cp_suffix', help='class path suffix', metavar='<arg>')
1655 self.add_argument('--J', dest='java_args', help='Java VM arguments (e.g. --J @-dsa)', metavar='@<args>') 1715 self.add_argument('--J', dest='java_args', help='Java VM arguments (e.g. --J @-dsa)', metavar='@<args>')
1656 self.add_argument('--Jp', action='append', dest='java_args_pfx', help='prefix Java VM arguments (e.g. --Jp @-dsa)', metavar='@<args>', default=[]) 1716 self.add_argument('--Jp', action='append', dest='java_args_pfx', help='prefix Java VM arguments (e.g. --Jp @-dsa)', metavar='@<args>', default=[])
1657 self.add_argument('--Ja', action='append', dest='java_args_sfx', help='suffix Java VM arguments (e.g. --Ja @-dsa)', metavar='@<args>', default=[]) 1717 self.add_argument('--Ja', action='append', dest='java_args_sfx', help='suffix Java VM arguments (e.g. --Ja @-dsa)', metavar='@<args>', default=[])
1681 opts.__dict__.setdefault('ptimeout', 0) 1741 opts.__dict__.setdefault('ptimeout', 0)
1682 1742
1683 if opts.very_verbose: 1743 if opts.very_verbose:
1684 opts.verbose = True 1744 opts.verbose = True
1685 1745
1686 if opts.java_home is None: 1746 opts.java_home = _handle_lookup_java_home(opts.java_home)
1687 opts.java_home = os.environ.get('JAVA_HOME') 1747 opts.extra_java_homes = _handle_lookup_extra_java_homes(opts.extra_java_homes)
1688 if opts.extra_java_homes is None:
1689 opts.extra_java_homes = os.environ.get('EXTRA_JAVA_HOMES')
1690
1691 if opts.java_home is None or opts.java_home == '':
1692 opts.java_home = _handle_missing_java_home()
1693 1748
1694 if opts.user_home is None or opts.user_home == '': 1749 if opts.user_home is None or opts.user_home == '':
1695 abort('Could not find user home. Use --user-home option or ensure HOME environment variable is set.') 1750 abort('Could not find user home. Use --user-home option or ensure HOME environment variable is set.')
1696 1751
1697 os.environ['JAVA_HOME'] = opts.java_home 1752 os.environ['JAVA_HOME'] = opts.java_home
1818 assert isinstance(args, types.ListType), "'args' must be a list: " + str(args) 1873 assert isinstance(args, types.ListType), "'args' must be a list: " + str(args)
1819 for arg in args: 1874 for arg in args:
1820 assert isinstance(arg, types.StringTypes), 'argument is not a string: ' + str(arg) 1875 assert isinstance(arg, types.StringTypes), 'argument is not a string: ' + str(arg)
1821 1876
1822 if env is None: 1877 if env is None:
1823 env = os.environ 1878 env = os.environ.copy()
1879
1880 # Ideally the command line could be communicated directly in an environment
1881 # variable. However, since environment variables share the same resource
1882 # space as the command line itself (on Unix at least), this would cause the
1883 # limit to be exceeded too easily.
1884 with tempfile.NamedTemporaryFile(suffix='', prefix='mx_subprocess_command.', mode='w', delete=False) as fp:
1885 subprocessCommandFile = fp.name
1886 for arg in args:
1887 # TODO: handle newlines in args once there's a use case
1888 assert '\n' not in arg
1889 print >> fp, arg
1890 env['MX_SUBPROCESS_COMMAND_FILE'] = subprocessCommandFile
1824 1891
1825 if _opts.verbose: 1892 if _opts.verbose:
1826 if _opts.very_verbose: 1893 if _opts.very_verbose:
1827 log('Environment variables:') 1894 log('Environment variables:')
1828 for key in sorted(env.keys()): 1895 for key in sorted(env.keys()):
1836 try: 1903 try:
1837 # On Unix, the new subprocess should be in a separate group so that a timeout alarm 1904 # On Unix, the new subprocess should be in a separate group so that a timeout alarm
1838 # can use os.killpg() to kill the whole subprocess group 1905 # can use os.killpg() to kill the whole subprocess group
1839 preexec_fn = None 1906 preexec_fn = None
1840 creationflags = 0 1907 creationflags = 0
1841 if get_os() == 'windows': 1908 if not is_jython():
1842 creationflags = subprocess.CREATE_NEW_PROCESS_GROUP 1909 if get_os() == 'windows':
1843 else: 1910 creationflags = subprocess.CREATE_NEW_PROCESS_GROUP
1844 preexec_fn = os.setsid 1911 else:
1845 1912 preexec_fn = os.setsid
1846 def redirect(stream, f): 1913 def redirect(stream, f):
1847 for line in iter(stream.readline, ''): 1914 for line in iter(stream.readline, ''):
1848 f(line) 1915 f(line)
1849 stream.close() 1916 stream.close()
1850 stdout = out if not callable(out) else subprocess.PIPE 1917 stdout = out if not callable(out) else subprocess.PIPE
1880 abort(e.errno) 1947 abort(e.errno)
1881 except KeyboardInterrupt: 1948 except KeyboardInterrupt:
1882 abort(1) 1949 abort(1)
1883 finally: 1950 finally:
1884 _removeSubprocess(sub) 1951 _removeSubprocess(sub)
1952 os.remove(subprocessCommandFile)
1885 1953
1886 if retcode and nonZeroIsFatal: 1954 if retcode and nonZeroIsFatal:
1887 if _opts.verbose: 1955 if _opts.verbose:
1888 if _opts.very_verbose: 1956 if _opts.very_verbose:
1889 raise subprocess.CalledProcessError(retcode, ' '.join(args)) 1957 raise subprocess.CalledProcessError(retcode, ' '.join(args))
2068 if not exists(outDir): 2136 if not exists(outDir):
2069 os.makedirs(outDir) 2137 os.makedirs(outDir)
2070 javaSource = join(myDir, 'ClasspathDump.java') 2138 javaSource = join(myDir, 'ClasspathDump.java')
2071 if not exists(join(outDir, 'ClasspathDump.class')): 2139 if not exists(join(outDir, 'ClasspathDump.class')):
2072 subprocess.check_call([self.javac, '-d', _cygpathU2W(outDir), _cygpathU2W(javaSource)], stderr=subprocess.PIPE, stdout=subprocess.PIPE) 2140 subprocess.check_call([self.javac, '-d', _cygpathU2W(outDir), _cygpathU2W(javaSource)], stderr=subprocess.PIPE, stdout=subprocess.PIPE)
2073 self._bootclasspath, self._extdirs, self._endorseddirs = [x if x != 'null' else None for x in subprocess.check_output([self.java, '-cp', _separatedCygpathU2W(outDir), 'ClasspathDump'], stderr=subprocess.PIPE).split('|')] 2141 self._bootclasspath, self._extdirs, self._endorseddirs = [x if x != 'null' else None for x in subprocess.check_output([self.java, '-cp', _cygpathU2W(outDir), 'ClasspathDump'], stderr=subprocess.PIPE).split('|')]
2074 if not self._bootclasspath or not self._extdirs or not self._endorseddirs: 2142 if not self._bootclasspath or not self._extdirs or not self._endorseddirs:
2075 warn("Could not find all classpaths: boot='" + str(self._bootclasspath) + "' extdirs='" + str(self._extdirs) + "' endorseddirs='" + str(self._endorseddirs) + "'") 2143 warn("Could not find all classpaths: boot='" + str(self._bootclasspath) + "' extdirs='" + str(self._extdirs) + "' endorseddirs='" + str(self._endorseddirs) + "'")
2076 self._bootclasspath = _filter_non_existant_paths(self._bootclasspath) 2144 self._bootclasspath = _filter_non_existant_paths(self._bootclasspath)
2077 self._extdirs = _filter_non_existant_paths(self._extdirs) 2145 self._extdirs = _filter_non_existant_paths(self._extdirs)
2078 self._endorseddirs = _filter_non_existant_paths(self._endorseddirs) 2146 self._endorseddirs = _filter_non_existant_paths(self._endorseddirs)
2079 2147
2148 def __repr__(self):
2149 return "JavaConfig(" + str(self.jdk) + ", " + str(self.debug_port) + ")"
2150
2151 def __str__(self):
2152 return "Java " + str(self.version) + " (" + str(self.javaCompliance) + ") from " + str(self.jdk)
2153
2080 def __hash__(self): 2154 def __hash__(self):
2081 return hash(self.jdk) 2155 return hash(self.jdk)
2082 2156
2083 def __cmp__(self, other): 2157 def __cmp__(self, other):
2084 if isinstance(other, JavaConfig): 2158 if isinstance(other, JavaConfig):
2085 return cmp(self.javaCompliance, other.javaCompliance) 2159 compilanceCmp = cmp(self.javaCompliance, other.javaCompliance)
2160 if compilanceCmp:
2161 return compilanceCmp
2162 versionCmp = cmp(self.version, other.version)
2163 if versionCmp:
2164 return versionCmp
2165 return cmp(self.jdk, other.jdk)
2086 raise TypeError() 2166 raise TypeError()
2087 2167
2088 def format_cmd(self, args, addDefaultArgs): 2168 def format_cmd(self, args, addDefaultArgs):
2089 if addDefaultArgs: 2169 if addDefaultArgs:
2090 return [self.java] + self.processArgs(args) 2170 return [self.java] + self.processArgs(args)
2216 _send_sigquit() 2296 _send_sigquit()
2217 2297
2218 def is_alive(p): 2298 def is_alive(p):
2219 if isinstance(p, subprocess.Popen): 2299 if isinstance(p, subprocess.Popen):
2220 return p.poll() is None 2300 return p.poll() is None
2221 assert isinstance(p, multiprocessing.Process), p 2301 assert is_jython() or isinstance(p, multiprocessing.Process), p
2222 return p.is_alive() 2302 return p.is_alive()
2223 2303
2224 for p, args in _currentSubprocesses: 2304 for p, args in _currentSubprocesses:
2225 if is_alive(p): 2305 if is_alive(p):
2226 try: 2306 try:
2275 with open(path, 'rb') as f: 2355 with open(path, 'rb') as f:
2276 old = f.read() 2356 old = f.read()
2277 2357
2278 if old == content: 2358 if old == content:
2279 return False 2359 return False
2360
2361 if existed and _opts.backup_modified:
2362 shutil.move(path, path + '.orig')
2280 2363
2281 with open(path, 'wb') as f: 2364 with open(path, 'wb') as f:
2282 f.write(content) 2365 f.write(content)
2283 2366
2284 log(('modified ' if existed else 'created ') + path) 2367 log(('modified ' if existed else 'created ') + path)
2379 log('JDT properties file {0} not found'.format(jdtProperties)) 2462 log('JDT properties file {0} not found'.format(jdtProperties))
2380 else: 2463 else:
2381 with open(jdtProperties) as fp: 2464 with open(jdtProperties) as fp:
2382 origContent = fp.read() 2465 origContent = fp.read()
2383 content = origContent 2466 content = origContent
2384 if args.jdt_warning_as_error: 2467 if self.proj.uses_annotation_processor_library():
2468 # unfortunately, the command line compiler doesn't let us ignore warnings for generated files only
2469 content = content.replace('=warning', '=ignore')
2470 elif args.jdt_warning_as_error:
2385 content = content.replace('=warning', '=error') 2471 content = content.replace('=warning', '=error')
2386 if not args.jdt_show_task_tags: 2472 if not args.jdt_show_task_tags:
2387 content = content + '\norg.eclipse.jdt.core.compiler.problem.tasks=ignore' 2473 content = content + '\norg.eclipse.jdt.core.compiler.problem.tasks=ignore'
2388 if origContent != content: 2474 if origContent != content:
2389 jdtPropertiesTmp = jdtProperties + '.tmp' 2475 jdtPropertiesTmp = jdtProperties + '.tmp'
2422 parser = ArgumentParser(prog='mx build') 2508 parser = ArgumentParser(prog='mx build')
2423 2509
2424 parser = parser if parser is not None else ArgumentParser(prog='mx build') 2510 parser = parser if parser is not None else ArgumentParser(prog='mx build')
2425 parser.add_argument('-f', action='store_true', dest='force', help='force build (disables timestamp checking)') 2511 parser.add_argument('-f', action='store_true', dest='force', help='force build (disables timestamp checking)')
2426 parser.add_argument('-c', action='store_true', dest='clean', help='removes existing build output') 2512 parser.add_argument('-c', action='store_true', dest='clean', help='removes existing build output')
2427 parser.add_argument('-p', action='store_true', dest='parallelize', help='parallelizes Java compilation') 2513 parser.add_argument('-p', action='store_true', dest='parallelize', help='parallelizes Java compilation if possible')
2428 parser.add_argument('--source', dest='compliance', help='Java compliance level for projects without an explicit one') 2514 parser.add_argument('--source', dest='compliance', help='Java compliance level for projects without an explicit one')
2429 parser.add_argument('--Wapi', action='store_true', dest='warnAPI', help='show warnings about using internal APIs') 2515 parser.add_argument('--Wapi', action='store_true', dest='warnAPI', help='show warnings about using internal APIs')
2430 parser.add_argument('--projects', action='store', help='comma separated projects to build (omit to build all projects)') 2516 parser.add_argument('--projects', action='store', help='comma separated projects to build (omit to build all projects)')
2431 parser.add_argument('--only', action='store', help='comma separated projects to build, without checking their dependencies (omit to build all projects)') 2517 parser.add_argument('--only', action='store', help='comma separated projects to build, without checking their dependencies (omit to build all projects)')
2432 parser.add_argument('--no-java', action='store_false', dest='java', help='do not build Java projects') 2518 parser.add_argument('--no-java', action='store_false', dest='java', help='do not build Java projects')
2442 if suppliedParser: 2528 if suppliedParser:
2443 parser.add_argument('remainder', nargs=REMAINDER, metavar='...') 2529 parser.add_argument('remainder', nargs=REMAINDER, metavar='...')
2444 2530
2445 args = parser.parse_args(args) 2531 args = parser.parse_args(args)
2446 2532
2533 if is_jython():
2534 if args.parallelize:
2535 logv('[multiprocessing not available in jython]')
2536 args.parallelize = False
2537
2447 jdtJar = None 2538 jdtJar = None
2448 if not args.javac and args.jdt is not None: 2539 if not args.javac and args.jdt is not None:
2449 if not args.jdt.endswith('.jar'): 2540 if not args.jdt.endswith('.jar'):
2450 abort('Path for Eclipse batch compiler does not look like a jar file: ' + args.jdt) 2541 abort('Path for Eclipse batch compiler does not look like a jar file: ' + args.jdt)
2451 jdtJar = args.jdt 2542 jdtJar = args.jdt
2598 def sortWorklist(tasks): 2689 def sortWorklist(tasks):
2599 for t in tasks: 2690 for t in tasks:
2600 t._d = None 2691 t._d = None
2601 return sorted(tasks, compareTasks) 2692 return sorted(tasks, compareTasks)
2602 2693
2603 cpus = multiprocessing.cpu_count() 2694 cpus = cpu_count()
2604 worklist = sortWorklist(tasks.values()) 2695 worklist = sortWorklist(tasks.values())
2605 active = [] 2696 active = []
2606 failed = [] 2697 failed = []
2607 while len(worklist) != 0: 2698 while len(worklist) != 0:
2608 while True: 2699 while True:
3521 srcDir = join(p.dir, src) 3612 srcDir = join(p.dir, src)
3522 if not exists(srcDir): 3613 if not exists(srcDir):
3523 os.mkdir(srcDir) 3614 os.mkdir(srcDir)
3524 out.element('classpathentry', {'kind' : 'src', 'path' : src}) 3615 out.element('classpathentry', {'kind' : 'src', 'path' : src})
3525 3616
3526 if len(p.annotation_processors()) > 0: 3617 processorPath = p.annotation_processors_path()
3618 if processorPath:
3527 genDir = p.source_gen_dir() 3619 genDir = p.source_gen_dir()
3528 if not exists(genDir): 3620 if not exists(genDir):
3529 os.mkdir(genDir) 3621 os.mkdir(genDir)
3530 out.element('classpathentry', {'kind' : 'src', 'path' : 'src_gen'}) 3622 out.open('classpathentry', {'kind' : 'src', 'path' : 'src_gen'})
3623 if p.uses_annotation_processor_library():
3624 # ignore warnings produced by third-party annotation processors
3625 out.open('attributes')
3626 out.element('attribute', {'name' : 'ignore_optional_problems', 'value' : 'true'})
3627 out.close('attributes')
3628 out.close('classpathentry')
3629
3531 if files: 3630 if files:
3532 files.append(genDir) 3631 files.append(genDir)
3533 3632
3534 # Every Java program depends on a JRE 3633 # Every Java program depends on a JRE
3535 out.element('classpathentry', {'kind' : 'con', 'path' : 'org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-' + str(p.javaCompliance)}) 3634 out.element('classpathentry', {'kind' : 'con', 'path' : 'org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-' + str(p.javaCompliance)})
3552 else: 3651 else:
3553 libraryDeps.add(dep) 3652 libraryDeps.add(dep)
3554 elif dep.isProject(): 3653 elif dep.isProject():
3555 projectDeps.add(dep) 3654 projectDeps.add(dep)
3556 3655
3557 for dep in containerDeps: 3656 for dep in sorted(containerDeps):
3558 out.element('classpathentry', {'exported' : 'true', 'kind' : 'con', 'path' : dep}) 3657 out.element('classpathentry', {'exported' : 'true', 'kind' : 'con', 'path' : dep})
3559 3658
3560 for dep in libraryDeps: 3659 for dep in sorted(libraryDeps):
3561 path = dep.path 3660 path = dep.path
3562 dep.get_path(resolve=True) 3661 dep.get_path(resolve=True)
3563 3662
3564 # Relative paths for "lib" class path entries have various semantics depending on the Eclipse 3663 # Relative paths for "lib" class path entries have various semantics depending on the Eclipse
3565 # version being used (e.g. see https://bugs.eclipse.org/bugs/show_bug.cgi?id=274737) so it's 3664 # version being used (e.g. see https://bugs.eclipse.org/bugs/show_bug.cgi?id=274737) so it's
3573 attributes['sourcepath'] = sourcePath 3672 attributes['sourcepath'] = sourcePath
3574 out.element('classpathentry', attributes) 3673 out.element('classpathentry', attributes)
3575 if libFiles: 3674 if libFiles:
3576 libFiles.append(path) 3675 libFiles.append(path)
3577 3676
3578 for dep in projectDeps: 3677 for dep in sorted(projectDeps):
3579 out.element('classpathentry', {'combineaccessrules' : 'false', 'exported' : 'true', 'kind' : 'src', 'path' : '/' + dep.name}) 3678 out.element('classpathentry', {'combineaccessrules' : 'false', 'exported' : 'true', 'kind' : 'src', 'path' : '/' + dep.name})
3580 3679
3581 out.element('classpathentry', {'kind' : 'output', 'path' : getattr(p, 'eclipse.output', 'bin')}) 3680 out.element('classpathentry', {'kind' : 'output', 'path' : getattr(p, 'eclipse.output', 'bin')})
3582 out.close('classpath') 3681 out.close('classpath')
3583 classpathFile = join(p.dir, '.classpath') 3682 classpathFile = join(p.dir, '.classpath')
3705 esdict[name] = os.path.abspath(join(projectSettingsDir, name)) 3804 esdict[name] = os.path.abspath(join(projectSettingsDir, name))
3706 3805
3707 # copy a possibly modified file to the project's .settings directory 3806 # copy a possibly modified file to the project's .settings directory
3708 for name, path in esdict.iteritems(): 3807 for name, path in esdict.iteritems():
3709 # ignore this file altogether if this project has no annotation processors 3808 # ignore this file altogether if this project has no annotation processors
3710 if name == "org.eclipse.jdt.apt.core.prefs" and not len(p.annotation_processors()) > 0: 3809 if name == "org.eclipse.jdt.apt.core.prefs" and not processorPath:
3711 continue 3810 continue
3712 3811
3713 with open(path) as f: 3812 with open(path) as f:
3714 content = f.read() 3813 content = f.read()
3715 content = content.replace('${javaCompliance}', str(p.javaCompliance)) 3814 content = content.replace('${javaCompliance}', str(p.javaCompliance))
3716 if len(p.annotation_processors()) > 0: 3815 if processorPath:
3717 content = content.replace('org.eclipse.jdt.core.compiler.processAnnotations=disabled', 'org.eclipse.jdt.core.compiler.processAnnotations=enabled') 3816 content = content.replace('org.eclipse.jdt.core.compiler.processAnnotations=disabled', 'org.eclipse.jdt.core.compiler.processAnnotations=enabled')
3718 update_file(join(settingsDir, name), content) 3817 update_file(join(settingsDir, name), content)
3719 if files: 3818 if files:
3720 files.append(join(settingsDir, name)) 3819 files.append(join(settingsDir, name))
3721 3820
3722 processorPath = p.annotation_processors_path()
3723 if processorPath: 3821 if processorPath:
3724 out = XMLDoc() 3822 out = XMLDoc()
3725 out.open('factorypath') 3823 out.open('factorypath')
3726 out.element('factorypathentry', {'kind' : 'PLUGIN', 'id' : 'org.eclipse.jst.ws.annotations.core', 'enabled' : 'true', 'runInBatchMode' : 'false'}) 3824 out.element('factorypathentry', {'kind' : 'PLUGIN', 'id' : 'org.eclipse.jst.ws.annotations.core', 'enabled' : 'true', 'runInBatchMode' : 'false'})
3727 for e in processorPath.split(os.pathsep): 3825 for e in processorPath.split(os.pathsep):
4449 compilerXml.element('entry', attributes={'name': '!?*.java'}) 4547 compilerXml.element('entry', attributes={'name': '!?*.java'})
4450 compilerXml.close('wildcardResourcePatterns') 4548 compilerXml.close('wildcardResourcePatterns')
4451 4549
4452 if annotationProcessorProfiles: 4550 if annotationProcessorProfiles:
4453 compilerXml.open('annotationProcessing') 4551 compilerXml.open('annotationProcessing')
4454 for processors, modules in annotationProcessorProfiles.items(): 4552 for processors, modules in sorted(annotationProcessorProfiles.iteritems()):
4455 compilerXml.open('profile', attributes={'default': 'false', 'name': '-'.join(processors), 'enabled': 'true'}) 4553 compilerXml.open('profile', attributes={'default': 'false', 'name': '-'.join(processors), 'enabled': 'true'})
4456 compilerXml.element('sourceOutputDir', attributes={'name': 'src_gen'}) # TODO use p.source_gen_dir() ? 4554 compilerXml.element('sourceOutputDir', attributes={'name': 'src_gen'}) # TODO use p.source_gen_dir() ?
4457 compilerXml.element('outputRelativeToContentRoot', attributes={'value': 'true'}) 4555 compilerXml.element('outputRelativeToContentRoot', attributes={'value': 'true'})
4458 compilerXml.open('processorPath', attributes={'useClasspath': 'false'}) 4556 compilerXml.open('processorPath', attributes={'useClasspath': 'false'})
4459 for apName in processors: 4557 for apName in processors:
5224 abort('mx: command \'{0}\' is ambiguous\n {1}'.format(command, ' '.join(hits))) 5322 abort('mx: command \'{0}\' is ambiguous\n {1}'.format(command, ' '.join(hits)))
5225 5323
5226 c, _ = _commands[command][:2] 5324 c, _ = _commands[command][:2]
5227 def term_handler(signum, frame): 5325 def term_handler(signum, frame):
5228 abort(1) 5326 abort(1)
5229 signal.signal(signal.SIGTERM, term_handler) 5327 if not is_jython():
5328 signal.signal(signal.SIGTERM, term_handler)
5230 5329
5231 def quit_handler(signum, frame): 5330 def quit_handler(signum, frame):
5232 _send_sigquit() 5331 _send_sigquit()
5233 if get_os() != 'windows': 5332 if not is_jython() and get_os() != 'windows':
5234 signal.signal(signal.SIGQUIT, quit_handler) 5333 signal.signal(signal.SIGQUIT, quit_handler)
5235 5334
5236 try: 5335 try:
5237 if opts.timeout != 0: 5336 if opts.timeout != 0:
5238 def alarm_handler(signum, frame): 5337 def alarm_handler(signum, frame):