--- a/mxtool/mx.py	Wed Jan 28 19:12:57 2015 +0100
+++ b/mxtool/mx.py	Wed Jan 28 19:32:47 2015 +0100
@@ -2,7 +2,7 @@
 #
 # ----------------------------------------------------------------------------------------------------
 #
-# Copyright (c) 2007, 2012, Oracle and/or its affiliates. All rights reserved.
+# Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved.
 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 #
 # This code is free software; you can redistribute it and/or modify it
@@ -34,7 +34,6 @@
 """
 
 import sys, os, errno, time, subprocess, shlex, types, StringIO, zipfile, signal, xml.sax.saxutils, tempfile, fnmatch, platform
-import multiprocessing
 import textwrap
 import socket
 import tarfile
@@ -62,6 +61,22 @@
         raise error
     return output
 
+# Support for jython
+def is_jython():
+    return sys.platform.startswith('java')
+
+if not is_jython():
+    import multiprocessing
+
+def cpu_count():
+    if is_jython():
+        from java.lang import Runtime
+        runtime = Runtime.getRuntime()
+        return runtime.availableProcessors()
+    else:
+        return multiprocessing.cpu_count()
+
+
 try: subprocess.check_output
 except: subprocess.check_output = check_output
 
@@ -251,6 +266,9 @@
         self.name = name
         self.suite = suite
 
+    def __cmp__(self, other):
+        return cmp(self.name, other.name)
+
     def __str__(self):
         return self.name
 
@@ -307,7 +325,7 @@
         Add the transitive set of dependencies for this project, including
         libraries if 'includeLibs' is true, to the 'deps' list.
         """
-        return self._all_deps_helper(deps, [], includeLibs, includeSelf, includeJreLibs, includeAnnotationProcessors)
+        return sorted(self._all_deps_helper(deps, [], includeLibs, includeSelf, includeJreLibs, includeAnnotationProcessors))
 
     def _all_deps_helper(self, deps, dependants, includeLibs, includeSelf=True, includeJreLibs=False, includeAnnotationProcessors=False):
         if self in dependants:
@@ -527,7 +545,7 @@
                 # Inherit annotation processors from dependencies
                 aps.update(p.annotation_processors())
 
-            self._annotationProcessors = list(aps)
+            self._annotationProcessors = sorted(list(aps))
         return self._annotationProcessors
 
     """
@@ -536,10 +554,17 @@
     """
     def annotation_processors_path(self):
         aps = [project(ap) for ap in self.annotation_processors()]
-        if len(aps):
-            return os.pathsep.join([ap.definedAnnotationProcessorsDist.path for ap in aps if ap.definedAnnotationProcessorsDist])
+        libAps = [dep for dep in self.all_deps([], includeLibs=True, includeSelf=False) if dep.isLibrary() and hasattr(dep, 'annotationProcessor') and getattr(dep, 'annotationProcessor').lower() == 'true']
+        if len(aps) + len(libAps):
+            return os.pathsep.join([ap.definedAnnotationProcessorsDist.path for ap in aps if ap.definedAnnotationProcessorsDist] + [lib.get_path(False) for lib in libAps])
         return None
 
+    def uses_annotation_processor_library(self):
+        for dep in self.all_deps([], includeLibs=True, includeSelf=False):
+            if dep.isLibrary() and hasattr(dep, 'annotationProcessor'):
+                return True
+        return False
+
     def update_current_annotation_processors_file(self):
         aps = self.annotation_processors()
         outOfDate = False
@@ -589,8 +614,9 @@
         return d.hexdigest()
 
 def download_file_with_sha1(name, path, urls, sha1, sha1path, resolve, mustExist, sources=False, canSymlink=True):
+    canSymlink = canSymlink and not (get_os() == 'windows' or get_os() == 'cygwin')
     def _download_lib():
-        cacheDir = get_env('MX_CACHE_DIR', join(_opts.user_home, '.mx', 'cache'))
+        cacheDir = _cygpathW2U(get_env('MX_CACHE_DIR', join(_opts.user_home, '.mx', 'cache')))
         if not exists(cacheDir):
             os.makedirs(cacheDir)
         base = basename(path)
@@ -680,7 +706,7 @@
         """
         if includeJreLibs and includeSelf and not self in deps:
             deps.append(self)
-        return deps
+        return sorted(deps)
 
 class Library(BaseLibrary):
     def __init__(self, suite, name, path, optional, urls, sha1, sourcePath, sourceUrls, sourceSha1, deps):
@@ -753,10 +779,10 @@
         Add the transitive set of dependencies for this library to the 'deps' list.
         """
         if not includeLibs:
-            return deps
+            return sorted(deps)
         childDeps = list(self.deps)
         if self in deps:
-            return deps
+            return sorted(deps)
         for name in childDeps:
             assert name != self.name
             dep = library(name)
@@ -764,7 +790,7 @@
                 dep.all_deps(deps, includeLibs=includeLibs, includeJreLibs=includeJreLibs, includeAnnotationProcessors=includeAnnotationProcessors)
         if not self in deps and includeSelf:
             deps.append(self)
-        return deps
+        return sorted(deps)
 
 class HgConfig:
     """
@@ -961,7 +987,7 @@
                 abort('Attribute "' + name + '" for ' + context + ' must be a list')
             return v
 
-        for name, attrs in projsMap.iteritems():
+        for name, attrs in sorted(projsMap.iteritems()):
             context = 'project ' + name
             srcDirs = pop_list(attrs, 'sourceDirs', context)
             deps = pop_list(attrs, 'dependencies', context)
@@ -983,14 +1009,14 @@
             p.__dict__.update(attrs)
             self.projects.append(p)
 
-        for name, attrs in jreLibsMap.iteritems():
+        for name, attrs in sorted(jreLibsMap.iteritems()):
             jar = attrs.pop('jar')
             # JRE libraries are optional by default
             optional = attrs.pop('optional', 'true') != 'false'
             l = JreLibrary(self, name, jar, optional)
             self.jreLibs.append(l)
 
-        for name, attrs in libsMap.iteritems():
+        for name, attrs in sorted(libsMap.iteritems()):
             context = 'library ' + name
             if "|" in name:
                 if name.count('|') != 2:
@@ -1011,7 +1037,7 @@
             l.__dict__.update(attrs)
             self.libs.append(l)
 
-        for name, attrs in distsMap.iteritems():
+        for name, attrs in sorted(distsMap.iteritems()):
             context = 'distribution ' + name
             path = attrs.pop('path')
             sourcesPath = attrs.pop('sourcesPath', None)
@@ -1289,11 +1315,29 @@
             result = result.replace('encoding="UTF-8"?>', 'encoding="UTF-8" standalone="' + str(standalone) + '"?>')
         return result
 
+def get_jython_os():
+    from java.lang import System as System
+    os_name = System.getProperty('os.name').lower()
+    if System.getProperty('isCygwin'):
+        return 'cygwin'
+    elif os_name.startswith('mac'):
+        return 'darwin'
+    elif os_name.startswith('linux'):
+        return 'linux'
+    elif os_name.startswith('sunos'):
+        return 'solaris'
+    elif os_name.startswith('win'):
+        return 'windows'
+    else:
+        abort('Unknown operating system ' + os_name)
+
 def get_os():
     """
     Get a canonical form of sys.platform.
     """
-    if sys.platform.startswith('darwin'):
+    if is_jython():
+        return get_jython_os()
+    elif sys.platform.startswith('darwin'):
         return 'darwin'
     elif sys.platform.startswith('linux'):
         return 'linux'
@@ -1313,7 +1357,7 @@
     """
     if p is None or get_os() != "cygwin":
         return p
-    return subprocess.check_output(['cygpath', '-w', p]).strip()
+    return subprocess.check_output(['cygpath', '-a', '-w', p]).strip()
 
 def _cygpathW2U(p):
     """
@@ -1322,7 +1366,7 @@
     """
     if p is None or get_os() != "cygwin":
         return p
-    return subprocess.check_output(['cygpath', '-u', p]).strip()
+    return subprocess.check_output(['cygpath', '-a', '-u', p]).strip()
 
 def _separatedCygpathU2W(p):
     """
@@ -1593,9 +1637,21 @@
         p.all_deps(deps, includeLibs=includeLibs, includeJreLibs=includeJreLibs, includeAnnotationProcessors=includeAnnotationProcessors)
     return deps
 
-def _handle_missing_java_home():
+def _handle_lookup_java_home(jdk):
+    return _handle_lookup_jdk(jdk, 'JAVA_HOME', '--java-home', False)
+
+def _handle_lookup_extra_java_homes(jdk):
+    return _handle_lookup_jdk(jdk, 'EXTRA_JAVA_HOMES', '--extra-java-homes', True)
+
+def _handle_lookup_jdk(jdk, varName, flagName, allowMultiple):
+    if jdk != None and jdk != '':
+        return jdk
+    jdk = os.environ.get(varName)
+    if jdk != None and jdk != '':
+        return jdk
+
     if not sys.stdout.isatty():
-        abort('Could not find bootstrap JDK. Use --java-home option or ensure JAVA_HOME environment variable is set.')
+        abort('Could not find bootstrap {0}. Use {1} option or ensure {2} environment variable is set.'.format(varName, flagName, varName))
 
     candidateJdks = []
     if get_os() == 'darwin':
@@ -1613,12 +1669,15 @@
 
     javaHome = None
     if len(candidateJdks) != 0:
-        javaHome = select_items(candidateJdks + ['<other>'], allowMultiple=False)
+        log('Missing value for {0}.'.format(varName))
+        javaHome = select_items(candidateJdks + ['<other>'], allowMultiple=allowMultiple)
         if javaHome == '<other>':
             javaHome = None
+        if javaHome != None and allowMultiple:
+            javaHome = os.pathsep.join(javaHome)
 
     while javaHome is None:
-        javaHome = raw_input('Enter path of bootstrap JDK: ')
+        javaHome = raw_input('Enter path of JDK for {0}: '.format(varName))
         rtJarPath = join(javaHome, 'jre', 'lib', 'rt.jar')
         if not exists(rtJarPath):
             log('Does not appear to be a valid JDK as ' + rtJarPath + ' does not exist')
@@ -1627,9 +1686,9 @@
             break
 
     envPath = join(_primary_suite.mxDir, 'env')
-    if ask_yes_no('Persist this setting by adding "JAVA_HOME=' + javaHome + '" to ' + envPath, 'y'):
+    if ask_yes_no('Persist this setting by adding "{0}={1}" to {2}'.format(varName, javaHome, envPath), 'y'):
         with open(envPath, 'a') as fp:
-            print >> fp, 'JAVA_HOME=' + javaHome
+            print >> fp, varName + '=' + javaHome
 
     return javaHome
 
@@ -1650,6 +1709,7 @@
         self.add_argument('-p', '--primary-suite-path', help='set the primary suite directory', metavar='<path>')
         self.add_argument('--dbg', type=int, dest='java_dbg_port', help='make Java processes wait on <port> for a debugger', metavar='<port>')
         self.add_argument('-d', action='store_const', const=8000, dest='java_dbg_port', help='alias for "-dbg 8000"')
+        self.add_argument('--backup-modified', action='store_true', help='backup generated files if they pre-existed and are modified')
         self.add_argument('--cp-pfx', dest='cp_prefix', help='class path prefix', metavar='<arg>')
         self.add_argument('--cp-sfx', dest='cp_suffix', help='class path suffix', metavar='<arg>')
         self.add_argument('--J', dest='java_args', help='Java VM arguments (e.g. --J @-dsa)', metavar='@<args>')
@@ -1683,13 +1743,8 @@
         if opts.very_verbose:
             opts.verbose = True
 
-        if opts.java_home is None:
-            opts.java_home = os.environ.get('JAVA_HOME')
-        if opts.extra_java_homes is None:
-            opts.extra_java_homes = os.environ.get('EXTRA_JAVA_HOMES')
-
-        if opts.java_home is None or opts.java_home == '':
-            opts.java_home = _handle_missing_java_home()
+        opts.java_home = _handle_lookup_java_home(opts.java_home)
+        opts.extra_java_homes = _handle_lookup_extra_java_homes(opts.extra_java_homes)
 
         if opts.user_home is None or opts.user_home == '':
             abort('Could not find user home. Use --user-home option or ensure HOME environment variable is set.')
@@ -1820,7 +1875,19 @@
         assert isinstance(arg, types.StringTypes), 'argument is not a string: ' + str(arg)
 
     if env is None:
-        env = os.environ
+        env = os.environ.copy()
+
+    # Ideally the command line could be communicated directly in an environment
+    # variable. However, since environment variables share the same resource
+    # space as the command line itself (on Unix at least), this would cause the
+    # limit to be exceeded too easily.
+    with tempfile.NamedTemporaryFile(suffix='', prefix='mx_subprocess_command.', mode='w', delete=False) as fp:
+        subprocessCommandFile = fp.name
+        for arg in args:
+            # TODO: handle newlines in args once there's a use case
+            assert '\n' not in arg
+            print >> fp, arg
+    env['MX_SUBPROCESS_COMMAND_FILE'] = subprocessCommandFile
 
     if _opts.verbose:
         if _opts.very_verbose:
@@ -1838,11 +1905,11 @@
         # can use os.killpg() to kill the whole subprocess group
         preexec_fn = None
         creationflags = 0
-        if get_os() == 'windows':
-            creationflags = subprocess.CREATE_NEW_PROCESS_GROUP
-        else:
-            preexec_fn = os.setsid
-
+        if not is_jython():
+            if get_os() == 'windows':
+                creationflags = subprocess.CREATE_NEW_PROCESS_GROUP
+            else:
+                preexec_fn = os.setsid
         def redirect(stream, f):
             for line in iter(stream.readline, ''):
                 f(line)
@@ -1882,6 +1949,7 @@
         abort(1)
     finally:
         _removeSubprocess(sub)
+        os.remove(subprocessCommandFile)
 
     if retcode and nonZeroIsFatal:
         if _opts.verbose:
@@ -2070,19 +2138,31 @@
         javaSource = join(myDir, 'ClasspathDump.java')
         if not exists(join(outDir, 'ClasspathDump.class')):
             subprocess.check_call([self.javac, '-d', _cygpathU2W(outDir), _cygpathU2W(javaSource)], stderr=subprocess.PIPE, stdout=subprocess.PIPE)
-        self._bootclasspath, self._extdirs, self._endorseddirs = [x if x != 'null' else None for x in subprocess.check_output([self.java, '-cp', _separatedCygpathU2W(outDir), 'ClasspathDump'], stderr=subprocess.PIPE).split('|')]
+        self._bootclasspath, self._extdirs, self._endorseddirs = [x if x != 'null' else None for x in subprocess.check_output([self.java, '-cp', _cygpathU2W(outDir), 'ClasspathDump'], stderr=subprocess.PIPE).split('|')]
         if not self._bootclasspath or not self._extdirs or not self._endorseddirs:
             warn("Could not find all classpaths: boot='" + str(self._bootclasspath) + "' extdirs='" + str(self._extdirs) + "' endorseddirs='" + str(self._endorseddirs) + "'")
         self._bootclasspath = _filter_non_existant_paths(self._bootclasspath)
         self._extdirs = _filter_non_existant_paths(self._extdirs)
         self._endorseddirs = _filter_non_existant_paths(self._endorseddirs)
 
+    def __repr__(self):
+        return "JavaConfig(" + str(self.jdk) + ", " + str(self.debug_port) + ")"
+
+    def __str__(self):
+        return "Java " + str(self.version) + " (" + str(self.javaCompliance) + ") from " + str(self.jdk)
+
     def __hash__(self):
         return hash(self.jdk)
 
     def __cmp__(self, other):
         if isinstance(other, JavaConfig):
-            return cmp(self.javaCompliance, other.javaCompliance)
+            compilanceCmp = cmp(self.javaCompliance, other.javaCompliance)
+            if compilanceCmp:
+                return compilanceCmp
+            versionCmp = cmp(self.version, other.version)
+            if versionCmp:
+                return versionCmp
+            return cmp(self.jdk, other.jdk)
         raise TypeError()
 
     def format_cmd(self, args, addDefaultArgs):
@@ -2218,7 +2298,7 @@
     def is_alive(p):
         if isinstance(p, subprocess.Popen):
             return p.poll() is None
-        assert isinstance(p, multiprocessing.Process), p
+        assert is_jython() or isinstance(p, multiprocessing.Process), p
         return p.is_alive()
 
     for p, args in _currentSubprocesses:
@@ -2278,6 +2358,9 @@
         if old == content:
             return False
 
+        if existed and _opts.backup_modified:
+            shutil.move(path, path + '.orig')
+
         with open(path, 'wb') as f:
             f.write(content)
 
@@ -2381,7 +2464,10 @@
                     with open(jdtProperties) as fp:
                         origContent = fp.read()
                         content = origContent
-                        if args.jdt_warning_as_error:
+                        if self.proj.uses_annotation_processor_library():
+                            # unfortunately, the command line compiler doesn't let us ignore warnings for generated files only
+                            content = content.replace('=warning', '=ignore')
+                        elif args.jdt_warning_as_error:
                             content = content.replace('=warning', '=error')
                         if not args.jdt_show_task_tags:
                             content = content + '\norg.eclipse.jdt.core.compiler.problem.tasks=ignore'
@@ -2424,7 +2510,7 @@
     parser = parser if parser is not None else ArgumentParser(prog='mx build')
     parser.add_argument('-f', action='store_true', dest='force', help='force build (disables timestamp checking)')
     parser.add_argument('-c', action='store_true', dest='clean', help='removes existing build output')
-    parser.add_argument('-p', action='store_true', dest='parallelize', help='parallelizes Java compilation')
+    parser.add_argument('-p', action='store_true', dest='parallelize', help='parallelizes Java compilation if possible')
     parser.add_argument('--source', dest='compliance', help='Java compliance level for projects without an explicit one')
     parser.add_argument('--Wapi', action='store_true', dest='warnAPI', help='show warnings about using internal APIs')
     parser.add_argument('--projects', action='store', help='comma separated projects to build (omit to build all projects)')
@@ -2444,6 +2530,11 @@
 
     args = parser.parse_args(args)
 
+    if is_jython():
+        if args.parallelize:
+            logv('[multiprocessing not available in jython]')
+            args.parallelize = False
+
     jdtJar = None
     if not args.javac and args.jdt is not None:
         if not args.jdt.endswith('.jar'):
@@ -2600,7 +2691,7 @@
                 t._d = None
             return sorted(tasks, compareTasks)
 
-        cpus = multiprocessing.cpu_count()
+        cpus = cpu_count()
         worklist = sortWorklist(tasks.values())
         active = []
         failed = []
@@ -3523,11 +3614,19 @@
             os.mkdir(srcDir)
         out.element('classpathentry', {'kind' : 'src', 'path' : src})
 
-    if len(p.annotation_processors()) > 0:
+    processorPath = p.annotation_processors_path()
+    if processorPath:
         genDir = p.source_gen_dir()
         if not exists(genDir):
             os.mkdir(genDir)
-        out.element('classpathentry', {'kind' : 'src', 'path' : 'src_gen'})
+        out.open('classpathentry', {'kind' : 'src', 'path' : 'src_gen'})
+        if p.uses_annotation_processor_library():
+            # ignore warnings produced by third-party annotation processors
+            out.open('attributes')
+            out.element('attribute', {'name' : 'ignore_optional_problems', 'value' : 'true'})
+            out.close('attributes')
+        out.close('classpathentry')
+
         if files:
             files.append(genDir)
 
@@ -3554,10 +3653,10 @@
         elif dep.isProject():
             projectDeps.add(dep)
 
-    for dep in containerDeps:
+    for dep in sorted(containerDeps):
         out.element('classpathentry', {'exported' : 'true', 'kind' : 'con', 'path' : dep})
 
-    for dep in libraryDeps:
+    for dep in sorted(libraryDeps):
         path = dep.path
         dep.get_path(resolve=True)
 
@@ -3575,7 +3674,7 @@
         if libFiles:
             libFiles.append(path)
 
-    for dep in projectDeps:
+    for dep in sorted(projectDeps):
         out.element('classpathentry', {'combineaccessrules' : 'false', 'exported' : 'true', 'kind' : 'src', 'path' : '/' + dep.name})
 
     out.element('classpathentry', {'kind' : 'output', 'path' : getattr(p, 'eclipse.output', 'bin')})
@@ -3707,19 +3806,18 @@
     # copy a possibly modified file to the project's .settings directory
     for name, path in esdict.iteritems():
         # ignore this file altogether if this project has no annotation processors
-        if name == "org.eclipse.jdt.apt.core.prefs" and not len(p.annotation_processors()) > 0:
+        if name == "org.eclipse.jdt.apt.core.prefs" and not processorPath:
             continue
 
         with open(path) as f:
             content = f.read()
         content = content.replace('${javaCompliance}', str(p.javaCompliance))
-        if len(p.annotation_processors()) > 0:
+        if processorPath:
             content = content.replace('org.eclipse.jdt.core.compiler.processAnnotations=disabled', 'org.eclipse.jdt.core.compiler.processAnnotations=enabled')
         update_file(join(settingsDir, name), content)
         if files:
             files.append(join(settingsDir, name))
 
-    processorPath = p.annotation_processors_path()
     if processorPath:
         out = XMLDoc()
         out.open('factorypath')
@@ -4451,7 +4549,7 @@
 
     if annotationProcessorProfiles:
         compilerXml.open('annotationProcessing')
-        for processors, modules in annotationProcessorProfiles.items():
+        for processors, modules in sorted(annotationProcessorProfiles.iteritems()):
             compilerXml.open('profile', attributes={'default': 'false', 'name': '-'.join(processors), 'enabled': 'true'})
             compilerXml.element('sourceOutputDir', attributes={'name': 'src_gen'})  # TODO use p.source_gen_dir() ?
             compilerXml.element('outputRelativeToContentRoot', attributes={'value': 'true'})
@@ -5226,11 +5324,12 @@
     c, _ = _commands[command][:2]
     def term_handler(signum, frame):
         abort(1)
-    signal.signal(signal.SIGTERM, term_handler)
+    if not is_jython():
+        signal.signal(signal.SIGTERM, term_handler)
 
     def quit_handler(signum, frame):
         _send_sigquit()
-    if get_os() != 'windows':
+    if not is_jython() and get_os() != 'windows':
         signal.signal(signal.SIGQUIT, quit_handler)
 
     try: