changeset 18876:c446d00f2cdf

mx: added support for Jython 2.7b3 and made ordering in generated files more deterministic\nContributed-by: Igor Ignatyev <igor.ignatyev@oracle.com>
author Doug Simon <doug.simon@oracle.com>
date Sat, 17 Jan 2015 11:26:18 +0100
parents d71bf1009d71
children 1ca36602c05b d0f34a3055cf
files mx/mx_graal.py mxtool/mx.py
diffstat 2 files changed, 80 insertions(+), 33 deletions(-) [+]
line wrap: on
line diff
--- a/mx/mx_graal.py	Fri Jan 16 14:24:04 2015 +0100
+++ b/mx/mx_graal.py	Sat Jan 17 11:26:18 2015 +0100
@@ -26,7 +26,7 @@
 #
 # ----------------------------------------------------------------------------------------------------
 
-import os, stat, errno, sys, shutil, zipfile, tarfile, tempfile, re, time, datetime, platform, subprocess, multiprocessing, StringIO, socket
+import os, stat, errno, sys, shutil, zipfile, tarfile, tempfile, re, time, datetime, platform, subprocess, StringIO, socket
 from os.path import join, exists, dirname, basename
 from argparse import ArgumentParser, RawDescriptionHelpFormatter, REMAINDER
 from outputparser import OutputParser, ValuesMatcher
@@ -697,7 +697,7 @@
     buildVars = {
         'ALT_BOOTDIR' : 'The location of the bootstrap JDK installation (default: ' + mx.java().jdk + ')',
         'ALT_OUTPUTDIR' : 'Build directory',
-        'HOTSPOT_BUILD_JOBS' : 'Number of CPUs used by make (default: ' + str(multiprocessing.cpu_count()) + ')',
+        'HOTSPOT_BUILD_JOBS' : 'Number of CPUs used by make (default: ' + str(mx.cpu_count()) + ')',
         'INSTALL' : 'Install the built VM into the JDK? (default: y)',
         'ZIP_DEBUGINFO_FILES' : 'Install zipped debug symbols file? (default: 0)',
     }
@@ -888,7 +888,7 @@
                 mx.log('Error building project')
                 return
         else:
-            cpus = multiprocessing.cpu_count()
+            cpus = mx.cpu_count()
             makeDir = join(_graal_home, 'make')
             runCmd = [mx.gmake_cmd(), '-C', makeDir]
 
--- a/mxtool/mx.py	Fri Jan 16 14:24:04 2015 +0100
+++ b/mxtool/mx.py	Sat Jan 17 11:26:18 2015 +0100
@@ -34,7 +34,6 @@
 """
 
 import sys, os, errno, time, subprocess, shlex, types, StringIO, zipfile, signal, xml.sax.saxutils, tempfile, fnmatch, platform
-import multiprocessing
 import textwrap
 import socket
 import tarfile
@@ -62,6 +61,22 @@
         raise error
     return output
 
+# Support for jython
+def is_jython():
+    return sys.platform.startswith('java')
+
+if not is_jython():
+    import multiprocessing
+
+def cpu_count():
+    if is_jython():
+        from java.lang import Runtime
+        runtime = Runtime.getRuntime()
+        return runtime.availableProcessors()
+    else:
+        return multiprocessing.cpu_count()
+
+
 try: subprocess.check_output
 except: subprocess.check_output = check_output
 
@@ -251,6 +266,9 @@
         self.name = name
         self.suite = suite
 
+    def __cmp__(self, other):
+        return cmp(self.name, other.name)
+
     def __str__(self):
         return self.name
 
@@ -307,7 +325,7 @@
         Add the transitive set of dependencies for this project, including
         libraries if 'includeLibs' is true, to the 'deps' list.
         """
-        return self._all_deps_helper(deps, [], includeLibs, includeSelf, includeJreLibs, includeAnnotationProcessors)
+        return sorted(self._all_deps_helper(deps, [], includeLibs, includeSelf, includeJreLibs, includeAnnotationProcessors))
 
     def _all_deps_helper(self, deps, dependants, includeLibs, includeSelf=True, includeJreLibs=False, includeAnnotationProcessors=False):
         if self in dependants:
@@ -527,7 +545,7 @@
                 # Inherit annotation processors from dependencies
                 aps.update(p.annotation_processors())
 
-            self._annotationProcessors = list(aps)
+            self._annotationProcessors = sorted(list(aps))
         return self._annotationProcessors
 
     """
@@ -589,8 +607,9 @@
         return d.hexdigest()
 
 def download_file_with_sha1(name, path, urls, sha1, sha1path, resolve, mustExist, sources=False, canSymlink=True):
+    canSymlink = canSymlink and not (get_os() == 'windows' or get_os() == 'cygwin')
     def _download_lib():
-        cacheDir = get_env('MX_CACHE_DIR', join(_opts.user_home, '.mx', 'cache'))
+        cacheDir = _cygpathW2U(get_env('MX_CACHE_DIR', join(_opts.user_home, '.mx', 'cache')))
         if not exists(cacheDir):
             os.makedirs(cacheDir)
         base = basename(path)
@@ -680,7 +699,7 @@
         """
         if includeJreLibs and includeSelf and not self in deps:
             deps.append(self)
-        return deps
+        return sorted(deps)
 
 class Library(BaseLibrary):
     def __init__(self, suite, name, path, optional, urls, sha1, sourcePath, sourceUrls, sourceSha1, deps):
@@ -753,10 +772,10 @@
         Add the transitive set of dependencies for this library to the 'deps' list.
         """
         if not includeLibs:
-            return deps
+            return sorted(deps)
         childDeps = list(self.deps)
         if self in deps:
-            return deps
+            return sorted(deps)
         for name in childDeps:
             assert name != self.name
             dep = library(name)
@@ -764,7 +783,7 @@
                 dep.all_deps(deps, includeLibs=includeLibs, includeJreLibs=includeJreLibs, includeAnnotationProcessors=includeAnnotationProcessors)
         if not self in deps and includeSelf:
             deps.append(self)
-        return deps
+        return sorted(deps)
 
 class HgConfig:
     """
@@ -961,7 +980,7 @@
                 abort('Attribute "' + name + '" for ' + context + ' must be a list')
             return v
 
-        for name, attrs in projsMap.iteritems():
+        for name, attrs in sorted(projsMap.iteritems()):
             context = 'project ' + name
             srcDirs = pop_list(attrs, 'sourceDirs', context)
             deps = pop_list(attrs, 'dependencies', context)
@@ -983,14 +1002,14 @@
             p.__dict__.update(attrs)
             self.projects.append(p)
 
-        for name, attrs in jreLibsMap.iteritems():
+        for name, attrs in sorted(jreLibsMap.iteritems()):
             jar = attrs.pop('jar')
             # JRE libraries are optional by default
             optional = attrs.pop('optional', 'true') != 'false'
             l = JreLibrary(self, name, jar, optional)
             self.jreLibs.append(l)
 
-        for name, attrs in libsMap.iteritems():
+        for name, attrs in sorted(libsMap.iteritems()):
             context = 'library ' + name
             if "|" in name:
                 if name.count('|') != 2:
@@ -1011,7 +1030,7 @@
             l.__dict__.update(attrs)
             self.libs.append(l)
 
-        for name, attrs in distsMap.iteritems():
+        for name, attrs in sorted(distsMap.iteritems()):
             context = 'distribution ' + name
             path = attrs.pop('path')
             sourcesPath = attrs.pop('sourcesPath', None)
@@ -1289,11 +1308,29 @@
             result = result.replace('encoding="UTF-8"?>', 'encoding="UTF-8" standalone="' + str(standalone) + '"?>')
         return result
 
+def get_jython_os():
+    from java.lang import System as System
+    os_name = System.getProperty('os.name').lower()
+    if System.getProperty('isCygwin'):
+        return 'cygwin'
+    elif os_name.startswith('mac'):
+        return 'darwin'
+    elif os_name.startswith('linux'):
+        return 'linux'
+    elif os_name.startswith('sunos'):
+        return 'solaris'
+    elif os_name.startswith('win'):
+        return 'windows'
+    else:
+        abort('Unknown operating system ' + os_name)
+
 def get_os():
     """
     Get a canonical form of sys.platform.
     """
-    if sys.platform.startswith('darwin'):
+    if is_jython():
+        return get_jython_os()
+    elif sys.platform.startswith('darwin'):
         return 'darwin'
     elif sys.platform.startswith('linux'):
         return 'linux'
@@ -1313,7 +1350,7 @@
     """
     if p is None or get_os() != "cygwin":
         return p
-    return subprocess.check_output(['cygpath', '-w', p]).strip()
+    return subprocess.check_output(['cygpath', '-a', '-w', p]).strip()
 
 def _cygpathW2U(p):
     """
@@ -1322,7 +1359,7 @@
     """
     if p is None or get_os() != "cygwin":
         return p
-    return subprocess.check_output(['cygpath', '-u', p]).strip()
+    return subprocess.check_output(['cygpath', '-a', '-u', p]).strip()
 
 def _separatedCygpathU2W(p):
     """
@@ -1665,6 +1702,7 @@
         self.add_argument('-p', '--primary-suite-path', help='set the primary suite directory', metavar='<path>')
         self.add_argument('--dbg', type=int, dest='java_dbg_port', help='make Java processes wait on <port> for a debugger', metavar='<port>')
         self.add_argument('-d', action='store_const', const=8000, dest='java_dbg_port', help='alias for "-dbg 8000"')
+        self.add_argument('--backup-modified', action='store_true', help='backup generated files if they pre-existed and are modified')
         self.add_argument('--cp-pfx', dest='cp_prefix', help='class path prefix', metavar='<arg>')
         self.add_argument('--cp-sfx', dest='cp_suffix', help='class path suffix', metavar='<arg>')
         self.add_argument('--J', dest='java_args', help='Java VM arguments (e.g. --J @-dsa)', metavar='@<args>')
@@ -1848,11 +1886,11 @@
         # can use os.killpg() to kill the whole subprocess group
         preexec_fn = None
         creationflags = 0
-        if get_os() == 'windows':
-            creationflags = subprocess.CREATE_NEW_PROCESS_GROUP
-        else:
-            preexec_fn = os.setsid
-
+        if not is_jython():
+            if get_os() == 'windows':
+                creationflags = subprocess.CREATE_NEW_PROCESS_GROUP
+            else:
+                preexec_fn = os.setsid
         def redirect(stream, f):
             for line in iter(stream.readline, ''):
                 f(line)
@@ -2080,7 +2118,7 @@
         javaSource = join(myDir, 'ClasspathDump.java')
         if not exists(join(outDir, 'ClasspathDump.class')):
             subprocess.check_call([self.javac, '-d', _cygpathU2W(outDir), _cygpathU2W(javaSource)], stderr=subprocess.PIPE, stdout=subprocess.PIPE)
-        self._bootclasspath, self._extdirs, self._endorseddirs = [x if x != 'null' else None for x in subprocess.check_output([self.java, '-cp', _separatedCygpathU2W(outDir), 'ClasspathDump'], stderr=subprocess.PIPE).split('|')]
+        self._bootclasspath, self._extdirs, self._endorseddirs = [x if x != 'null' else None for x in subprocess.check_output([self.java, '-cp', _cygpathU2W(outDir), 'ClasspathDump'], stderr=subprocess.PIPE).split('|')]
         if not self._bootclasspath or not self._extdirs or not self._endorseddirs:
             warn("Could not find all classpaths: boot='" + str(self._bootclasspath) + "' extdirs='" + str(self._extdirs) + "' endorseddirs='" + str(self._endorseddirs) + "'")
         self._bootclasspath = _filter_non_existant_paths(self._bootclasspath)
@@ -2228,7 +2266,7 @@
     def is_alive(p):
         if isinstance(p, subprocess.Popen):
             return p.poll() is None
-        assert isinstance(p, multiprocessing.Process), p
+        assert is_jython() or isinstance(p, multiprocessing.Process), p
         return p.is_alive()
 
     for p, args in _currentSubprocesses:
@@ -2288,6 +2326,9 @@
         if old == content:
             return False
 
+        if existed and _opts.backup_modified:
+            shutil.move(path, path + '.orig')
+
         with open(path, 'wb') as f:
             f.write(content)
 
@@ -2434,7 +2475,7 @@
     parser = parser if parser is not None else ArgumentParser(prog='mx build')
     parser.add_argument('-f', action='store_true', dest='force', help='force build (disables timestamp checking)')
     parser.add_argument('-c', action='store_true', dest='clean', help='removes existing build output')
-    parser.add_argument('-p', action='store_true', dest='parallelize', help='parallelizes Java compilation')
+    parser.add_argument('-p', action='store_true', dest='parallelize', help='parallelizes Java compilation if possible')
     parser.add_argument('--source', dest='compliance', help='Java compliance level for projects without an explicit one')
     parser.add_argument('--Wapi', action='store_true', dest='warnAPI', help='show warnings about using internal APIs')
     parser.add_argument('--projects', action='store', help='comma separated projects to build (omit to build all projects)')
@@ -2454,6 +2495,11 @@
 
     args = parser.parse_args(args)
 
+    if is_jython():
+        if args.parallelize:
+            logv('[multiprocessing not available in jython]')
+            args.parallelize = False
+
     jdtJar = None
     if not args.javac and args.jdt is not None:
         if not args.jdt.endswith('.jar'):
@@ -2610,7 +2656,7 @@
                 t._d = None
             return sorted(tasks, compareTasks)
 
-        cpus = multiprocessing.cpu_count()
+        cpus = cpu_count()
         worklist = sortWorklist(tasks.values())
         active = []
         failed = []
@@ -3564,10 +3610,10 @@
         elif dep.isProject():
             projectDeps.add(dep)
 
-    for dep in containerDeps:
+    for dep in sorted(containerDeps):
         out.element('classpathentry', {'exported' : 'true', 'kind' : 'con', 'path' : dep})
 
-    for dep in libraryDeps:
+    for dep in sorted(libraryDeps):
         path = dep.path
         dep.get_path(resolve=True)
 
@@ -3585,7 +3631,7 @@
         if libFiles:
             libFiles.append(path)
 
-    for dep in projectDeps:
+    for dep in sorted(projectDeps):
         out.element('classpathentry', {'combineaccessrules' : 'false', 'exported' : 'true', 'kind' : 'src', 'path' : '/' + dep.name})
 
     out.element('classpathentry', {'kind' : 'output', 'path' : getattr(p, 'eclipse.output', 'bin')})
@@ -4461,7 +4507,7 @@
 
     if annotationProcessorProfiles:
         compilerXml.open('annotationProcessing')
-        for processors, modules in annotationProcessorProfiles.items():
+        for processors, modules in sorted(annotationProcessorProfiles.iteritems()):
             compilerXml.open('profile', attributes={'default': 'false', 'name': '-'.join(processors), 'enabled': 'true'})
             compilerXml.element('sourceOutputDir', attributes={'name': 'src_gen'})  # TODO use p.source_gen_dir() ?
             compilerXml.element('outputRelativeToContentRoot', attributes={'value': 'true'})
@@ -5236,11 +5282,12 @@
     c, _ = _commands[command][:2]
     def term_handler(signum, frame):
         abort(1)
-    signal.signal(signal.SIGTERM, term_handler)
+    if not is_jython():
+        signal.signal(signal.SIGTERM, term_handler)
 
     def quit_handler(signum, frame):
         _send_sigquit()
-    if get_os() != 'windows':
+    if not is_jython() and get_os() != 'windows':
         signal.signal(signal.SIGQUIT, quit_handler)
 
     try: