# HG changeset patch # User Doug Simon # Date 1421490378 -3600 # Node ID c446d00f2cdfdbe2788d1a1420ab49a1af919e9e # Parent d71bf1009d71dbf2d6e337f00366e4b08caac21d mx: added support for Jython 2.7b3 and made ordering in generated files more deterministic\nContributed-by: Igor Ignatyev diff -r d71bf1009d71 -r c446d00f2cdf mx/mx_graal.py --- a/mx/mx_graal.py Fri Jan 16 14:24:04 2015 +0100 +++ b/mx/mx_graal.py Sat Jan 17 11:26:18 2015 +0100 @@ -26,7 +26,7 @@ # # ---------------------------------------------------------------------------------------------------- -import os, stat, errno, sys, shutil, zipfile, tarfile, tempfile, re, time, datetime, platform, subprocess, multiprocessing, StringIO, socket +import os, stat, errno, sys, shutil, zipfile, tarfile, tempfile, re, time, datetime, platform, subprocess, StringIO, socket from os.path import join, exists, dirname, basename from argparse import ArgumentParser, RawDescriptionHelpFormatter, REMAINDER from outputparser import OutputParser, ValuesMatcher @@ -697,7 +697,7 @@ buildVars = { 'ALT_BOOTDIR' : 'The location of the bootstrap JDK installation (default: ' + mx.java().jdk + ')', 'ALT_OUTPUTDIR' : 'Build directory', - 'HOTSPOT_BUILD_JOBS' : 'Number of CPUs used by make (default: ' + str(multiprocessing.cpu_count()) + ')', + 'HOTSPOT_BUILD_JOBS' : 'Number of CPUs used by make (default: ' + str(mx.cpu_count()) + ')', 'INSTALL' : 'Install the built VM into the JDK? (default: y)', 'ZIP_DEBUGINFO_FILES' : 'Install zipped debug symbols file? (default: 0)', } @@ -888,7 +888,7 @@ mx.log('Error building project') return else: - cpus = multiprocessing.cpu_count() + cpus = mx.cpu_count() makeDir = join(_graal_home, 'make') runCmd = [mx.gmake_cmd(), '-C', makeDir] diff -r d71bf1009d71 -r c446d00f2cdf mxtool/mx.py --- a/mxtool/mx.py Fri Jan 16 14:24:04 2015 +0100 +++ b/mxtool/mx.py Sat Jan 17 11:26:18 2015 +0100 @@ -34,7 +34,6 @@ """ import sys, os, errno, time, subprocess, shlex, types, StringIO, zipfile, signal, xml.sax.saxutils, tempfile, fnmatch, platform -import multiprocessing import textwrap import socket import tarfile @@ -62,6 +61,22 @@ raise error return output +# Support for jython +def is_jython(): + return sys.platform.startswith('java') + +if not is_jython(): + import multiprocessing + +def cpu_count(): + if is_jython(): + from java.lang import Runtime + runtime = Runtime.getRuntime() + return runtime.availableProcessors() + else: + return multiprocessing.cpu_count() + + try: subprocess.check_output except: subprocess.check_output = check_output @@ -251,6 +266,9 @@ self.name = name self.suite = suite + def __cmp__(self, other): + return cmp(self.name, other.name) + def __str__(self): return self.name @@ -307,7 +325,7 @@ Add the transitive set of dependencies for this project, including libraries if 'includeLibs' is true, to the 'deps' list. """ - return self._all_deps_helper(deps, [], includeLibs, includeSelf, includeJreLibs, includeAnnotationProcessors) + return sorted(self._all_deps_helper(deps, [], includeLibs, includeSelf, includeJreLibs, includeAnnotationProcessors)) def _all_deps_helper(self, deps, dependants, includeLibs, includeSelf=True, includeJreLibs=False, includeAnnotationProcessors=False): if self in dependants: @@ -527,7 +545,7 @@ # Inherit annotation processors from dependencies aps.update(p.annotation_processors()) - self._annotationProcessors = list(aps) + self._annotationProcessors = sorted(list(aps)) return self._annotationProcessors """ @@ -589,8 +607,9 @@ return d.hexdigest() def download_file_with_sha1(name, path, urls, sha1, sha1path, resolve, mustExist, sources=False, canSymlink=True): + canSymlink = canSymlink and not (get_os() == 'windows' or get_os() == 'cygwin') def _download_lib(): - cacheDir = get_env('MX_CACHE_DIR', join(_opts.user_home, '.mx', 'cache')) + cacheDir = _cygpathW2U(get_env('MX_CACHE_DIR', join(_opts.user_home, '.mx', 'cache'))) if not exists(cacheDir): os.makedirs(cacheDir) base = basename(path) @@ -680,7 +699,7 @@ """ if includeJreLibs and includeSelf and not self in deps: deps.append(self) - return deps + return sorted(deps) class Library(BaseLibrary): def __init__(self, suite, name, path, optional, urls, sha1, sourcePath, sourceUrls, sourceSha1, deps): @@ -753,10 +772,10 @@ Add the transitive set of dependencies for this library to the 'deps' list. """ if not includeLibs: - return deps + return sorted(deps) childDeps = list(self.deps) if self in deps: - return deps + return sorted(deps) for name in childDeps: assert name != self.name dep = library(name) @@ -764,7 +783,7 @@ dep.all_deps(deps, includeLibs=includeLibs, includeJreLibs=includeJreLibs, includeAnnotationProcessors=includeAnnotationProcessors) if not self in deps and includeSelf: deps.append(self) - return deps + return sorted(deps) class HgConfig: """ @@ -961,7 +980,7 @@ abort('Attribute "' + name + '" for ' + context + ' must be a list') return v - for name, attrs in projsMap.iteritems(): + for name, attrs in sorted(projsMap.iteritems()): context = 'project ' + name srcDirs = pop_list(attrs, 'sourceDirs', context) deps = pop_list(attrs, 'dependencies', context) @@ -983,14 +1002,14 @@ p.__dict__.update(attrs) self.projects.append(p) - for name, attrs in jreLibsMap.iteritems(): + for name, attrs in sorted(jreLibsMap.iteritems()): jar = attrs.pop('jar') # JRE libraries are optional by default optional = attrs.pop('optional', 'true') != 'false' l = JreLibrary(self, name, jar, optional) self.jreLibs.append(l) - for name, attrs in libsMap.iteritems(): + for name, attrs in sorted(libsMap.iteritems()): context = 'library ' + name if "|" in name: if name.count('|') != 2: @@ -1011,7 +1030,7 @@ l.__dict__.update(attrs) self.libs.append(l) - for name, attrs in distsMap.iteritems(): + for name, attrs in sorted(distsMap.iteritems()): context = 'distribution ' + name path = attrs.pop('path') sourcesPath = attrs.pop('sourcesPath', None) @@ -1289,11 +1308,29 @@ result = result.replace('encoding="UTF-8"?>', 'encoding="UTF-8" standalone="' + str(standalone) + '"?>') return result +def get_jython_os(): + from java.lang import System as System + os_name = System.getProperty('os.name').lower() + if System.getProperty('isCygwin'): + return 'cygwin' + elif os_name.startswith('mac'): + return 'darwin' + elif os_name.startswith('linux'): + return 'linux' + elif os_name.startswith('sunos'): + return 'solaris' + elif os_name.startswith('win'): + return 'windows' + else: + abort('Unknown operating system ' + os_name) + def get_os(): """ Get a canonical form of sys.platform. """ - if sys.platform.startswith('darwin'): + if is_jython(): + return get_jython_os() + elif sys.platform.startswith('darwin'): return 'darwin' elif sys.platform.startswith('linux'): return 'linux' @@ -1313,7 +1350,7 @@ """ if p is None or get_os() != "cygwin": return p - return subprocess.check_output(['cygpath', '-w', p]).strip() + return subprocess.check_output(['cygpath', '-a', '-w', p]).strip() def _cygpathW2U(p): """ @@ -1322,7 +1359,7 @@ """ if p is None or get_os() != "cygwin": return p - return subprocess.check_output(['cygpath', '-u', p]).strip() + return subprocess.check_output(['cygpath', '-a', '-u', p]).strip() def _separatedCygpathU2W(p): """ @@ -1665,6 +1702,7 @@ self.add_argument('-p', '--primary-suite-path', help='set the primary suite directory', metavar='') self.add_argument('--dbg', type=int, dest='java_dbg_port', help='make Java processes wait on for a debugger', metavar='') self.add_argument('-d', action='store_const', const=8000, dest='java_dbg_port', help='alias for "-dbg 8000"') + self.add_argument('--backup-modified', action='store_true', help='backup generated files if they pre-existed and are modified') self.add_argument('--cp-pfx', dest='cp_prefix', help='class path prefix', metavar='') self.add_argument('--cp-sfx', dest='cp_suffix', help='class path suffix', metavar='') self.add_argument('--J', dest='java_args', help='Java VM arguments (e.g. --J @-dsa)', metavar='@') @@ -1848,11 +1886,11 @@ # can use os.killpg() to kill the whole subprocess group preexec_fn = None creationflags = 0 - if get_os() == 'windows': - creationflags = subprocess.CREATE_NEW_PROCESS_GROUP - else: - preexec_fn = os.setsid - + if not is_jython(): + if get_os() == 'windows': + creationflags = subprocess.CREATE_NEW_PROCESS_GROUP + else: + preexec_fn = os.setsid def redirect(stream, f): for line in iter(stream.readline, ''): f(line) @@ -2080,7 +2118,7 @@ javaSource = join(myDir, 'ClasspathDump.java') if not exists(join(outDir, 'ClasspathDump.class')): subprocess.check_call([self.javac, '-d', _cygpathU2W(outDir), _cygpathU2W(javaSource)], stderr=subprocess.PIPE, stdout=subprocess.PIPE) - self._bootclasspath, self._extdirs, self._endorseddirs = [x if x != 'null' else None for x in subprocess.check_output([self.java, '-cp', _separatedCygpathU2W(outDir), 'ClasspathDump'], stderr=subprocess.PIPE).split('|')] + self._bootclasspath, self._extdirs, self._endorseddirs = [x if x != 'null' else None for x in subprocess.check_output([self.java, '-cp', _cygpathU2W(outDir), 'ClasspathDump'], stderr=subprocess.PIPE).split('|')] if not self._bootclasspath or not self._extdirs or not self._endorseddirs: warn("Could not find all classpaths: boot='" + str(self._bootclasspath) + "' extdirs='" + str(self._extdirs) + "' endorseddirs='" + str(self._endorseddirs) + "'") self._bootclasspath = _filter_non_existant_paths(self._bootclasspath) @@ -2228,7 +2266,7 @@ def is_alive(p): if isinstance(p, subprocess.Popen): return p.poll() is None - assert isinstance(p, multiprocessing.Process), p + assert is_jython() or isinstance(p, multiprocessing.Process), p return p.is_alive() for p, args in _currentSubprocesses: @@ -2288,6 +2326,9 @@ if old == content: return False + if existed and _opts.backup_modified: + shutil.move(path, path + '.orig') + with open(path, 'wb') as f: f.write(content) @@ -2434,7 +2475,7 @@ parser = parser if parser is not None else ArgumentParser(prog='mx build') parser.add_argument('-f', action='store_true', dest='force', help='force build (disables timestamp checking)') parser.add_argument('-c', action='store_true', dest='clean', help='removes existing build output') - parser.add_argument('-p', action='store_true', dest='parallelize', help='parallelizes Java compilation') + parser.add_argument('-p', action='store_true', dest='parallelize', help='parallelizes Java compilation if possible') parser.add_argument('--source', dest='compliance', help='Java compliance level for projects without an explicit one') parser.add_argument('--Wapi', action='store_true', dest='warnAPI', help='show warnings about using internal APIs') parser.add_argument('--projects', action='store', help='comma separated projects to build (omit to build all projects)') @@ -2454,6 +2495,11 @@ args = parser.parse_args(args) + if is_jython(): + if args.parallelize: + logv('[multiprocessing not available in jython]') + args.parallelize = False + jdtJar = None if not args.javac and args.jdt is not None: if not args.jdt.endswith('.jar'): @@ -2610,7 +2656,7 @@ t._d = None return sorted(tasks, compareTasks) - cpus = multiprocessing.cpu_count() + cpus = cpu_count() worklist = sortWorklist(tasks.values()) active = [] failed = [] @@ -3564,10 +3610,10 @@ elif dep.isProject(): projectDeps.add(dep) - for dep in containerDeps: + for dep in sorted(containerDeps): out.element('classpathentry', {'exported' : 'true', 'kind' : 'con', 'path' : dep}) - for dep in libraryDeps: + for dep in sorted(libraryDeps): path = dep.path dep.get_path(resolve=True) @@ -3585,7 +3631,7 @@ if libFiles: libFiles.append(path) - for dep in projectDeps: + for dep in sorted(projectDeps): out.element('classpathentry', {'combineaccessrules' : 'false', 'exported' : 'true', 'kind' : 'src', 'path' : '/' + dep.name}) out.element('classpathentry', {'kind' : 'output', 'path' : getattr(p, 'eclipse.output', 'bin')}) @@ -4461,7 +4507,7 @@ if annotationProcessorProfiles: compilerXml.open('annotationProcessing') - for processors, modules in annotationProcessorProfiles.items(): + for processors, modules in sorted(annotationProcessorProfiles.iteritems()): compilerXml.open('profile', attributes={'default': 'false', 'name': '-'.join(processors), 'enabled': 'true'}) compilerXml.element('sourceOutputDir', attributes={'name': 'src_gen'}) # TODO use p.source_gen_dir() ? compilerXml.element('outputRelativeToContentRoot', attributes={'value': 'true'}) @@ -5236,11 +5282,12 @@ c, _ = _commands[command][:2] def term_handler(signum, frame): abort(1) - signal.signal(signal.SIGTERM, term_handler) + if not is_jython(): + signal.signal(signal.SIGTERM, term_handler) def quit_handler(signum, frame): _send_sigquit() - if get_os() != 'windows': + if not is_jython() and get_os() != 'windows': signal.signal(signal.SIGQUIT, quit_handler) try: