Mercurial > hg > truffle
changeset 3723:6c5f528c7aac
Added a copy of the mxtool to repo.
| author | Doug Simon <doug.simon@oracle.com> |
|---|---|
| date | Fri, 16 Dec 2011 16:46:33 +0100 |
| parents | 7c5524a4e86e |
| children | 44445a59a718 |
| files | mxtool/README mxtool/URLConnectionDownload.java mxtool/mx mxtool/mx.py |
| diffstat | 4 files changed, 1289 insertions(+), 0 deletions(-) [+] |
line wrap: on
line diff
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mxtool/README Fri Dec 16 16:46:33 2011 +0100 @@ -0,0 +1,6 @@ +This is as a read-only copy of the mxtool (https://hg.kenai.com/hg/maxine~mxtool). +Having a copy makes the GraalVM repo more self-contained so that every revision +will always be built with a specific version of this tool, ensuring repeatability. + +Changes to the tool should be made in a clone of the Kenai repo, pushed back +to Kenai and copied here.
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mxtool/URLConnectionDownload.java Fri Dec 16 16:46:33 2011 +0100 @@ -0,0 +1,114 @@ +/* + * Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +import java.io.*; +import java.net.*; +import java.util.*; +import java.util.regex.*; + +/** + * Downloads content from a given URL to a given file. + * + * @param path where to write the content + * @param urls the URLs to try, stopping after the first successful one + */ +public class URLConnectionDownload { + + /** + * Downloads content from a given URL to a given file. + * + * @param args + * arg[0] is the path where to write the content. The remainder + * of args are the URLs to try, stopping after the first + * successful one + */ + public static void main(String[] args) { + File path = new File(args[0]); + String[] urls = new String[args.length - 1]; + System.arraycopy(args, 1, urls, 0, urls.length); + + File parent = path.getParentFile(); + makeDirectory(parent); + + // Enable use of system proxies + System.setProperty("java.net.useSystemProxies", "true"); + + String proxy = System.getenv("HTTP_PROXY"); + String proxyMsg = ""; + if (proxy != null) { + Pattern p = Pattern.compile("(?:http://)?([^:]+)(:\\d+)?"); + Matcher m = p.matcher(proxy); + if (m.matches()) { + String host = m.group(1); + String port = m.group(2); + System.setProperty("http.proxyHost", host); + if (port != null) { + port = port.substring(1); // strip ':' + System.setProperty("http.proxyPort", port); + } + proxyMsg = " via proxy " + proxy; + } else { + System.err.println("Value of HTTP_PROXY is not valid: " + proxy); + } + } else { + System.err.println("** If behind a firewall without direct internet access, use the HTTP_PROXY environment variable (e.g. 'env HTTP_PROXY=proxy.company.com:80 max ...') or download manually with a web browser."); + } + + for (String s : urls) { + try { + System.err.println("Downloading " + s + " to " + path + proxyMsg); + URL url = new URL(s); + URLConnection conn = url.openConnection(); + // 10 second timeout to establish connection + conn.setConnectTimeout(10000); + InputStream in = conn.getInputStream(); + int size = conn.getContentLength(); + FileOutputStream out = new FileOutputStream(path); + int read = 0; + byte[] buf = new byte[8192]; + int n = 0; + while ((read = in.read(buf)) != -1) { + n += read; + System.err.print("\r " + n + " bytes " + (size == -1 ? "" : " ( " + (n * 100 / size) + "%)")); + out.write(buf, 0, read); + } + System.err.println(); + out.close(); + in.close(); + return; + } catch (MalformedURLException e) { + throw new Error("Error in URL " + s, e); + } catch (IOException e) { + System.err.println("Error reading from " + s + ": " + e); + path.delete(); + } + } + throw new Error("Could not download content to " + path + " from " + Arrays.toString(urls)); + } + + private static void makeDirectory(File directory) { + if (!directory.exists() && !directory.mkdirs()) { + throw new Error("Could not make directory " + directory); + } + } +} +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mxtool/mx Fri Dec 16 16:46:33 2011 +0100 @@ -0,0 +1,49 @@ +#!/bin/bash +# +# ---------------------------------------------------------------------------------------------------- +# +# Copyright (c) 2007, 2011, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# This code is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 2 only, as +# published by the Free Software Foundation. +# +# This code is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# version 2 for more details (a copy is included in the LICENSE file that +# accompanied this code). +# +# You should have received a copy of the GNU General Public License version +# 2 along with this work; if not, write to the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA +# or visit www.oracle.com if you need additional information or have any +# questions. +# +# ---------------------------------------------------------------------------------------------------- + +dir=`/bin/pwd` + +# Resolve location of this script so that mx.py can be found in the same directory +source="${BASH_SOURCE[0]}" +while [ -h "$source" ] ; do source="$(readlink "$source")"; done +dir="$( cd -P "$( dirname "$source" )" && pwd )" + +if [ ! -f "$dir/mx.py" ]; then + echo "Cannot find mx.py in $dir" + exit 1 +fi + +python <<END +import sys +major, minor, micro, _, _ = sys.version_info +if major != 2 or minor != 7: + raise SystemExit('The mx.py script requires Python 2.7, not {0}.{1}.{2}'.format(major, minor, micro)) +END +if [ $? -eq 0 ]; then + exec python "$dir/mx.py" "$@" +fi +
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mxtool/mx.py Fri Dec 16 16:46:33 2011 +0100 @@ -0,0 +1,1120 @@ +#!/usr/bin/python +# +# ---------------------------------------------------------------------------------------------------- +# +# Copyright (c) 2007, 2011, Oracle and/or its affiliates. All rights reserved. +# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. +# +# This code is free software; you can redistribute it and/or modify it +# under the terms of the GNU General Public License version 2 only, as +# published by the Free Software Foundation. +# +# This code is distributed in the hope that it will be useful, but WITHOUT +# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or +# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License +# version 2 for more details (a copy is included in the LICENSE file that +# accompanied this code). +# +# You should have received a copy of the GNU General Public License version +# 2 along with this work; if not, write to the Free Software Foundation, +# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. +# +# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA +# or visit www.oracle.com if you need additional information or have any +# questions. +# +# ---------------------------------------------------------------------------------------------------- +# +# mx is a command line tool inspired by mvn (http://maven.apache.org/) +# and hg (http://mercurial.selenic.com/). It includes a mechanism for +# managing the dependencies between a set of projects (like Maven) +# as well as making it simple to run commands +# (like hg is the interface to the Mercurial commands). +# +# When launched, mx looks for a mx configuration (i.e. a directory named 'mx') +# in the current working directory. This is the primary mx configuration. Any +# other mx configurations are included mx configurations. +# +# If an mx configuration exists, then the following files in the configuration +# are processed (if they exist): +# +# projects - Lists projects, libraries and dependencies between them +# commands.py - Extensions to the commands launchable by mx. This is only processed +# for the primary mx configuration. +# includes - Other directories containing mx configurations to be loaded. +# This is a recursive action. +# env - A set of environment variable definitions. +# +# The MX_INCLUDES environment variable can also be used to specify +# other directories containing mx configurations. +# This value of this variable has the same format as a Java class path. +# +# The includes and env files are typically not put under version control +# as they usually contain local filesystem paths. +# +# The projects file is like the pom.xml file from Maven except that +# it is in a properties file format instead of XML. Each non-comment line +# in the file specifies an attribute of a project or library. The main +# difference between a project and a library is that the former contains +# source code built by the mx tool where as the latter is an external +# dependency. The format of the projects file is +# +# Library specification format: +# +# library@<name>@<prop>=<value> +# +# Built-in library properties (* = required): +# +# *path: the file system path for the library to appear on a class path +# urls: a comma seperated list of URLs from which the library can be downloaded +# optional: if "true" then this library will be omitted from a class path if it doesn't exist on the file system and no URLs are specified +# +# Project specification format: +# +# project@<name>@<prop>=<value> +# +# The name of a project also denotes the directory it is in. +# +# Built-in project properties: +# +# *sourceDirs: a comma separated list of source directoriy names (relative to the project directory) +# dependencies: a comma separated list of the libraries and project the project depends upon (transitive dependencies may be omitted) +# checkstyle: the project whose Checkstyle configuration (i.e. <project>/.checkstyle_checks.xml) is used +# +# Other properties can be specified for projects and libraries for use by extension commands. +# +# Values can use environment variables with Bash syntax (e.g. ${HOME}). + +import sys +import os +import subprocess +from collections import Callable +from threading import Thread +from argparse import ArgumentParser, REMAINDER +from os.path import join, dirname, exists, getmtime, isabs, expandvars, isdir +import shlex +import types +import urllib2 +import contextlib +import StringIO +import zipfile +import shutil, fnmatch, re, xml.dom.minidom + +DEFAULT_JAVA_ARGS = '-ea -Xss2m -Xmx1g' + +class Dependency: + def __init__(self, name, baseDir): + self.name = name + self.baseDir = baseDir + self.env = None + + def __str__(self): + return self.name + + def __eq__(self, other): + return self.name == other.name + + def __ne__(self, other): + return self.name != other.name + + def __hash__(self): + return hash(self.name) + + def isLibrary(self): + return isinstance(self, Library) + +class Project(Dependency): + def __init__(self, baseDir, name, srcDirs, deps): + Dependency.__init__(self, name, baseDir) + self.srcDirs = srcDirs + self.deps = deps + self.checkstyleProj = name + self.dir = join(baseDir, name) + self.native = False + + def all_deps(self, deps, pdb, includeLibs): + if self in deps: + return deps + for name in self.deps: + assert name != self.name + dep = pdb.libs.get(name, None) + if dep is not None: + if includeLibs and not dep in deps: + deps.append(dep) + else: + dep = pdb.project(name) + if not dep in deps: + dep.all_deps(deps, pdb, includeLibs) + if not self in deps: + deps.append(self) + return deps + + def _compute_max_dep_distances(self, name, distances, dist, pdb): + currentDist = distances.get(name); + if currentDist is None or currentDist < dist: + distances[name] = dist + if pdb.projects.has_key(name): + p = pdb.project(name) + for dep in p.deps: + self._compute_max_dep_distances(dep, distances, dist + 1, pdb) + + + def canonical_deps(self, env, pdb): + distances = dict() + result = set() + self._compute_max_dep_distances(self.name, distances, 0, pdb) + for n,d in distances.iteritems(): + assert d > 0 or n == self.name + if d == 1: + result.add(n) + + + if len(result) == len(self.deps) and frozenset(self.deps) == result: + return self.deps + return result; + + + def source_dirs(self): + return [join(self.baseDir, self.name, s) for s in self.srcDirs] + + def output_dir(self): + return join(self.baseDir, self.name, 'bin') + + def classpath(self, resolve, env): + classesDir = join(self.baseDir, 'classes') + if exists(classesDir): + return [self.output_dir(), classesDir] + return [self.output_dir()] + + + +class Library(Dependency): + def __init__(self, baseDir, name, path, mustExist, urls): + Dependency.__init__(self, name, baseDir) + self.path = path + self.urls = urls + self.mustExist = mustExist + + def classpath(self, resolve, env): + path = self.path + if not isabs(path): + path = join(self.baseDir, path) + if resolve and self.mustExist and not exists(path): + assert not len(self.urls) == 0, 'cannot find required library ' + self.name + " " + path; + env.download(path, self.urls) + + if exists(path) or not resolve: + return [path] + return [] + +class ProjectsDB(): + + def __init__(self, env): + self.env = env + self.projects = dict() + self.libs = dict() + self.commandModules = dict() + self.baseDirs = [] + self.primary = '' + + def _load_projects(self, mxDir, baseDir): + env = self.env + libsMap = dict() + projsMap = dict() + projectsFile = join(mxDir, 'projects') + if not exists(projectsFile): + return + with open(projectsFile) as f: + for line in f: + line = line.strip() + if len(line) != 0 and line[0] != '#': + key, value = line.split('=', 1) + + parts = key.split('@') + if len(parts) != 3: + env.abort('Property name does not have 3 parts separated by "@": ' + key) + kind, name, attr = parts + if kind == 'project': + m = projsMap + elif kind == 'library': + m = libsMap + else: + env.abort('Property name does not start with "project@" or "library@": ' + key) + + attrs = m.get(name) + if attrs is None: + attrs = dict() + m[name] = attrs + value = env.expandvars_in_property(value) + attrs[attr] = value + + def pop_list(attrs, name): + v = attrs.pop(name, None) + if v is None or len(v.strip()) == 0: + return [] + return [n.strip() for n in v.split(',')] + + for name, attrs in projsMap.iteritems(): + if self.projects.has_key(name): + env.abort('cannot override project ' + name + ' in ' + self.project(name).baseDir + " with project of the same name in " + mxDir) + srcDirs = pop_list(attrs, 'sourceDirs') + deps = pop_list(attrs, 'dependencies') + p = Project(baseDir, name, srcDirs, deps) + p.checkstyleProj = attrs.pop('checkstyle', name) + p.native = attrs.pop('native', '') == 'true' + p.__dict__.update(attrs) + self.projects[name] = p + + for name, attrs in libsMap.iteritems(): + if self.libs.has_key(name): + env.abort('cannot redefine library ' + name) + + path = attrs['path'] + mustExist = attrs.pop('optional', 'false') != 'true' + urls = pop_list(attrs, 'urls') + l = Library(baseDir, name, path, mustExist, urls) + l.__dict__.update(attrs) + self.libs[name] = l + + def _load_commands(self, mxDir, baseDir): + env = self.env + commands = join(mxDir, 'commands.py') + if exists(commands): + # temporarily extend the Python path + sys.path.insert(0, mxDir) + + mod = __import__('commands') + + # revert the Python path + del sys.path[0] + + if not hasattr(mod, 'mx_init'): + env.abort(commands + ' must define an mx_init(env) function') + + mod.mx_init(env) + + name = baseDir + '.commands' + sfx = 1 + while sys.modules.has_key(name): + name = baseDir + str(sfx) + '.commands' + sfx += 1 + + sys.modules[name] = sys.modules.pop('commands') + self.commandModules[name] = mod + + def _load_includes(self, mxDir, baseDir): + includes = join(mxDir, 'includes') + if exists(includes): + with open(includes) as f: + for line in f: + includeMxDir = join(self.env.expandvars_in_property(line.strip()), 'mx') + self.load(includeMxDir) + + def _load_env(self, mxDir, baseDir): + env = join(mxDir, 'env') + if exists(env): + with open(env) as f: + for line in f: + line = line.strip() + if len(line) != 0 and line[0] != '#': + key, value = line.split('=', 1) + os.environ[key.strip()] = self.env.expandvars_in_property(value.strip()) + + def load(self, mxDir, primary=False): + """ loads the mx data from a given directory """ + if not exists(mxDir) or not isdir(mxDir): + self.env.abort('Directory does not exist: ' + mxDir) + baseDir = dirname(mxDir) + if primary: + self.primary = baseDir + if not baseDir in self.baseDirs: + self.baseDirs.append(baseDir) + self._load_includes(mxDir, baseDir) + self._load_projects(mxDir, baseDir) + self._load_env(mxDir, baseDir) + if primary: + self._load_commands(mxDir, baseDir) + + def project_names(self): + return ' '.join(self.projects.keys()) + + def project(self, name, fatalIfMissing=True): + p = self.projects.get(name) + if p is None: + self.env.abort('project named ' + name + ' not found') + return p + + def library(self, name): + l = self.libs.get(name) + if l is None: + self.env.abort('library named ' + name + ' not found') + return l + + def _as_classpath(self, deps, resolve): + cp = [] + if self.env.cp_prefix is not None: + cp = [self.env.cp_prefix] + for d in deps: + cp += d.classpath(resolve, self.env) + if self.env.cp_suffix is not None: + cp += [self.env.cp_suffix] + return os.pathsep.join(cp) + + def classpath(self, names=None, resolve=True): + if names is None: + return self._as_classpath(self.sorted_deps(True), resolve) + deps = [] + if isinstance(names, types.StringTypes): + self.project(names).all_deps(deps, self, True) + else: + for n in names: + self.project(n).all_deps(deps, self, True) + return self._as_classpath(deps, resolve) + + def sorted_deps(self, includeLibs=False): + deps = [] + for p in self.projects.itervalues(): + p.all_deps(deps, self, includeLibs) + return deps + +class Env(ArgumentParser): + + def format_commands(self): + msg = '\navailable commands:\n\n' + for cmd in sorted(self.commands.iterkeys()): + c, _ = self.commands[cmd][:2] + doc = c.__doc__ + if doc is None: + doc = '' + msg += ' {0:<20} {1}\n'.format(cmd, doc.split('\n', 1)[0]) + return msg + '\n' + + # Override parent to append the list of available commands + def format_help(self): + return ArgumentParser.format_help(self) + self.format_commands() + + + def __init__(self): + self.java_initialized = False + self.pdb = ProjectsDB(self) + self.commands = dict() + ArgumentParser.__init__(self, prog='mx') + + self.add_argument('-v', action='store_true', dest='verbose', help='enable verbose output') + self.add_argument('-d', action='store_true', dest='java_dbg', help='make Java processes wait on port 8000 for a debugger') + self.add_argument('--cp-pfx', dest='cp_prefix', help='class path prefix', metavar='<arg>') + self.add_argument('--cp-sfx', dest='cp_suffix', help='class path suffix', metavar='<arg>') + self.add_argument('--J', dest='java_args', help='Java VM arguments (e.g. --J @-dsa)', metavar='@<args>', default=DEFAULT_JAVA_ARGS) + self.add_argument('--Jp', action='append', dest='java_args_pfx', help='prefix Java VM arguments (e.g. --Jp @-dsa)', metavar='@<args>', default=[]) + self.add_argument('--Ja', action='append', dest='java_args_sfx', help='suffix Java VM arguments (e.g. --Ja @-dsa)', metavar='@<args>', default=[]) + self.add_argument('--user-home', help='users home directory', metavar='<path>', default=os.path.expanduser('~')) + self.add_argument('--java-home', help='JDK installation directory (must be JDK 6 or later)', metavar='<path>', default=self.default_java_home()) + self.add_argument('--java', help='Java VM executable (default: bin/java under $JAVA_HOME)', metavar='<path>') + self.add_argument('--os', dest='os', help='operating system override') + + def _parse_cmd_line(self, args=None): + if args is None: + args = sys.argv[1:] + + self.add_argument('commandAndArgs', nargs=REMAINDER, metavar='command args...') + + self.parse_args(namespace=self) + + if self.java_home is None or self.java_home == '': + self.abort('Could not find Java home. Use --java-home option or ensure JAVA_HOME environment variable is set.') + + if self.user_home is None or self.user_home == '': + self.abort('Could not find user home. Use --user-home option or ensure HOME environment variable is set.') + + if self.os is None: + self.remote = False + if sys.platform.startswith('darwin'): + self.os = 'darwin' + elif sys.platform.startswith('linux'): + self.os = 'linux' + elif sys.platform.startswith('sunos'): + self.os = 'solaris' + elif sys.platform.startswith('win32') or sys.platform.startswith('cygwin'): + self.os = 'windows' + else: + print 'Supported operating system could not be derived from', sys.platform, '- use --os option explicitly.' + sys.exit(1) + else: + self.java_args += ' -Dmax.os=' + self.os + self.remote = True + + if self.java is None: + self.java = join(self.java_home, 'bin', 'java') + + os.environ['JAVA_HOME'] = self.java_home + os.environ['HOME'] = self.user_home + + self.javac = join(self.java_home, 'bin', 'javac') + + for mod in self.pdb.commandModules.itervalues(): + if hasattr(mod, 'mx_post_parse_cmd_line'): + mod.mx_post_parse_cmd_line(self) + + def expandvars_in_property(self, value): + result = expandvars(value) + if '$' in result or '%' in result: + self.abort('Property contains an undefined environment variable: ' + value) + return result + + + def load_config_file(self, configFile, override=False): + """ adds attributes to this object from a file containing key=value lines """ + if exists(configFile): + with open(configFile) as f: + for line in f: + k, v = line.split('=', 1) + k = k.strip().lower() + if (override or not hasattr(self, k)): + setattr(self, k, self.expandvars_in_property(v.strip())) + + def format_java_cmd(self, args): + self.init_java() + return [self.java] + self.java_args_pfx + self.java_args + self.java_args_sfx + args + + def run_java(self, args, nonZeroIsFatal=True, out=None, err=None, cwd=None): + return self.run(self.format_java_cmd(args), nonZeroIsFatal=nonZeroIsFatal, out=out, err=err, cwd=cwd) + + def run(self, args, nonZeroIsFatal=True, out=None, err=None, cwd=None): + """ + Run a command in a subprocess, wait for it to complete and return the exit status of the process. + If the exit status is non-zero and `nonZeroIsFatal` is true, then the program is exited with + the same exit status. + Each line of the standard output and error streams of the subprocess are redirected to the + provided out and err functions if they are not None. + """ + + assert isinstance(args, types.ListType), "'args' must be a list: " + str(args) + for arg in args: + assert isinstance(arg, types.StringTypes), 'argument is not a string: ' + str(arg) + + if self.verbose: + self.log(' '.join(args)) + + try: + if out is None and err is None: + retcode = subprocess.call(args, cwd=cwd) + else: + def redirect(stream, f): + for line in iter(stream.readline, ''): + f(line) + stream.close() + p = subprocess.Popen(args, cwd=cwd, stdout=None if out is None else subprocess.PIPE, stderr=None if err is None else subprocess.PIPE) + if out is not None: + t = Thread(target=redirect, args=(p.stdout, out)) + t.daemon = True # thread dies with the program + t.start() + if err is not None: + t = Thread(target=redirect, args=(p.stderr, err)) + t.daemon = True # thread dies with the program + t.start() + retcode = p.wait() + except OSError as e: + self.log('Error executing \'' + ' '.join(args) + '\': ' + str(e)) + if self.verbose: + raise e + self.abort(e.errno) + + + if retcode and nonZeroIsFatal: + if self.verbose: + raise subprocess.CalledProcessError(retcode, ' '.join(args)) + self.abort(retcode) + + return retcode + + def check_get_env(self, key): + """ + Gets an environment variable, aborting with a useful message if it is not set. + """ + value = os.environ.get(key) + if value is None: + self.abort('Required environment variable ' + key + ' must be set (e.g. in ' + join(self.pdb.primary, 'env') + ')') + return value + + def exe_suffix(self, name): + """ + Gets the platform specific suffix for an executable + """ + if self.os == 'windows': + return name + '.exe' + return name + + def log(self, msg=None): + """ + Write a message to the console. + All script output goes through this method thus allowing a subclass + to redirect it. + """ + if msg is None: + print + else: + print msg + + def expand_project_in_class_path_arg(self, cpArg): + cp = [] + for part in cpArg.split(os.pathsep): + if part.startswith('@'): + cp += self.pdb.classpath(part[1:]).split(os.pathsep) + else: + cp.append(part) + return os.pathsep.join(cp) + + def expand_project_in_args(self, args): + for i in range(len(args)): + if args[i] == '-cp' or args[i] == '-classpath': + if i + 1 < len(args): + args[i + 1] = self.expand_project_in_class_path_arg(args[i + 1]) + return + + + def init_java(self): + """ + Lazy initialization and preprocessing of this object's fields before running a Java command. + """ + if self.java_initialized: + return + + def delAtAndSplit(s): + return shlex.split(s.lstrip('@')) + + self.java_args = delAtAndSplit(self.java_args) + self.java_args_pfx = sum(map(delAtAndSplit, self.java_args_pfx), []) + self.java_args_sfx = sum(map(delAtAndSplit, self.java_args_sfx), []) + + # Prepend the -d64 VM option only if the java command supports it + output = '' + try: + output = subprocess.check_output([self.java, '-d64', '-version'], stderr=subprocess.STDOUT) + self.java_args = ['-d64'] + self.java_args + except subprocess.CalledProcessError as e: + try: + output = subprocess.check_output([self.java, '-version'], stderr=subprocess.STDOUT) + except subprocess.CalledProcessError as e: + print e.output + self.abort(e.returncode) + + output = output.split() + assert output[0] == 'java' or output[0] == 'openjdk' + assert output[1] == 'version' + version = output[2] + if not version.startswith('"1.6') and not version.startswith('"1.7'): + self.abort('Requires Java version 1.6 or 1.7, got version ' + version) + + if self.java_dbg: + self.java_args += ['-Xdebug', '-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000'] + + self.java_initialized = True + + def default_java_home(self): + javaHome = os.getenv('JAVA_HOME') + if javaHome is None: + if exists('/usr/lib/java/java-6-sun'): + javaHome = '/usr/lib/java/java-6-sun' + elif exists('/System/Library/Frameworks/JavaVM.framework/Versions/1.6/Home'): + javaHome = '/System/Library/Frameworks/JavaVM.framework/Versions/1.6/Home' + elif exists('/usr/jdk/latest'): + javaHome = '/usr/jdk/latest' + return javaHome + + def gmake_cmd(self): + for a in ['make', 'gmake', 'gnumake']: + try: + output = subprocess.check_output([a, '--version']) + if 'GNU' in output: + return a; + except: + pass + self.abort('Could not find a GNU make executable on the current path.') + + + def abort(self, codeOrMessage): + """ + Aborts the program with a SystemExit exception. + If 'codeOrMessage' is a plain integer, it specifies the system exit status; + if it is None, the exit status is zero; if it has another type (such as a string), + the object's value is printed and the exit status is one. + """ + raise SystemExit(codeOrMessage) + + def download(self, path, urls): + """ + Attempts to downloads content for each URL in a list, stopping after the first successful download. + If the content cannot be retrieved from any URL, the program is aborted. The downloaded content + is written to the file indicated by 'path'. + """ + d = dirname(path) + if d != '' and not exists(d): + os.makedirs(d) + + def url_open(url): + userAgent = 'Mozilla/5.0 (compatible)' + headers = { 'User-Agent' : userAgent } + req = urllib2.Request(url, headers=headers) + return urllib2.urlopen(req); + + for url in urls: + try: + self.log('Downloading ' + url + ' to ' + path) + if url.startswith('zip:') or url.startswith('jar:'): + i = url.find('!/') + if i == -1: + self.abort('Zip or jar URL does not contain "!/": ' + url) + url, _, entry = url[len('zip:'):].partition('!/') + with contextlib.closing(url_open(url)) as f: + data = f.read() + zipdata = StringIO.StringIO(f.read()) + + zf = zipfile.ZipFile(zipdata, 'r') + data = zf.read(entry) + with open(path, 'w') as f: + f.write(data) + else: + with contextlib.closing(url_open(url)) as f: + data = f.read() + with open(path, 'w') as f: + f.write(data) + return + except IOError as e: + self.log('Error reading from ' + url + ': ' + str(e)) + except zipfile.BadZipfile as e: + self.log('Error in zip file downloaded from ' + url + ': ' + str(e)) + + # now try it with Java - urllib2 does not handle meta refreshes which are used by Sourceforge + myDir = dirname(__file__) + + javaSource = join(myDir, 'URLConnectionDownload.java') + javaClass = join(myDir, 'URLConnectionDownload.class') + if not exists(javaClass) or getmtime(javaClass) < getmtime(javaSource): + subprocess.check_call([self.javac, '-d', myDir, javaSource]) + if self.run([self.java, '-cp', myDir, 'URLConnectionDownload', path] + urls) != 0: + self.abort('Could not download to ' + path + ' from any of the following URLs:\n\n ' + + '\n '.join(urls) + '\n\nPlease use a web browser to do the download manually') + + def update_file(self, path, content): + """ + Updates a file with some given content if the content differs from what's in + the file already. The return value indicates if the file was updated. + """ + existed = exists(path) + try: + old = None + if existed: + with open(path) as f: + old = f.read() + + if old == content: + return False + + with open(path, 'w') as f: + f.write(content) + + self.log(('modified ' if existed else 'created ') + path) + return True; + except IOError as e: + self.abort('Error while writing to ' + path + ': ' + str(e)); + +# Builtin commands + +def build(env, args): + """compile the Java and C sources, linking the latter + + Compile all the Java source code using the appropriate compilers + and linkers for the various source code types.""" + + parser = ArgumentParser(prog='mx build'); + parser.add_argument('-f', action='store_true', dest='force', help='force compilation even if class files are up to date') + parser.add_argument('-c', action='store_true', dest='clean', help='removes existing build output') + parser.add_argument('--no-native', action='store_false', dest='native', help='do not build com.oracle.max.vm.native') + parser.add_argument('--jdt', help='Eclipse installation or path to ecj.jar for using the Eclipse batch compiler instead of javac', metavar='<path>') + + args = parser.parse_args(args) + + jdtJar = None + if args.jdt is not None: + if args.jdt.endswith('.jar'): + jdtJar=args.jdt + elif isdir(args.jdt): + plugins = join(args.jdt, 'plugins') + choices = [f for f in os.listdir(plugins) if fnmatch.fnmatch(f, 'org.eclipse.jdt.core_*.jar')] + if len(choices) != 0: + jdtJar = join(plugins, sorted(choices, reverse=True)[0]) + + projects = [p.name for p in env.pdb.sorted_deps()] + built = set() + for project in projects: + p = env.pdb.project(project) + projectDir = join(p.baseDir, project) + + if p.native: + if env.os == 'windows': + env.log('Skipping C compilation on Windows until it is supported') + pass + + env.log('Compiling C sources in {0}...'.format(projectDir)) + + if args.clean: + env.run([env.gmake_cmd(), 'clean'], cwd=projectDir) + + env.run([env.gmake_cmd()], cwd=projectDir) + built.add(project) + continue + + outputDir = p.output_dir() + if exists(outputDir): + if args.clean: + env.log('Cleaning {0}...'.format(outputDir)) + shutil.rmtree(outputDir) + os.mkdir(outputDir) + else: + os.mkdir(outputDir) + + classpath = env.pdb.classpath(project) + sourceDirs = env.pdb.project(project).source_dirs() + mustBuild = args.force + if not mustBuild: + for dep in p.all_deps([], env.pdb, False): + if dep.name in built: + mustBuild = True + + for sourceDir in sourceDirs: + javafilelist = [] + nonjavafilelist = [] + for root, _, files in os.walk(sourceDir): + javafiles = [join(root, name) for name in files if name.endswith('.java') and name != 'package-info.java'] + javafilelist += javafiles + nonjavafilelist += [join(root, name) for name in files if not name.endswith('.java')] + if not mustBuild: + for javafile in javafiles: + classfile = outputDir + javafile[len(sourceDir):-len('java')] + 'class' + if not exists(classfile) or os.path.getmtime(javafile) > os.path.getmtime(classfile): + mustBuild = True + break + + if not mustBuild: + env.log('[all class files in {0} are up to date - skipping]'.format(sourceDir)) + continue + + if len(javafilelist) == 0: + env.log('[no Java sources in {0} - skipping]'.format(sourceDir)) + continue + + built.add(project) + + argfileName = join(projectDir, 'javafilelist.txt') + argfile = open(argfileName, 'w') + argfile.write('\n'.join(javafilelist)) + argfile.close() + + try: + if jdtJar is None: + env.log('Compiling Java sources in {0} with javac...'.format(sourceDir)) + + class Filter: + """ + Class to filter the 'is Sun proprietary API and may be removed in a future release' + warning when compiling the VM classes. + + """ + def __init__(self): + self.c = 0 + + def eat(self, line): + if 'Sun proprietary API' in line: + self.c = 2 + elif self.c != 0: + self.c -= 1 + else: + print line.rstrip() + + env.run([env.javac, '-g', '-J-Xmx1g', '-classpath', classpath, '-d', outputDir, '@' + argfile.name], err=Filter().eat) + else: + env.log('Compiling Java sources in {0} with JDT...'.format(sourceDir)) + jdtProperties = join(projectDir, '.settings', 'org.eclipse.jdt.core.prefs') + if not exists(jdtProperties): + raise SystemError('JDT properties file {0} not found'.format(jdtProperties)) + env.run([env.java, '-Xmx1g', '-jar', jdtJar, '-1.6', '-cp', classpath, '-g', + '-properties', jdtProperties, + '-warn:-unusedImport,-unchecked', + '-d', outputDir, '@' + argfile.name]) + finally: + os.remove(argfileName) + + + for name in nonjavafilelist: + dst = join(outputDir, name[len(sourceDir) + 1:]) + if exists(dirname(dst)): + shutil.copyfile(name, dst) + +def canonicalizeprojects(env, args): + """process all project files to canonicalize the dependencies + + The exit code of this command reflects how many files were updated.""" + + changedFiles = 0 + pdb = env.pdb + for d in pdb.baseDirs: + projectsFile = join(d, 'mx', 'projects') + if not exists(projectsFile): + continue + with open(projectsFile) as f: + out = StringIO.StringIO() + pattern = re.compile('project@([^@]+)@dependencies=.*') + for line in f: + line = line.strip() + m = pattern.match(line) + if m is None: + out.write(line + '\n') + else: + p = pdb.project(m.group(1)) + out.write('project@' + m.group(1) + '@dependencies=' + ','.join(p.canonical_deps(env, pdb)) + '\n') + content = out.getvalue() + if env.update_file(projectsFile, content): + changedFiles += 1 + return changedFiles; + +def checkstyle(env, args): + """run Checkstyle on the Java sources + + Run Checkstyle over the Java sources. Any errors or warnings + produced by Checkstyle result in a non-zero exit code. + +If no projects are given, then all Java projects are checked.""" + + allProjects = [p.name for p in env.pdb.sorted_deps()] + if len(args) == 0: + projects = allProjects + else: + projects = args + unknown = set(projects).difference(allProjects) + if len(unknown) != 0: + env.error('unknown projects: ' + ', '.join(unknown)) + + for project in projects: + p = env.pdb.project(project) + projectDir = join(p.baseDir, project) + sourceDirs = env.pdb.project(project).source_dirs() + dotCheckstyle = join(projectDir, '.checkstyle') + + if not exists(dotCheckstyle): + continue + + for sourceDir in sourceDirs: + javafilelist = [] + for root, _, files in os.walk(sourceDir): + javafilelist += [join(root, name) for name in files if name.endswith('.java') and name != 'package-info.java'] + if len(javafilelist) == 0: + env.log('[no Java sources in {0} - skipping]'.format(sourceDir)) + continue + + timestampFile = join(p.baseDir, 'mx', '.checkstyle' + sourceDir[len(p.baseDir):].replace(os.sep, '_') + '.timestamp') + mustCheck = False + if exists(timestampFile): + timestamp = os.path.getmtime(timestampFile) + for f in javafilelist: + if os.path.getmtime(f) > timestamp: + mustCheck = True + break + else: + mustCheck = True + + if not mustCheck: + env.log('[all Java sources in {0} already checked - skipping]'.format(sourceDir)) + continue + + if exists(timestampFile): + os.utime(timestampFile, None) + else: + file(timestampFile, 'a') + + dotCheckstyleXML = xml.dom.minidom.parse(dotCheckstyle) + localCheckConfig = dotCheckstyleXML.getElementsByTagName('local-check-config')[0] + configLocation = localCheckConfig.getAttribute('location') + if configLocation.startswith('/'): + config = join(p.baseDir, configLocation.lstrip('/')) + else: + config = join(projectDir, configLocation) + + exclude = join(projectDir, '.checkstyle.exclude') + + + + + if exists(exclude): + with open(exclude) as f: + # Convert patterns to OS separators + patterns = [name.rstrip().replace('/', os.sep) for name in f.readlines()] + def match(name): + for p in patterns: + if p in name: + env.log('excluding: ' + name) + return True + return False + + javafilelist = [name for name in javafilelist if not match(name)] + + auditfileName = join(projectDir, 'checkstyleOutput.txt') + env.log('Running Checkstyle on {0} using {1}...'.format(sourceDir, config)) + + try: + + # Checkstyle is unable to read the filenames to process from a file, and the + # CreateProcess function on Windows limits the length of a command line to + # 32,768 characters (http://msdn.microsoft.com/en-us/library/ms682425%28VS.85%29.aspx) + # so calling Checkstyle must be done in batches. + while len(javafilelist) != 0: + i = 0 + size = 0 + while i < len(javafilelist): + s = len(javafilelist[i]) + 1 + if (size + s < 30000): + size += s + i += 1 + else: + break + + batch = javafilelist[:i] + javafilelist = javafilelist[i:] + try: + env.run_java(['-Xmx1g', '-jar', env.pdb.library('CHECKSTYLE').classpath(True, env)[0], '-c', config, '-o', auditfileName] + batch) + finally: + if exists(auditfileName): + with open(auditfileName) as f: + warnings = [line.strip() for line in f if 'warning:' in line] + if len(warnings) != 0: + map(env.log, warnings) + return 1 + finally: + if exists(auditfileName): + os.unlink(auditfileName) + return 0 + +def clean(env, args): + """remove all class files, images, and executables + + Removes all files created by a build, including Java class files, executables, and + generated images. + """ + + projects = env.pdb.projects.keys() + for project in projects: + p = env.pdb.project(project) + if p.native: + env.run([env.gmake_cmd(), '-C', p.dir, 'clean']) + else: + outputDir = p.output_dir() + if outputDir != '' and exists(outputDir): + env.log('Removing {0}...'.format(outputDir)) + shutil.rmtree(outputDir) + +def help_(env, args): + """show help for a given command + +With no arguments, print a list of commands and short help for each command. + +Given a command name, print help for that command.""" + if len(args) == 0: + env.print_help() + return + + name = args[0] + if not env.commands.has_key(name): + env.error('unknown command: ' + name) + + value = env.commands[name] + (func, usage) = value[:2] + doc = func.__doc__ + if len(value) > 2: + docArgs = value[2:] + fmtArgs = [] + for d in docArgs: + if isinstance(d, Callable): + fmtArgs += [d(env)] + else: + fmtArgs += [str(d)] + doc = doc.format(*fmtArgs) + print 'mx {0} {1}\n\n{2}\n'.format(name, usage, doc) + + +# Commands are in alphabetical order in this file. + +def javap(env, args): + """launch javap with a -classpath option denoting all available classes + + Run the JDK javap class file disassembler with the following prepended options: + + -private -verbose -classpath <path to project classes>""" + + javap = join(env.java_home, 'bin', 'javap') + if not exists(javap): + env.abort('The javap executable does not exists: ' + javap) + else: + env.run([javap, '-private', '-verbose', '-classpath', env.pdb.classpath()] + args) + +def projects(env, args): + """show all loaded projects""" + pdb = env.pdb + for d in pdb.baseDirs: + projectsFile = join(d, 'mx', 'projects') + if exists(projectsFile): + env.log('# file: ' + projectsFile) + for p in pdb.projects.values(): + if p.baseDir == d: + env.log(p.name) + + +def main(env): + + # Table of commands in alphabetical order. + # Keys are command names, value are lists: [<function>, <usage msg>, <format args to doc string of function>...] + # If any of the format args are instances of Callable, then they are called with an 'env' are before being + # used in the call to str.format(). + # Extensions should update this table directly + env.commands = { + 'build': [build, '[options] projects...'], + 'checkstyle': [checkstyle, 'projects...'], + 'canonicalizeprojects': [canonicalizeprojects, ''], + 'clean': [clean, ''], + 'help': [help_, '[command]'], + 'javap': [javap, ''], + 'projects': [projects, ''], + } + + MX_INCLUDES = os.environ.get('MX_INCLUDES', None) + if MX_INCLUDES is not None: + for path in MX_INCLUDES.split(os.pathsep): + d = join(path, 'mx') + if exists(d) and isdir(d): + env.pdb.load(d) + + cwdMxDir = join(os.getcwd(), 'mx') + if exists(cwdMxDir) and isdir(cwdMxDir): + env.pdb.load(cwdMxDir, primary=True) + + env._parse_cmd_line() + + if len(env.commandAndArgs) == 0: + env.print_help() + return + + env.command = env.commandAndArgs[0] + env.command_args = env.commandAndArgs[1:] + + if not env.commands.has_key(env.command): + env.abort('mx: unknown command \'{0}\'\n{1}use "mx help" for more options'.format(env.command, env.format_commands())) + + c, _ = env.commands[env.command][:2] + try: + retcode = c(env, env.command_args) + if retcode is not None and retcode != 0: + env.abort(retcode) + except KeyboardInterrupt: + # no need to show the stack trace when the user presses CTRL-C + env.abort(1) + +if __name__ == '__main__': + main(Env())