Mercurial > hg > truffle
changeset 22013:6223a23b02ad
removed mxtool/
author | Doug Simon <doug.simon@oracle.com> |
---|---|
date | Wed, 08 Jul 2015 22:16:09 +0200 |
parents | 72010b401152 |
children | 9de5cff8e156 |
files | mxtool/.project mxtool/.pydevproject mxtool/.pylintrc mxtool/CheckCopyright.java mxtool/ClasspathDump.java mxtool/URLConnectionDownload.java mxtool/copyrights/oracle.copyright.hash mxtool/copyrights/oracle.copyright.regex.hash mxtool/copyrights/oracle.copyright.regex.star mxtool/copyrights/oracle.copyright.star mxtool/mx mxtool/mx.py |
diffstat | 12 files changed, 0 insertions(+), 7281 deletions(-) [+] |
line wrap: on
line diff
--- a/mxtool/.project Thu Jul 02 17:33:58 2015 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,17 +0,0 @@ -<?xml version="1.0" encoding="UTF-8"?> -<projectDescription> - <name>mxtool</name> - <comment></comment> - <projects> - </projects> - <buildSpec> - <buildCommand> - <name>org.python.pydev.PyDevBuilder</name> - <arguments> - </arguments> - </buildCommand> - </buildSpec> - <natures> - <nature>org.python.pydev.pythonNature</nature> - </natures> -</projectDescription>
--- a/mxtool/.pydevproject Thu Jul 02 17:33:58 2015 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,10 +0,0 @@ -<?xml version="1.0" encoding="UTF-8" standalone="no"?> -<?eclipse-pydev version="1.0"?> - -<pydev_project> -<pydev_pathproperty name="org.python.pydev.PROJECT_SOURCE_PATH"> -<path>/mxtool</path> -</pydev_pathproperty> -<pydev_property name="org.python.pydev.PYTHON_PROJECT_VERSION">python 2.7</pydev_property> -<pydev_property name="org.python.pydev.PYTHON_PROJECT_INTERPRETER">Default</pydev_property> -</pydev_project>
--- a/mxtool/.pylintrc Thu Jul 02 17:33:58 2015 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,281 +0,0 @@ -[MASTER] - -# Specify a configuration file. -#rcfile= - -# Python code to execute, usually for sys.path manipulation such as -# pygtk.require(). -#init-hook= - -# Profiled execution. -profile=no - -# Add files or directories to the blacklist. They should be base names, not -# paths. -ignore=CVS - -# Pickle collected data for later comparisons. -persistent=no - -# List of plugins (as comma separated values of python modules names) to load, -# usually to register additional checkers. -load-plugins= - - -[MESSAGES CONTROL] - -# Enable the message, report, category or checker with the given id(s). You can -# either give multiple identifier separated by comma (,) or put this option -# multiple time. See also the "--disable" option for examples. -#enable= - -# Disable the message, report, category or checker with the given id(s). You -# can either give multiple identifiers separated by comma (,) or put this -# option multiple times (only on the command line, not in the configuration -# file where it should appear only once).You can also use "--disable=all" to -# disable everything first and then reenable specific checks. For example, if -# you want to run only the similarities checker, you can use "--disable=all -# --enable=similarities". If you want to run only the classes checker, but have -# no Warning level messages displayed, use"--disable=all --enable=classes -# --disable=W" -disable=attribute-defined-outside-init,arguments-differ, - bare-except,global-statement,protected-access,redefined-outer-name, - unused-argument,star-args,pointless-string-statement,old-style-class, - too-many-lines,missing-docstring,no-init,no-self-use,too-many-statements, - too-many-locals,too-few-public-methods,too-many-instance-attributes, - too-many-arguments,too-many-branches,too-many-public-methods, - multiple-statements,abstract-method,F0401,no-member,non-parent-init-called, - maybe-no-member - -# F0401: http://www.logilab.org/ticket/9386 - -[REPORTS] - -# Set the output format. Available formats are text, parseable, colorized, msvs -# (visual studio) and html. You can also give a reporter class, eg -# mypackage.mymodule.MyReporterClass. -output-format=text - -# Put messages in a separate file for each module / package specified on the -# command line instead of printing them on stdout. Reports (if any) will be -# written in a file name "pylint_global.[txt|html]". -files-output=no - -# Tells whether to display a full report or only the messages -reports=no - -# Python expression which should return a note less than 10 (10 is the highest -# note). You have access to the variables errors warning, statement which -# respectively contain the number of errors / warnings messages and the total -# number of statements analyzed. This is used by the global evaluation report -# (RP0004). -evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) - -# Add a comment according to your evaluation note. This is used by the global -# evaluation report (RP0004). -comment=no - -# Template used to display messages. This is a python new-style format string -# used to format the massage information. See doc for all details -#msg-template= - - -[BASIC] - -# Required attributes for module, separated by a comma -required-attributes= - -# List of builtins function names that should not be used, separated by a comma -bad-functions=filter,apply,input - -# Regular expression which should only match correct module names -module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ - -# Regular expression which should only match correct module level names -const-rgx=[a-zA-Z0-9_]{2,30}$ - -# Regular expression which should only match correct class names -class-rgx=[A-Z_][a-zA-Z0-9]+$ - -# Regular expression which should only match correct function names -function-rgx=[a-z_][a-zA-Z0-9_]{1,40}$ - -# Regular expression which should only match correct method names -method-rgx=[a-z_][a-zA-Z0-9_]{2,40}$ - -# Regular expression which should only match correct instance attribute names -attr-rgx=[a-z_][a-zA-Z0-9_]{1,30}$ - -# Regular expression which should only match correct argument names -argument-rgx=[a-z_][a-zA-Z0-9_]{0,30}$ - -# Regular expression which should only match correct variable names -variable-rgx=[a-z_][a-zA-Z0-9_]{0,30}$ - -# Regular expression which should only match correct attribute names in class -# bodies -class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ - -# Regular expression which should only match correct list comprehension / -# generator expression variable names -inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ - -# Good variable names which should always be accepted, separated by a comma -good-names=i,j,k,ex,Run,_ - -# Bad variable names which should always be refused, separated by a comma -bad-names=foo,bar,baz,toto,tutu,tata - -# Regular expression which should only match function or class names that do -# not require a docstring. -no-docstring-rgx=.* - -# Minimum line length for functions/classes that require docstrings, shorter -# ones are exempt. -docstring-min-length=-1 - - -[FORMAT] - -# Maximum number of characters on a single line. -max-line-length=300 - -# Regexp for a line that is allowed to be longer than the limit. -ignore-long-lines=^\s*(# )?<?https?://\S+>?$ - -# Maximum number of lines in a module -max-module-lines=1000 - -# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 -# tab). -indent-string=' ' - - -[MISCELLANEOUS] - -# List of note tags to take in consideration, separated by a comma. -notes=FIXME - - -[SIMILARITIES] - -# Minimum lines number of a similarity. -min-similarity-lines=4 - -# Ignore comments when computing similarities. -ignore-comments=yes - -# Ignore docstrings when computing similarities. -ignore-docstrings=yes - -# Ignore imports when computing similarities. -ignore-imports=no - - -[TYPECHECK] - -# Tells whether missing members accessed in mixin class should be ignored. A -# mixin class is detected if its name ends with "mixin" (case insensitive). -ignore-mixin-members=yes - -# List of classes names for which member attributes should not be checked -# (useful for classes with attributes dynamically set). -ignored-classes=SQLObject - -# When zope mode is activated, add a predefined set of Zope acquired attributes -# to generated-members. -zope=no - -# List of members which are set dynamically and missed by pylint inference -# system, and so shouldn't trigger E0201 when accessed. Python regular -# expressions are accepted. -generated-members=REQUEST,acl_users,aq_parent - - -[VARIABLES] - -# Tells whether we should check for unused import in __init__ files. -init-import=no - -# A regular expression matching the beginning of the name of dummy variables -# (i.e. not used). -dummy-variables-rgx=_$|dummy - -# List of additional names supposed to be defined in builtins. Remember that -# you should avoid to define new builtins when possible. -additional-builtins= - - -[CLASSES] - -# List of interface methods to ignore, separated by a comma. This is used for -# instance to not check methods defines in Zope's Interface base class. -ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by - -# List of method names used to declare (i.e. assign) instance attributes. -defining-attr-methods=__init__,__new__,setUp - -# List of valid names for the first argument in a class method. -valid-classmethod-first-arg=cls - -# List of valid names for the first argument in a metaclass class method. -valid-metaclass-classmethod-first-arg=mcs - - -[DESIGN] - -# Maximum number of arguments for function / method -max-args=5 - -# Argument names that match this expression will be ignored. Default to name -# with leading underscore -ignored-argument-names=_.* - -# Maximum number of locals for function / method body -max-locals=15 - -# Maximum number of return / yield for function / method body -max-returns=6 - -# Maximum number of branch for function / method body -max-branches=12 - -# Maximum number of statements in function / method body -max-statements=50 - -# Maximum number of parents for a class (see R0901). -max-parents=7 - -# Maximum number of attributes for a class (see R0902). -max-attributes=7 - -# Minimum number of public methods for a class (see R0903). -min-public-methods=2 - -# Maximum number of public methods for a class (see R0904). -max-public-methods=20 - - -[IMPORTS] - -# Deprecated modules which should not be used, separated by a comma -deprecated-modules=regsub,TERMIOS,Bastion,rexec - -# Create a graph of every (i.e. internal and external) dependencies in the -# given file (report RP0402 must not be disabled) -import-graph= - -# Create a graph of external dependencies in the given file (report RP0402 must -# not be disabled) -ext-import-graph= - -# Create a graph of internal dependencies in the given file (report RP0402 must -# not be disabled) -int-import-graph= - - -[EXCEPTIONS] - -# Exceptions that will emit a warning when being caught. Defaults to -# "Exception" -overgeneral-exceptions=Exception
--- a/mxtool/CheckCopyright.java Thu Jul 02 17:33:58 2015 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,918 +0,0 @@ -/* - * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -import java.io.*; -import java.net.URISyntaxException; -import java.net.URL; -import java.util.*; -import java.util.regex.*; - - -/** - * A program to check the existence and correctness of the copyright notice on a given set of sources. - * Sources are defined to be those under management by Mercurial and various options are available - * to limit the set of sources scanned. - */ -public class CheckCopyright { - - static class YearInfo { - - final int firstYear; - final int lastYear; - - YearInfo(int firstYear, int lastYear) { - this.firstYear = firstYear; - this.lastYear = lastYear; - } - - @Override - public boolean equals(Object other) { - final YearInfo yearInfo = (YearInfo) other; - return yearInfo.firstYear == firstYear && yearInfo.lastYear == lastYear; - } - - @Override - public int hashCode() { - return firstYear ^ lastYear; - } - } - - static class Info extends YearInfo { - - final String fileName; - - Info(String fileName, int firstYear, int lastYear) { - super(firstYear, lastYear); - this.fileName = fileName; - } - - @Override - public String toString() { - return fileName + " " + firstYear + ", " + lastYear; - } - } - - private static abstract class CopyrightHandler { - enum CommentType{ - STAR, HASH - } - - private static Map<String, CopyrightHandler> copyrightMap; - private static String copyrightFiles = ".*/makefile|.*/Makefile|.*\\.sh|.*\\.bash|.*\\.mk|.*\\.java|.*\\.c|.*\\.h|.*\\.py|.*\\.g|.*\\.r"; - private static Pattern copyrightFilePattern; - - protected final String suffix; - private CopyrightHandler customHandler; - - CopyrightHandler(CommentType commentType) { - this.suffix = commentType.name().toLowerCase(); - initCopyrightMap(); - } - - void addCustomhandler(CopyrightHandler copyrightHandler) { - this.customHandler = copyrightHandler; - } - - /** - * Add @code extension to files handled by this {@code CopyrightKind} - */ - protected void updateMap(String extension) { - copyrightMap.put(extension, this); - } - - static void addCopyrightFilesPattern(String pattern) { - copyrightFiles += "|" + pattern; - } - - protected abstract void readCopyrights() throws IOException; - - protected abstract Matcher getMatcher(String fileName, String fileContent) throws IOException; - - protected abstract String getText(String fileName) throws IOException ; - - protected abstract boolean handlesFile(String fileName); - - /** - * Checks that the Oracle copyright year info was correct. - * @return {@code false} if the year info was incorrect and was not fixed otherwise return {@code true} - * @throws IOException - */ - protected abstract boolean checkYearInfo(String fileName, String fileContent, Matcher matcher, Info info) throws IOException; - - static String getCopyrightText(String fileName) throws IOException { - return getCopyrightHandler(fileName).getText(fileName); - } - - private static CopyrightHandler getCopyrightHandler(String fileName) { - initCopyrightMap(); - if (!copyrightFilePattern.matcher(fileName).matches()) { - return null; - } - CopyrightHandler ck = getDefaultHandler(fileName); - if (ck.customHandler != null && ck.customHandler.handlesFile(fileName)) { - return ck.customHandler; - } else { - return ck; - } - } - - private static void initCopyrightMap() { - if (copyrightMap == null) { - copyrightMap = new HashMap<String, CopyrightHandler>(); - copyrightFilePattern = Pattern.compile(copyrightFiles); - } - } - - static CopyrightHandler getDefaultHandler(String fileName) { - int index = fileName.lastIndexOf(File.separatorChar); - if (index > 0) { - fileName = fileName.substring(index + 1); - } - String ext = ""; - index = fileName.lastIndexOf('.'); - if (index > 0) { - ext = fileName.substring(index + 1); - } - if (fileName.equals("makefile")) { - ext = "mk"; - } - CopyrightHandler ck = copyrightMap.get(ext); - assert ck != null : fileName; - return ck; - } - - protected String readCopyright(InputStream is) throws IOException { - byte[] b = new byte[16384]; - int n = is.read(b); - is.close(); - return new String(b, 0, n); - } - - } - - private static class DefaultCopyrightHandler extends CopyrightHandler { - private static String ORACLE_COPYRIGHT = "oracle.copyright"; - private static String ORACLE_COPYRIGHT_REGEX = "oracle.copyright.regex"; - - private String copyrightRegex; - private String copyright; - Pattern copyrightPattern; - - DefaultCopyrightHandler(CopyrightHandler.CommentType commentType) throws IOException { - super(commentType); - if (commentType == CopyrightHandler.CommentType.STAR) { - updateMap("java"); - updateMap("c"); - updateMap("h"); - updateMap("g"); - } else { - updateMap("r"); - updateMap("R"); - updateMap("py"); - updateMap("sh"); - updateMap("mk"); - updateMap("bash"); - updateMap(""); - } - readCopyrights(); - } - - private String readCopyright(String name) throws IOException { - String copyRightDir = COPYRIGHT_DIR.getValue(); - String fileName = "copyrights/" + name + "." + suffix; - String copyrightPath; - if (copyRightDir != null) { - copyrightPath = new File(new File(copyRightDir), fileName).getAbsolutePath(); - } else { - URL url = CheckCopyright.class.getResource(fileName); - try { - copyrightPath = url.toURI().getPath(); - } catch (URISyntaxException ex) { - throw new IOException(ex); - } - } - InputStream is = new FileInputStream(copyrightPath); - return readCopyright(is); - } - - @Override - protected void readCopyrights() throws IOException { - copyright = readCopyright(ORACLE_COPYRIGHT); - copyrightRegex = readCopyright(ORACLE_COPYRIGHT_REGEX); - copyrightPattern = Pattern.compile(copyrightRegex, Pattern.DOTALL); - } - - @Override - protected Matcher getMatcher(String fileName, String fileContent) { - return copyrightPattern.matcher(fileContent); - } - - @Override - protected String getText(String fileName) { - return copyright; - } - - @Override - protected boolean handlesFile(String fileName) { - return true; - } - - /** - * Check the year info against the copyright header. - * N.B. In the case of multiple matching groups, only the last group is checked. - * I.e., only the last lines containing year info is checked/updated. - */ - @Override - protected boolean checkYearInfo(String fileName, String fileContent, Matcher matcher, Info info) throws IOException { - int yearInCopyright; - int yearInCopyrightIndex; - int groupCount = matcher.groupCount(); - String yearInCopyrightString = matcher.group(groupCount); - yearInCopyright = Integer.parseInt(yearInCopyrightString); - yearInCopyrightIndex = matcher.start(groupCount); - if (yearInCopyright != info.lastYear) { - System.out.println(fileName + " copyright last modified year " + yearInCopyright + ", hg last modified year " + info.lastYear); - if (FIX.getValue()) { - // Use currentYear as that is what it will be when it's checked in! - System.out.println("updating last modified year of " + fileName + " to " + info.lastYear); - // If the previous copyright only specified a single (initial) year, we convert it to the pair form - String newContent = fileContent.substring(0, yearInCopyrightIndex); - if (matcher.group(groupCount - 1) == null) { - // single year form - newContent += yearInCopyrightString + ", "; - } - newContent += info.lastYear + fileContent.substring(yearInCopyrightIndex + 4); - final FileOutputStream os = new FileOutputStream(fileName); - os.write(newContent.getBytes()); - os.close(); - return true; - } else { - return false; - } - } - return true; - } - - } - - private static class CustomCopyrightHandler extends CopyrightHandler { - private Map<String, String> overrides = new HashMap<String, String>(); - private CopyrightHandler defaultHandler; - - CustomCopyrightHandler(CopyrightHandler.CommentType commentType, CopyrightHandler defaultHandler) { - super(commentType); - this.defaultHandler = defaultHandler; - } - - void addFile(String fileName, String copyright) { - overrides.put(fileName, copyright); - } - - @Override - protected void readCopyrights() throws IOException { - } - - @Override - protected Matcher getMatcher(String fileName, String fileContent) throws IOException { - String copyright = overrides.get(fileName); - assert copyright != null : fileName; - try (InputStream fs = new FileInputStream(copyright + "." + suffix + ".regex")) { - return Pattern.compile(readCopyright(fs), Pattern.DOTALL).matcher(fileContent); - } - } - - @Override - protected String getText(String fileName) throws IOException { - String copyright = overrides.get(fileName); - assert copyright != null : fileName; - try (InputStream fs = new FileInputStream(copyright + "." + suffix)) { - return readCopyright(fs); - } - } - - @Override - protected boolean handlesFile(String fileName) { - return overrides.get(fileName) != null; - } - - @Override - protected boolean checkYearInfo(String fileName, String fileContent, Matcher matcher, Info info) throws IOException { - // This is a bit tacky - String copyright = overrides.get(fileName); - if (copyright.endsWith("no.copyright")) { - return true; - } - return defaultHandler.checkYearInfo(fileName, fileContent, matcher, info); - } - } - - private static void initCopyrightKinds() throws IOException { - CopyrightHandler starHandler = new DefaultCopyrightHandler(CopyrightHandler.CommentType.STAR); - CopyrightHandler hashHandler = new DefaultCopyrightHandler(CopyrightHandler.CommentType.HASH); - - String customCopyrightDir = CUSTOM_COPYRIGHT_DIR.getValue(); - if (customCopyrightDir != null) { - CustomCopyrightHandler customStarHandler = new CustomCopyrightHandler(CopyrightHandler.CommentType.STAR, starHandler); - CustomCopyrightHandler customHashHandler = new CustomCopyrightHandler(CopyrightHandler.CommentType.HASH, hashHandler); - starHandler.addCustomhandler(customStarHandler); - hashHandler.addCustomhandler(customHashHandler); - - File overrides = new File(new File(customCopyrightDir), "overrides"); - if (overrides.exists()) { - ArrayList<String> lines = new ArrayList<>(); - boolean changed = false; - try (BufferedReader br = new BufferedReader(new FileReader( - overrides))) { - while (true) { - String line = br.readLine(); - if (line == null) { - break; - } - if (line.length() == 0 || line.startsWith("#")) { - lines.add(line); - continue; - } - String[] parts = line.split(","); - // filename,copyright-file - CopyrightHandler defaultHandler = CopyrightHandler.getDefaultHandler(parts[0]); - if (defaultHandler == null) { - System.err.println("no default copyright handler for: " + parts[0]); - System.exit(1); - } - if (!new File(parts[0]).exists()) { - System.err.printf("file %s in overrides file does not exist", parts[0]); - if (FIX.getValue()) { - System.err.print(" - removing"); - line = null; - changed = true; - } - System.err.println(); - } - if (line != null) { - lines.add(line); - } - CustomCopyrightHandler customhandler = (CustomCopyrightHandler) defaultHandler.customHandler; - customhandler.addFile(parts[0], new File(new File(customCopyrightDir), parts[1]).getAbsolutePath()); - } - } - if (changed) { - try (BufferedWriter bw = new BufferedWriter(new FileWriter( - overrides))) { - for (String line : lines) { - bw.write(line); - bw.write('\n'); - } - } - } - } - } - } - - private static int currentYear = Calendar.getInstance().get(Calendar.YEAR); - private static Options options = new Options(); - private static Option<Boolean> help = options.newBooleanOption("help", false, "Show help message and exit."); - private static Option<String> COPYRIGHT_DIR = options.newStringOption("copyright-dir", null, "override default location of copyright files"); - private static Option<List<String>> FILES_TO_CHECK = options.newStringListOption("files", null, "list of files to check"); - private static Option<String> FILE_LIST = options.newStringOption("file-list", null, "file containing list of files to check"); - private static Option<Boolean> DIR_WALK = options.newBooleanOption("list-dir", false, "check all files in directory tree requiring a copyright (ls -R)"); - private static Option<Boolean> HG_ALL = options.newBooleanOption("hg-all", false, "check all hg managed files requiring a copyright (hg status --all)"); - private static Option<Boolean> HG_MODIFIED = options.newBooleanOption("hg-modified", false, "check all modified hg managed files requiring a copyright (hg status)"); - private static Option<Boolean> HG_OUTGOING = options.newBooleanOption("hg-outgoing", false, "check outgoing hg managed files requiring a copyright (hg outgoing)"); - private static Option<String> HG_LOG = options.newStringOption("hg-last", "0", "check hg managed files requiring a copyright in last N changesets (hg log -l N)"); - private static Option<List<String>> PROJECT = options.newStringListOption("projects", null, "filter files to specific projects"); - private static Option<String> OUTGOING_REPO = options.newStringOption("hg-repo", null, "override outgoing repository"); - private static Option<Boolean> EXHAUSTIVE = options.newBooleanOption("hg-exhaustive", false, "check all hg managed files"); - private static Option<Boolean> FIX = options.newBooleanOption("fix", false, "fix all copyright errors"); - private static Option<String> FILE_PATTERN = options.newStringOption("file-pattern", null, "append additional file patterns for copyright checks"); - private static Option<Boolean> REPORT_ERRORS = options.newBooleanOption("report-errors", false, "report non-fatal errors"); - private static Option<Boolean> HALT_ON_ERROR = options.newBooleanOption("halt-on-error", false, "continue after normally fatal error"); - private static Option<String> HG_PATH = options.newStringOption("hg-path", "hg", "path to hg executable"); - private static Option<Boolean> VERBOSE = options.newBooleanOption("verbose", false, "verbose output"); - private static Option<Boolean> VERY_VERBOSE = options.newBooleanOption("very-verbose", false, "very verbose output"); - private static Option<String> CUSTOM_COPYRIGHT_DIR = options.newStringOption("custom-copyright-dir", null, "file containing filenames with custom copyrights"); - - private static String CANNOT_FOLLOW_FILE = "abort: cannot follow"; - private static String hgPath; - private static boolean error; -// private static File workSpaceDirectory; - private static boolean verbose; - private static boolean veryVerbose; - - public static void main(String[] args) { - // parse the arguments - options.parseArguments(args); - if (help.getValue()) { - options.printHelp(); - return; - } - - verbose = VERBOSE.getValue(); - veryVerbose = VERY_VERBOSE.getValue(); - - hgPath = HG_PATH.getValue(); - - if (FILE_PATTERN.getValue() != null) { - CopyrightHandler.addCopyrightFilesPattern(FILE_PATTERN.getValue()); - } - - try { - initCopyrightKinds(); - List<String> filesToCheck = null; - if (HG_ALL.getValue()) { - filesToCheck = getAllFiles(true); - } else if (HG_OUTGOING.getValue()) { - filesToCheck = getOutgoingFiles(); - } else if (HG_MODIFIED.getValue()) { - filesToCheck = getAllFiles(false); - } else if (Integer.parseInt(HG_LOG.getValue()) > 0) { - filesToCheck = getLastNFiles(Integer.parseInt(HG_LOG.getValue())); - } else if (FILE_LIST.getValue() != null) { - filesToCheck = readFileList(FILE_LIST.getValue()); - } else if (DIR_WALK.getValue()) { - filesToCheck = getDirWalkFiles(); - } else if (FILES_TO_CHECK.getValue() != null) { - filesToCheck = FILES_TO_CHECK.getValue(); - } else { - // no option set, default to HG_ALL - filesToCheck = getAllFiles(true); - } - if (filesToCheck != null && filesToCheck.size() > 0) { - processFiles(filesToCheck); - } else { - System.out.println("nothing to check"); - } - System.exit(error ? 1 : 0); - } catch (Exception ex) { - System.err.println("processing failed: " + ex); - ex.printStackTrace(); - } - } - - private static void processFiles(List<String> fileNames) throws Exception { - final List<String> projects = PROJECT.getValue(); - Calendar cal = Calendar.getInstance(); - for (String fileName : fileNames) { - if (projects == null || isInProjects(fileName, projects)) { - File file = new File(fileName); - if (file.isDirectory()) { - continue; - } - if (verbose) { - System.out.println("checking " + fileName); - } - try { - Info info = null; - if (DIR_WALK.getValue()) { - info = getFromLastModified(cal, fileName); - } else { - final List<String> logInfo = hglog(fileName); - if (logInfo.size() == 0) { - // an added file, so go with last modified - info = getFromLastModified(cal, fileName); - } else { - info = getInfo(fileName, true, logInfo); - } - } - checkFile(fileName, info); - } catch (Exception e) { - System.err.format("COPYRIGHT CHECK WARNING: error while processing %s: %s%n", fileName, e.getMessage()); - } - } - } - } - - private static Info getFromLastModified(Calendar cal, String fileName) { - File file = new File(fileName); - cal.setTimeInMillis(file.lastModified()); - int year = cal.get(Calendar.YEAR); - return new Info(fileName, year, year); - } - - private static boolean isInProjects(String fileName, List<String> projects) { - final int ix = fileName.indexOf(File.separatorChar); - if (ix < 0) { - return false; - } - final String fileProject = fileName.substring(0, ix); - for (String project : projects) { - if (fileProject.equals(project)) { - return true; - } - } - return false; - } - - private static List<String> readFileList(String fileListName) throws IOException { - final List<String> result = new ArrayList<String>(); - BufferedReader b = null; - try { - b = new BufferedReader(new FileReader(fileListName)); - while (true) { - final String fileName = b.readLine(); - if (fileName == null) { - break; - } - if (fileName.length() == 0) { - continue; - } - result.add(fileName); - } - } finally { - if (b != null) { - b.close(); - } - } - return result; - } - - private static Info getInfo(String fileName, boolean lastOnly, List<String> logInfo) { - // process sequence of changesets - int lastYear = 0; - int firstYear = 0; - int ix = 0; - - while (ix < logInfo.size()) { - Map<String, String> tagMap = new HashMap<>(); - ix = getChangeset(logInfo, ix, tagMap); - String date = tagMap.get("date"); - assert date != null; - final int csYear = getYear(date); - if (lastYear == 0) { - lastYear = csYear; - firstYear = lastYear; - } else { - firstYear = csYear; - } - // if we only want the last modified year, quit now - if (lastOnly) { - break; - } - - } - - if (HG_MODIFIED.getValue()) { - // We are only looking at modified and, therefore, uncommitted files. - // This means that the lastYear value will be the current year once the - // file is committed, so that is what we want to check against. - lastYear = currentYear; - } - return new Info(fileName, firstYear, lastYear); - } - - /** - * Process all the changeset data, storing in {@outMap}. - * Return updated value of {@code ix}. - */ - private static int getChangeset(List<String> logInfo, int ixx, Map<String, String> outMap) { - int ix = ixx; - String s = logInfo.get(ix++); - while (s.length() > 0) { - int cx = s.indexOf(':'); - String tag = s.substring(0, cx); - String value = s.substring(cx + 1); - outMap.put(tag, value); - s = logInfo.get(ix++); - } - return ix; - } - - private static int getYear(String dateLine) { - final String[] parts = dateLine.split(" "); - assert parts[parts.length - 2].startsWith("20"); - return Integer.parseInt(parts[parts.length - 2]); - } - - private static void checkFile(String c, Info info) throws IOException { - String fileName = info.fileName; - File file = new File(fileName); - if (!file.exists()) { - System.err.println("COPYRIGHT CHECK WARNING: file " + file + " doesn't exist"); - return; - } - int fileLength = (int) file.length(); - byte[] fileContentBytes = new byte[fileLength]; - FileInputStream is = new FileInputStream(file); - is.read(fileContentBytes); - is.close(); - final String fileContent = new String(fileContentBytes); - CopyrightHandler copyrightHandler = CopyrightHandler.getCopyrightHandler(fileName); - if (file.getName().equals("Makefile")) { - System.console(); - } - if (copyrightHandler != null) { - Matcher copyrightMatcher = copyrightHandler.getMatcher(fileName, fileContent); - if (copyrightMatcher.matches()) { - error = error | !copyrightHandler.checkYearInfo(fileName, fileContent, copyrightMatcher, info); - } else { - // If copyright is missing, insert it, otherwise user has to manually fix existing copyright. - if (!fileContent.contains("Copyright")) { - System.out.print("file " + fileName + " has missing copyright"); - if (FIX.getValue()) { - final FileOutputStream os = new FileOutputStream(file); - os.write(CopyrightHandler.getCopyrightText(fileName) - .getBytes()); - os.write(fileContentBytes); - os.close(); - System.out.println("...fixed"); - } else { - System.out.println(); - error = true; - } - } else { - System.out.println("file " + fileName + " has malformed copyright" + (FIX.getValue() ? " not fixing" : "")); - error = true; - } - } - } else if (EXHAUSTIVE.getValue()) { - System.out.println("ERROR: file " + fileName + " has no copyright"); - error = true; - } - } - - - private static List<String> hglog(String fileName) throws Exception { - final String[] cmd = new String[] {hgPath, "log", "-f", fileName}; - return exec(null, cmd, true); - } - - private static List<String> getLastNFiles(int n) throws Exception { - final String[] cmd = new String[] {hgPath, "log", "-v", "-l", Integer.toString(n)}; - return getFilesFiles(exec(null, cmd, false)); - } - - private static List<String> getAllFiles(boolean all) throws Exception { - final String[] cmd; - if (HG_MODIFIED.getValue()) { - cmd = new String[] {hgPath, "status"}; - } else { - cmd = new String[] {hgPath, "status", "--all"}; - } - List<String> output = exec(null, cmd, true); - final List<String> result = new ArrayList<String>(output.size()); - for (String s : output) { - final char ch = s.charAt(0); - if (!(ch == 'R' || ch == 'I' || ch == '?' || ch == '!')) { - result.add(s.substring(2)); - } - } - return result; - } - - private static List<String> getOutgoingFiles() throws Exception { - final String[] cmd; - if (OUTGOING_REPO.getValue() == null) { - cmd = new String[] {hgPath, "-v", "outgoing"}; - } else { - cmd = new String[] {hgPath, "-v", "outgoing", OUTGOING_REPO.getValue()}; - } - - final List<String> output = exec(null, cmd, false); // no outgoing exits with result 1 - return getFilesFiles(output); - } - - private static List<String> getFilesFiles(List<String> output) { - // there may be multiple changesets so merge the "files:" - final Map<String, String> outSet = new TreeMap<String, String>(); - for (String s : output) { - if (s.startsWith("files:")) { - int ix = s.indexOf(' '); - while (ix < s.length() && s.charAt(ix) == ' ') { - ix++; - } - final String[] files = s.substring(ix).split(" "); - for (String file : files) { - outSet.put(file, file); - } - } - } - return new ArrayList<String>(outSet.values()); - } - - private static List<String> getDirWalkFiles() { - File cwd = new File(System.getProperty("user.dir")); - ArrayList<String> result = new ArrayList<String>(); - getDirWalkFiles(cwd, result); - // remove "user.dir" prefix to make files relative as per hg - String cwdPath = cwd.getAbsolutePath() + '/'; - for (int i = 0; i < result.size(); i++) { - String path = result.get(i); - result.set(i, path.replace(cwdPath, "")); - } - return result; - } - - private static void getDirWalkFiles(File dir, ArrayList<String> list) { - File[] files = dir.listFiles(); - for (File file : files) { - if (ignoreFile(file.getName())) { - continue; - } - if (file.isDirectory()) { - getDirWalkFiles(file, list); - } else { - list.add(file.getAbsolutePath()); - } - } - } - - private static final String IGNORE_LIST = "\\.hg|.*\\.class|bin|src_gen"; - private static final Pattern ignorePattern = Pattern.compile(IGNORE_LIST); - - private static boolean ignoreFile(String name) { - return ignorePattern.matcher(name).matches(); - } - - private static List<String> exec(File workingDir, String[] command, boolean failOnError) throws IOException, InterruptedException { - List<String> result = new ArrayList<String>(); - if (veryVerbose) { - System.out.println("Executing process in directory: " + workingDir); - for (String c : command) { - System.out.println(" " + c); - } - } - final Process process = Runtime.getRuntime().exec(command, null, workingDir); - try { - result = readOutput(process.getInputStream()); - final int exitValue = process.waitFor(); - if (exitValue != 0) { - final List<String> errorResult = readOutput(process.getErrorStream()); - if (REPORT_ERRORS.getValue()) { - System.err.print("execution of command: "); - for (String c : command) { - System.err.print(c); - System.err.print(' '); - } - System.err.println("failed with result " + exitValue); - for (String e : errorResult) { - System.err.println(e); - } - } - if (failOnError && HALT_ON_ERROR.getValue()) { - if (!cannotFollowNonExistentFile(errorResult)) { - throw new Error("terminating"); - } - } - } - } finally { - process.destroy(); - } - return result; - } - - private static boolean cannotFollowNonExistentFile(List<String> errorResult) { - return errorResult.size() == 1 && errorResult.get(0).startsWith(CANNOT_FOLLOW_FILE); - } - - private static List<String> readOutput(InputStream is) throws IOException { - final List<String> result = new ArrayList<String>(); - BufferedReader bs = null; - try { - bs = new BufferedReader(new InputStreamReader(is)); - while (true) { - final String line = bs.readLine(); - if (line == null) { - break; - } - result.add(line); - } - } finally { - if (bs != null) { - bs.close(); - } - } - return result; - } - - private static class Options { - private static Map<String, Option<?>> optionMap = new TreeMap<>(); - - private Option<Boolean> newBooleanOption(String name, boolean defaultValue, String help) { - Option<Boolean> option = new Option<Boolean>(name, help, defaultValue, false, false); - optionMap.put(key(name), option); - return option; - } - - private Option<String> newStringOption(String name, String defaultValue, String help) { - Option<String> option = new Option<String>(name, help, defaultValue); - optionMap.put(key(name), option); - return option; - } - - private Option<List<String>> newStringListOption(String name, List<String> defaultValue, String help) { - Option<List<String>> option = new Option<List<String>>(name, help, defaultValue, true, true); - optionMap.put(key(name), option); - return option; - } - - private static String key(String name) { - return "--" + name; - } - - void parseArguments(String[] args) { - for (int i = 0; i < args.length; i++) { - final String arg = args[i]; - if (arg.startsWith("--")) { - Option<?> option = optionMap.get(arg); - if (option == null || (option.consumesNext() && i == args.length - 1)) { - System.out.println("usage:"); - printHelp(); - System.exit(1); - } - if (option.consumesNext()) { - i++; - option.setValue(args[i]); - } else { - option.setValue(true); - } - } - } - } - - void printHelp() { - int maxKeyLen = 0; - for (Map.Entry<String, Option<?>> entrySet : optionMap.entrySet()) { - int l = entrySet.getKey().length(); - if (l > maxKeyLen) { - maxKeyLen = l; - } - } - for (Map.Entry<String, Option<?>> entrySet : optionMap.entrySet()) { - String key = entrySet.getKey(); - System.out.printf(" %s", key); - for (int i = 0; i < maxKeyLen - key.length(); i++) { - System.out.print(' '); - } - System.out.printf(" %s%n", entrySet.getValue().help); - } - } -} - - private static class Option<T> { - private final String name; - private final String help; - private final boolean consumesNext; - private final boolean isList; - private T value; - - Option(String name, String help, T defaultValue, boolean consumesNext, boolean isList) { - this.name = name; - this.help = help; - this.value = defaultValue; - this.consumesNext = consumesNext; - this.isList = isList; - - } - - Option(String name, String help, T defaultValue) { - this(name, help, defaultValue, true, false); - } - - T getValue() { - return value; - } - - boolean consumesNext() { - return consumesNext; - } - - @SuppressWarnings("unchecked") - void setValue(boolean value) { - this.value = (T) new Boolean(value); - } - - @SuppressWarnings("unchecked") - void setValue(String value) { - if (isList) { - String[] parts = value.split(","); - this.value = (T) Arrays.asList(parts); - } else { - this.value = (T) value; - } - } - - @SuppressWarnings("unused") - String getName() { - return name; - } - } - -}
--- a/mxtool/ClasspathDump.java Thu Jul 02 17:33:58 2015 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,9 +0,0 @@ -public class ClasspathDump { - public static void main(String[] args) { - System.out.print(System.getProperty("sun.boot.class.path")); - System.out.print("|"); - System.out.print(System.getProperty("java.ext.dirs")); - System.out.print("|"); - System.out.print(System.getProperty("java.endorsed.dirs")); - } -} \ No newline at end of file
--- a/mxtool/URLConnectionDownload.java Thu Jul 02 17:33:58 2015 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,163 +0,0 @@ -/* - * Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -import java.io.*; -import java.net.*; -import java.util.*; -import java.util.regex.*; - -/** - * Downloads content from a given URL to a given file. - * - * @param path where to write the content - * @param urls the URLs to try, stopping after the first successful one - */ -public class URLConnectionDownload { - /** - * Iterate over list of environment variable to find one that correctly specify an proxy. - * - * @param propPrefix indicates which proxy property to set (i.e., http or https) - * @param proxyEnvVariableNames list of environment variable - * @return a string specifying the proxy url - */ - private static String setProxy(String[] proxyEnvVariableNames, String propPrefix) { - String proxy = null; - String proxyEnvVar = ""; - for (String envvar : proxyEnvVariableNames) { - proxy = System.getenv(envvar); - if (proxy != null) { - proxyEnvVar = envvar; - break; - } - } - if (proxy != null) { - Pattern p = Pattern.compile("(?:http://)?([^:]+)(:\\d+)?"); - Matcher m = p.matcher(proxy); - if (m.matches()) { - String host = m.group(1); - String port = m.group(2); - System.setProperty(propPrefix + ".proxyHost", host); - if (port != null) { - port = port.substring(1); // strip ':' - System.setProperty(propPrefix + ".proxyPort", port); - } - return proxy; - } else { - System.err.println("Value of " + proxyEnvVar + " is not valid: " + proxy); - } - } else { - System.err.println("** If behind a firewall without direct internet access, use the " + proxyEnvVariableNames[0] + " environment variable (e.g. 'env " + proxyEnvVariableNames[0] + - "=proxy.company.com:80 max ...') or download manually with a web browser."); - } - return ""; - } - - /** - * Downloads content from a given URL to a given file. - * - * @param args arg[0] is the path where to write the content. The remainder of args are the URLs - * to try, stopping after the first successful one - */ - public static void main(String[] args) { - File path = new File(args[0]); - boolean verbose = args[1].equals("-v"); - int offset = verbose ? 2 : 1; - String[] urls = new String[args.length - offset]; - System.arraycopy(args, offset, urls, 0, urls.length); - - File parent = path.getParentFile(); - makeDirectory(parent); - - // Enable use of system proxies - System.setProperty("java.net.useSystemProxies", "true"); - - // Set standard proxy if any - String proxy = setProxy(new String[]{"HTTP_PROXY", "http_proxy"}, "http"); - // Set proxy for secure http if explicitely set, default to http proxy otherwise - String secureProxy = setProxy(new String[]{"HTTPS_PROXY", "https_proxy", "HTTP_PROXY", "http_proxy"}, "https"); - String proxyMsg = ""; - if (secureProxy.length() > 0 && proxy.length() > 0 && !secureProxy.equals(proxy)) { - proxyMsg = " via " + proxy + " / " + secureProxy; - } else if (proxy.length() > 0) { - proxyMsg = " via " + proxy; - } else if (secureProxy.length() > 0) { - proxyMsg = " via " + secureProxy; - } - - for (String s : urls) { - try { - while (true) { - System.err.println("Downloading " + s + " to " + path + proxyMsg); - URL url = new URL(s); - URLConnection conn = url.openConnection(); - // 10 second timeout to establish connection - conn.setConnectTimeout(10000); - - if (conn instanceof HttpURLConnection) { - // HttpURLConnection per default follows redirections, - // but not if it changes the protocol (e.g. http -> - // https). While this is a sane default, in our - // situation it's okay to follow a protocol transition. - HttpURLConnection httpconn = (HttpURLConnection) conn; - switch (httpconn.getResponseCode()) { - case HttpURLConnection.HTTP_MOVED_PERM: - case HttpURLConnection.HTTP_MOVED_TEMP: - System.err.println("follow redirect..."); - s = httpconn.getHeaderField("Location"); - continue; - } - } - InputStream in = conn.getInputStream(); - int size = conn.getContentLength(); - FileOutputStream out = new FileOutputStream(path); - int read = 0; - byte[] buf = new byte[8192]; - int n = 0; - while ((read = in.read(buf)) != -1) { - n += read; - if (verbose) { - long percent = ((long) n * 100 / size); - System.err.print("\r " + n + " bytes " + (size == -1 ? "" : " (" + percent + "%)")); - } - out.write(buf, 0, read); - } - System.err.println(); - out.close(); - in.close(); - return; - } - } catch (MalformedURLException e) { - throw new Error("Error in URL " + s, e); - } catch (IOException e) { - System.err.println("Error reading from " + s + ": " + e); - path.delete(); - } - } - throw new Error("Could not download content to " + path + " from " + Arrays.toString(urls)); - } - - private static void makeDirectory(File directory) { - if (!directory.exists() && !directory.mkdirs()) { - throw new Error("Could not make directory " + directory); - } - } -}
--- a/mxtool/copyrights/oracle.copyright.hash Thu Jul 02 17:33:58 2015 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,22 +0,0 @@ -# -# Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. -# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. -# -# This code is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License version 2 only, as -# published by the Free Software Foundation. -# -# This code is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# version 2 for more details (a copy is included in the LICENSE file that -# accompanied this code). -# -# You should have received a copy of the GNU General Public License version -# 2 along with this work; if not, write to the Free Software Foundation, -# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA -# or visit www.oracle.com if you need additional information or have any -# questions. -#
--- a/mxtool/copyrights/oracle.copyright.regex.hash Thu Jul 02 17:33:58 2015 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1 +0,0 @@ -(?:#!.*\n#\n#\ -*\n)?#\n# Copyright \(c\) (?:(20[0-9][0-9]), )?(20[0-9][0-9]), Oracle and/or its affiliates. All rights reserved.\n# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.\n#\n# This code is free software; you can redistribute it and/or modify it\n# under the terms of the GNU General Public License version 2 only, as\n# published by the Free Software Foundation.\n#\n# This code is distributed in the hope that it will be useful, but WITHOUT\n# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or\n# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License\n# version 2 for more details \(a copy is included in the LICENSE file that\n# accompanied this code\).\n#\n# You should have received a copy of the GNU General Public License version\n# 2 along with this work; if not, write to the Free Software Foundation,\n# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.\n#\n# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA\n# or visit www.oracle.com if you need additional information or have any\n# questions.\n#\n.*
--- a/mxtool/copyrights/oracle.copyright.regex.star Thu Jul 02 17:33:58 2015 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,1 +0,0 @@ -/\*\n \* Copyright \(c\) (?:(20[0-9][0-9]), )?(20[0-9][0-9]), Oracle and/or its affiliates. All rights reserved.\n \* DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.\n \*\n \* This code is free software; you can redistribute it and/or modify it\n \* under the terms of the GNU General Public License version 2 only, as\n \* published by the Free Software Foundation.\n \*\n \* This code is distributed in the hope that it will be useful, but WITHOUT\n \* ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or\n \* FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License\n \* version 2 for more details \(a copy is included in the LICENSE file that\n \* accompanied this code\).\n \*\n \* You should have received a copy of the GNU General Public License version\n \* 2 along with this work; if not, write to the Free Software Foundation,\n \* Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.\n \*\n \* Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA\n \* or visit www.oracle.com if you need additional information or have any\n \* questions.\n \*/\n.*
--- a/mxtool/copyrights/oracle.copyright.star Thu Jul 02 17:33:58 2015 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,22 +0,0 @@ -/* - * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */
--- a/mxtool/mx Thu Jul 02 17:33:58 2015 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,67 +0,0 @@ -#!/bin/bash -# -# ---------------------------------------------------------------------------------------------------- -# -# Copyright (c) 2007, 2011, Oracle and/or its affiliates. All rights reserved. -# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. -# -# This code is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License version 2 only, as -# published by the Free Software Foundation. -# -# This code is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# version 2 for more details (a copy is included in the LICENSE file that -# accompanied this code). -# -# You should have received a copy of the GNU General Public License version -# 2 along with this work; if not, write to the Free Software Foundation, -# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA -# or visit www.oracle.com if you need additional information or have any -# questions. -# -# ---------------------------------------------------------------------------------------------------- - -dir=`/bin/pwd` - -# Resolve location of this script so that mx.py can be found in the same directory -source="${BASH_SOURCE[${#BASH_SOURCE[@]} - 1]}" -while [ -h "$source" ]; do - dir="$(cd -P "$(dirname "$source")" && pwd)" - source="$(readlink "$source")" - [[ $source != /* ]] && source="$dir/$source" -done -dir="$(cd -P "$(dirname "$source")" && pwd)" - -if [ ! -f "$dir/mx.py" ]; then - echo "Cannot find mx.py in $dir" - exit 1 -fi - -# Not all systems (e.g. Mac OS X) have 'python2' on the path. -type python2.7 >/dev/null 2>&1 -if [ $? -eq 0 ]; then - python_exe=python2.7 -else - type python2 > /dev/null 2>&1 - if [ $? -eq 0 ]; then - python_exe=python2 - else - python_exe=python - fi -fi - -$python_exe <<END -import sys -major, minor, micro, _, _ = sys.version_info -if major != 2 or minor != 7: - raise SystemExit('The mx.py script requires Python 2.7, not {0}.{1}.{2}'.format(major, minor, micro)) -END -if [ $? -eq 0 ]; then - exec $python_exe -u "$dir/mx.py" "$@" -fi - -#end of file
--- a/mxtool/mx.py Thu Jul 02 17:33:58 2015 +0200 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,5770 +0,0 @@ -#!/usr/bin/env python2.7 -# -# ---------------------------------------------------------------------------------------------------- -# -# Copyright (c) 2007, 2015, Oracle and/or its affiliates. All rights reserved. -# DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. -# -# This code is free software; you can redistribute it and/or modify it -# under the terms of the GNU General Public License version 2 only, as -# published by the Free Software Foundation. -# -# This code is distributed in the hope that it will be useful, but WITHOUT -# ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or -# FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License -# version 2 for more details (a copy is included in the LICENSE file that -# accompanied this code). -# -# You should have received a copy of the GNU General Public License version -# 2 along with this work; if not, write to the Free Software Foundation, -# Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. -# -# Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA -# or visit www.oracle.com if you need additional information or have any -# questions. -# -# ---------------------------------------------------------------------------------------------------- -# -r""" -mx is a command line tool for managing the development of Java code organized as suites of projects. - -Version 1.x supports a single suite of projects. - -Full documentation can be found at https://wiki.openjdk.java.net/display/Graal/The+mx+Tool -""" - -import sys, os, errno, time, subprocess, shlex, types, StringIO, zipfile, signal, xml.sax.saxutils, tempfile, fnmatch, platform -import textwrap -import socket -import tarfile -import hashlib -import xml.parsers.expat -import shutil, re, xml.dom.minidom -import pipes -import difflib -from collections import Callable -from threading import Thread -from argparse import ArgumentParser, REMAINDER -from os.path import join, basename, dirname, exists, getmtime, isabs, expandvars, isdir, isfile - -try: - # needed to work around https://bugs.python.org/issue1927 - import readline - #then make pylint happy.. - readline.get_line_buffer() -except ImportError: - pass - -# Support for Python 2.6 -def check_output(*popenargs, **kwargs): - process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) - output, _ = process.communicate() - retcode = process.poll() - if retcode: - cmd = kwargs.get("args") - if cmd is None: - cmd = popenargs[0] - error = subprocess.CalledProcessError(retcode, cmd) - error.output = output - raise error - return output - -# Support for jython -def is_jython(): - return sys.platform.startswith('java') - -if not is_jython(): - import multiprocessing - -def cpu_count(): - if is_jython(): - from java.lang import Runtime - runtime = Runtime.getRuntime() - return runtime.availableProcessors() - else: - return multiprocessing.cpu_count() - - -try: subprocess.check_output -except: subprocess.check_output = check_output - -try: zipfile.ZipFile.__enter__ -except: - zipfile.ZipFile.__enter__ = lambda self: self - zipfile.ZipFile.__exit__ = lambda self, t, value, traceback: self.close() - -_projects = dict() -_libs = dict() -_jreLibs = dict() -_dists = dict() -_suites = dict() -_annotationProcessors = None -_primary_suite_path = None -_primary_suite = None -_opts = None -_extra_java_homes = [] -_default_java_home = None -_warn = False - -""" -A distribution is a jar or zip file containing the output from one or more Java projects. -""" -class Distribution: - def __init__(self, suite, name, path, sourcesPath, deps, mainClass, excludedDependencies, distDependencies, javaCompliance, isProcessorDistribution=False): - self.suite = suite - self.name = name - self.path = path.replace('/', os.sep) - self.path = _make_absolute(self.path, suite.dir) - self.sourcesPath = _make_absolute(sourcesPath.replace('/', os.sep), suite.dir) if sourcesPath else None - self.deps = deps - self.update_listeners = set() - self.mainClass = mainClass - self.excludedDependencies = excludedDependencies - self.distDependencies = distDependencies - self.javaCompliance = JavaCompliance(javaCompliance) if javaCompliance else None - self.isProcessorDistribution = isProcessorDistribution - - def sorted_deps(self, includeLibs=False, transitive=False, includeAnnotationProcessors=False): - deps = [] - if transitive: - for depDist in [distribution(name) for name in self.distDependencies]: - for d in depDist.sorted_deps(includeLibs=includeLibs, transitive=True): - if d not in deps: - deps.append(d) - try: - excl = [dependency(d) for d in self.excludedDependencies] - except SystemExit as e: - abort('invalid excluded dependency for {0} distribution: {1}'.format(self.name, e)) - return deps + [d for d in sorted_deps(self.deps, includeLibs=includeLibs, includeAnnotationProcessors=includeAnnotationProcessors) if d not in excl] - - def __str__(self): - return self.name - - def add_update_listener(self, listener): - self.update_listeners.add(listener) - - def get_dist_deps(self, includeSelf=True, transitive=False): - deps = [] - if includeSelf: - deps.append(self) - for name in self.distDependencies: - dist = distribution(name) - if dist not in deps: - deps.append(dist) - if transitive: - for depName in self.distDependencies: - for recDep in distribution(depName).get_dist_deps(False, True): - if recDep not in deps: - deps.append(recDep) - return list(deps) - - """ - Gets the directory in which the IDE project configuration - for this distribution is generated. If this is a distribution - derived from a project defining an annotation processor, then - None is return to indicate no IDE configuration should be - created for this distribution. - """ - def get_ide_project_dir(self): - if hasattr(self, 'definingProject') and self.definingProject.definedAnnotationProcessorsDist == self: - return None - if hasattr(self, 'subDir'): - return join(self.suite.dir, self.subDir, self.name + '.dist') - else: - return join(self.suite.dir, self.name + '.dist') - - def make_archive(self): - # are sources combined into main archive? - unified = self.path == self.sourcesPath - - with Archiver(self.path) as arc: - with Archiver(None if unified else self.sourcesPath) as srcArcRaw: - srcArc = arc if unified else srcArcRaw - services = {} - def overwriteCheck(zf, arcname, source): - if os.path.basename(arcname).startswith('.'): - logv('Excluding dotfile: ' + source) - return True - if not hasattr(zf, '_provenance'): - zf._provenance = {} - existingSource = zf._provenance.get(arcname, None) - isOverwrite = False - if existingSource and existingSource != source: - if arcname[-1] != os.path.sep: - logv('warning: ' + self.path + ': avoid overwrite of ' + arcname + '\n new: ' + source + '\n old: ' + existingSource) - isOverwrite = True - zf._provenance[arcname] = source - return isOverwrite - - if self.mainClass: - manifest = "Manifest-Version: 1.0\nMain-Class: %s\n\n" % (self.mainClass) - if not overwriteCheck(arc.zf, "META-INF/MANIFEST.MF", "project files"): - arc.zf.writestr("META-INF/MANIFEST.MF", manifest) - - for dep in self.sorted_deps(includeLibs=True): - isCoveredByDependecy = False - for d in self.distDependencies: - if dep in _dists[d].sorted_deps(includeLibs=True, transitive=True): - logv("Excluding {0} from {1} because it's provided by the dependency {2}".format(dep.name, self.path, d)) - isCoveredByDependecy = True - break - - if isCoveredByDependecy: - continue - - if dep.isLibrary(): - l = dep - # merge library jar into distribution jar - logv('[' + self.path + ': adding library ' + l.name + ']') - lpath = l.get_path(resolve=True) - libSourcePath = l.get_source_path(resolve=True) - if lpath: - with zipfile.ZipFile(lpath, 'r') as lp: - for arcname in lp.namelist(): - if arcname.startswith('META-INF/services/') and not arcname == 'META-INF/services/': - service = arcname[len('META-INF/services/'):] - assert '/' not in service - services.setdefault(service, []).extend(lp.read(arcname).splitlines()) - else: - if not overwriteCheck(arc.zf, arcname, lpath + '!' + arcname): - arc.zf.writestr(arcname, lp.read(arcname)) - if srcArc.zf and libSourcePath: - with zipfile.ZipFile(libSourcePath, 'r') as lp: - for arcname in lp.namelist(): - if not overwriteCheck(srcArc.zf, arcname, lpath + '!' + arcname): - srcArc.zf.writestr(arcname, lp.read(arcname)) - elif dep.isProject(): - p = dep - - if self.javaCompliance: - if p.javaCompliance > self.javaCompliance: - abort("Compliance level doesn't match: Distribution {0} requires {1}, but {2} is {3}.".format(self.name, self.javaCompliance, p.name, p.javaCompliance)) - - logv('[' + self.path + ': adding project ' + p.name + ']') - outputDir = p.output_dir() - for root, _, files in os.walk(outputDir): - relpath = root[len(outputDir) + 1:] - if relpath == join('META-INF', 'services'): - for service in files: - with open(join(root, service), 'r') as fp: - services.setdefault(service, []).extend([provider.strip() for provider in fp.readlines()]) - elif relpath == join('META-INF', 'providers'): - for provider in files: - with open(join(root, provider), 'r') as fp: - for service in fp: - services.setdefault(service.strip(), []).append(provider) - else: - for f in files: - arcname = join(relpath, f).replace(os.sep, '/') - if not overwriteCheck(arc.zf, arcname, join(root, f)): - arc.zf.write(join(root, f), arcname) - if srcArc.zf: - sourceDirs = p.source_dirs() - if p.source_gen_dir(): - sourceDirs.append(p.source_gen_dir()) - for srcDir in sourceDirs: - for root, _, files in os.walk(srcDir): - relpath = root[len(srcDir) + 1:] - for f in files: - if f.endswith('.java'): - arcname = join(relpath, f).replace(os.sep, '/') - if not overwriteCheck(srcArc.zf, arcname, join(root, f)): - srcArc.zf.write(join(root, f), arcname) - - for service, providers in services.iteritems(): - arcname = 'META-INF/services/' + service - arc.zf.writestr(arcname, '\n'.join(providers)) - - self.notify_updated() - - - def notify_updated(self): - for l in self.update_listeners: - l(self) - -""" -A dependency is a library or project specified in a suite. -""" -class Dependency: - def __init__(self, suite, name): - self.name = name - self.suite = suite - - def __cmp__(self, other): - return cmp(self.name, other.name) - - def __str__(self): - return self.name - - def __eq__(self, other): - return self.name == other.name - - def __ne__(self, other): - return self.name != other.name - - def __hash__(self): - return hash(self.name) - - def isLibrary(self): - return isinstance(self, Library) - - def isJreLibrary(self): - return isinstance(self, JreLibrary) - - def isProject(self): - return isinstance(self, Project) - -class Project(Dependency): - def __init__(self, suite, name, srcDirs, deps, javaCompliance, workingSets, d): - Dependency.__init__(self, suite, name) - self.srcDirs = srcDirs - self.deps = deps - self.checkstyleProj = name - self.javaCompliance = JavaCompliance(javaCompliance) if javaCompliance is not None else None - self.native = False - self.workingSets = workingSets - self.dir = d - - # The annotation processors defined by this project - self.definedAnnotationProcessors = None - self.definedAnnotationProcessorsDist = None - - # Create directories for projects that don't yet exist - if not exists(d): - os.mkdir(d) - for s in self.source_dirs(): - if not exists(s): - os.mkdir(s) - - def all_deps(self, deps, includeLibs, includeSelf=True, includeJreLibs=False, includeAnnotationProcessors=False): - """ - Add the transitive set of dependencies for this project, including - libraries if 'includeLibs' is true, to the 'deps' list. - """ - return sorted(self._all_deps_helper(deps, [], includeLibs, includeSelf, includeJreLibs, includeAnnotationProcessors)) - - def _all_deps_helper(self, deps, dependants, includeLibs, includeSelf=True, includeJreLibs=False, includeAnnotationProcessors=False): - if self in dependants: - abort(str(self) + 'Project dependency cycle found:\n ' + - '\n |\n V\n '.join(map(str, dependants[dependants.index(self):])) + - '\n |\n V\n ' + self.name) - childDeps = list(self.deps) - if includeAnnotationProcessors and len(self.annotation_processors()) > 0: - childDeps = self.annotation_processors() + childDeps - if self in deps: - return deps - for name in childDeps: - assert name != self.name - dep = dependency(name) - if not dep in deps: - if dep.isProject(): - dep._all_deps_helper(deps, dependants + [self], includeLibs=includeLibs, includeJreLibs=includeJreLibs, includeAnnotationProcessors=includeAnnotationProcessors) - elif dep.isProject or (dep.isLibrary() and includeLibs) or (dep.isJreLibrary() and includeJreLibs): - dep.all_deps(deps, includeLibs=includeLibs, includeJreLibs=includeJreLibs, includeAnnotationProcessors=includeAnnotationProcessors) - if not self in deps and includeSelf: - deps.append(self) - return deps - - def _compute_max_dep_distances(self, name, distances, dist): - currentDist = distances.get(name) - if currentDist is None or currentDist < dist: - distances[name] = dist - p = project(name, False) - if p is not None: - for dep in p.deps: - self._compute_max_dep_distances(dep, distances, dist + 1) - - def canonical_deps(self): - """ - Get the dependencies of this project that are not recursive (i.e. cannot be reached - via other dependencies). - """ - distances = dict() - result = set() - self._compute_max_dep_distances(self.name, distances, 0) - for n, d in distances.iteritems(): - assert d > 0 or n == self.name - if d == 1: - result.add(n) - - if len(result) == len(self.deps) and frozenset(self.deps) == result: - return self.deps - return result - - def max_depth(self): - """ - Get the maximum canonical distance between this project and its most distant dependency. - """ - distances = dict() - self._compute_max_dep_distances(self.name, distances, 0) - return max(distances.values()) - - def source_dirs(self): - """ - Get the directories in which the sources of this project are found. - """ - return [join(self.dir, s) for s in self.srcDirs] - - def source_gen_dir(self): - """ - Get the directory in which source files generated by the annotation processor are found/placed. - """ - if self.native: - return None - return join(self.dir, 'src_gen') - - def output_dir(self): - """ - Get the directory in which the class files of this project are found/placed. - """ - if self.native: - return None - return join(self.dir, 'bin') - - def jasmin_output_dir(self): - """ - Get the directory in which the Jasmin assembled class files of this project are found/placed. - """ - if self.native: - return None - return join(self.dir, 'jasmin_classes') - - def append_to_classpath(self, cp, resolve): - if not self.native: - cp.append(self.output_dir()) - - def find_classes_with_matching_source_line(self, pkgRoot, function, includeInnerClasses=False): - """ - Scan the sources of this project for Java source files containing a line for which - 'function' returns true. A map from class name to source file path for each existing class - corresponding to a matched source file is returned. - """ - result = dict() - pkgDecl = re.compile(r"^package\s+([a-zA-Z_][\w\.]*)\s*;$") - for srcDir in self.source_dirs(): - outputDir = self.output_dir() - for root, _, files in os.walk(srcDir): - for name in files: - if name.endswith('.java') and name != 'package-info.java': - matchFound = False - source = join(root, name) - with open(source) as f: - pkg = None - for line in f: - if line.startswith("package "): - match = pkgDecl.match(line) - if match: - pkg = match.group(1) - if function(line.strip()): - matchFound = True - if pkg and matchFound: - break - - if matchFound: - simpleClassName = name[:-len('.java')] - assert pkg is not None, 'could not find package statement in file ' + name - if pkgRoot is None or pkg.startswith(pkgRoot): - pkgOutputDir = join(outputDir, pkg.replace('.', os.path.sep)) - if exists(pkgOutputDir): - for e in os.listdir(pkgOutputDir): - if includeInnerClasses: - if e.endswith('.class') and (e.startswith(simpleClassName) or e.startswith(simpleClassName + '$')): - className = pkg + '.' + e[:-len('.class')] - result[className] = source - elif e == simpleClassName + '.class': - className = pkg + '.' + simpleClassName - result[className] = source - return result - - def _init_packages_and_imports(self): - if not hasattr(self, '_defined_java_packages'): - packages = set() - extendedPackages = set() - depPackages = set() - for d in self.all_deps([], includeLibs=False, includeSelf=False): - depPackages.update(d.defined_java_packages()) - imports = set() - importRe = re.compile(r'import\s+(?:static\s+)?([^;]+);') - for sourceDir in self.source_dirs(): - for root, _, files in os.walk(sourceDir): - javaSources = [name for name in files if name.endswith('.java')] - if len(javaSources) != 0: - pkg = root[len(sourceDir) + 1:].replace(os.sep, '.') - if not pkg in depPackages: - packages.add(pkg) - else: - # A project extends a package already defined by one of it dependencies - extendedPackages.add(pkg) - imports.add(pkg) - - for n in javaSources: - with open(join(root, n)) as fp: - content = fp.read() - imports.update(importRe.findall(content)) - self._defined_java_packages = frozenset(packages) - self._extended_java_packages = frozenset(extendedPackages) - - importedPackages = set() - for imp in imports: - name = imp - while not name in depPackages and len(name) > 0: - lastDot = name.rfind('.') - if lastDot == -1: - name = None - break - name = name[0:lastDot] - if name is not None: - importedPackages.add(name) - self._imported_java_packages = frozenset(importedPackages) - - def defined_java_packages(self): - """Get the immutable set of Java packages defined by the Java sources of this project""" - self._init_packages_and_imports() - return self._defined_java_packages - - def extended_java_packages(self): - """Get the immutable set of Java packages extended by the Java sources of this project""" - self._init_packages_and_imports() - return self._extended_java_packages - - def imported_java_packages(self): - """Get the immutable set of Java packages defined by other Java projects that are - imported by the Java sources of this project.""" - self._init_packages_and_imports() - return self._imported_java_packages - - """ - Gets the list of projects defining the annotation processors that will be applied - when compiling this project. This includes the projects declared by the annotationProcessors property - of this project and any of its project dependencies. It also includes - any project dependencies that define an annotation processors. - """ - def annotation_processors(self): - if not hasattr(self, '_annotationProcessors'): - aps = set() - if hasattr(self, '_declaredAnnotationProcessors'): - aps = set(self._declaredAnnotationProcessors) - for ap in aps: - if project(ap).definedAnnotationProcessorsDist is None: - config = join(project(ap).source_dirs()[0], 'META-INF', 'services', 'javax.annotation.processing.Processor') - if not exists(config): - TimeStampFile(config).touch() - abort('Project ' + ap + ' declared in annotationProcessors property of ' + self.name + ' does not define any annotation processors.\n' + - 'Please specify the annotation processors in ' + config) - - allDeps = self.all_deps([], includeLibs=False, includeSelf=False, includeAnnotationProcessors=False) - for p in allDeps: - # Add an annotation processor dependency - if p.definedAnnotationProcessorsDist is not None: - aps.add(p.name) - - # Inherit annotation processors from dependencies - aps.update(p.annotation_processors()) - - self._annotationProcessors = sorted(list(aps)) - return self._annotationProcessors - - """ - Gets the class path composed of the distribution jars containing the - annotation processors that will be applied when compiling this project. - """ - def annotation_processors_path(self): - aps = [project(ap) for ap in self.annotation_processors()] - libAps = [dep for dep in self.all_deps([], includeLibs=True, includeSelf=False) if dep.isLibrary() and hasattr(dep, 'annotationProcessor') and getattr(dep, 'annotationProcessor').lower() == 'true'] - if len(aps) + len(libAps): - return os.pathsep.join([ap.definedAnnotationProcessorsDist.path for ap in aps if ap.definedAnnotationProcessorsDist] + [lib.get_path(False) for lib in libAps]) - return None - - def uses_annotation_processor_library(self): - for dep in self.all_deps([], includeLibs=True, includeSelf=False): - if dep.isLibrary() and hasattr(dep, 'annotationProcessor'): - return True - return False - - def update_current_annotation_processors_file(self): - aps = self.annotation_processors() - outOfDate = False - currentApsFile = join(self.suite.mxDir, 'currentAnnotationProcessors', self.name) - currentApsFileExists = exists(currentApsFile) - if currentApsFileExists: - with open(currentApsFile) as fp: - currentAps = [l.strip() for l in fp.readlines()] - if currentAps != aps: - outOfDate = True - if outOfDate or not currentApsFileExists: - if not exists(dirname(currentApsFile)): - os.mkdir(dirname(currentApsFile)) - with open(currentApsFile, 'w') as fp: - for ap in aps: - print >> fp, ap - return outOfDate - - def make_archive(self, path=None): - outputDir = self.output_dir() - if not path: - path = join(self.dir, self.name + '.jar') - with Archiver(path) as arc: - for root, _, files in os.walk(outputDir): - for f in files: - relpath = root[len(outputDir) + 1:] - arcname = join(relpath, f).replace(os.sep, '/') - arc.zf.write(join(root, f), arcname) - return path - -def _make_absolute(path, prefix): - """ - Makes 'path' absolute if it isn't already by prefixing 'prefix' - """ - if not isabs(path): - return join(prefix, path) - return path - -def sha1OfFile(path): - with open(path, 'rb') as f: - d = hashlib.sha1() - while True: - buf = f.read(4096) - if not buf: - break - d.update(buf) - return d.hexdigest() - -def download_file_with_sha1(name, path, urls, sha1, sha1path, resolve, mustExist, sources=False, canSymlink=True): - canSymlink = canSymlink and not (get_os() == 'windows' or get_os() == 'cygwin') - def _download_lib(): - cacheDir = _cygpathW2U(get_env('MX_CACHE_DIR', join(_opts.user_home, '.mx', 'cache'))) - if not exists(cacheDir): - os.makedirs(cacheDir) - base = basename(path) - cachePath = join(cacheDir, base + '_' + sha1) - - if not exists(cachePath) or sha1OfFile(cachePath) != sha1: - if exists(cachePath): - log('SHA1 of ' + cachePath + ' does not match expected value (' + sha1 + ') - found ' + sha1OfFile(cachePath) + ' - re-downloading') - print 'Downloading ' + ("sources " if sources else "") + name + ' from ' + str(urls) - download(cachePath, urls) - - d = dirname(path) - if d != '' and not exists(d): - os.makedirs(d) - - if canSymlink and 'symlink' in dir(os): - if exists(path): - os.unlink(path) - try: - os.symlink(cachePath, path) - except OSError as e: - # When doing parallel building, the symlink can fail - # if another thread wins the race to create the symlink - if not exists(path): - # It was some other error - raise e - - else: - shutil.copy(cachePath, path) - - def _sha1Cached(): - with open(sha1path, 'r') as f: - return f.read()[0:40] - - def _writeSha1Cached(): - with open(sha1path, 'w') as f: - f.write(sha1OfFile(path)) - - if resolve and mustExist and not exists(path): - assert not len(urls) == 0, 'cannot find required library ' + name + ' ' + path - _download_lib() - - if exists(path): - if sha1 and not exists(sha1path): - _writeSha1Cached() - - if sha1 and sha1 != _sha1Cached(): - _download_lib() - if sha1 != sha1OfFile(path): - abort("SHA1 does not match for " + name + ". Broken download? SHA1 not updated in projects file?") - _writeSha1Cached() - - return path - -class BaseLibrary(Dependency): - def __init__(self, suite, name, optional): - Dependency.__init__(self, suite, name) - self.optional = optional - - def __ne__(self, other): - result = self.__eq__(other) - if result is NotImplemented: - return result - return not result - -""" -A library that will be provided by the JRE but may be absent. -Any project or normal library that depends on a missing library -will be removed from the global project and library dictionaries -(i.e., _projects and _libs). - -This mechanism exists primarily to be able to support code -that may use functionality in one JRE (e.g., Oracle JRE) -that is not present in another JRE (e.g., OpenJDK). A -motivating example is the Java Flight Recorder library -found in the Oracle JRE. -""" -class JreLibrary(BaseLibrary): - def __init__(self, suite, name, jar, optional): - BaseLibrary.__init__(self, suite, name, optional) - self.jar = jar - - def __eq__(self, other): - if isinstance(other, JreLibrary): - return self.jar == other.jar - else: - return NotImplemented - - def is_present_in_jdk(self, jdk): - return jdk.containsJar(self.jar) - - def all_deps(self, deps, includeLibs, includeSelf=True, includeJreLibs=False, includeAnnotationProcessors=False): - """ - Add the transitive set of dependencies for this JRE library to the 'deps' list. - """ - if includeJreLibs and includeSelf and not self in deps: - deps.append(self) - return sorted(deps) - -class Library(BaseLibrary): - def __init__(self, suite, name, path, optional, urls, sha1, sourcePath, sourceUrls, sourceSha1, deps): - BaseLibrary.__init__(self, suite, name, optional) - self.path = path.replace('/', os.sep) - self.urls = urls - self.sha1 = sha1 - self.sourcePath = sourcePath - self.sourceUrls = sourceUrls - if sourcePath == path: - assert sourceSha1 is None or sourceSha1 == sha1 - sourceSha1 = sha1 - self.sourceSha1 = sourceSha1 - self.deps = deps - abspath = _make_absolute(path, self.suite.dir) - if not optional and not exists(abspath): - if not len(urls): - abort('Non-optional library {0} must either exist at {1} or specify one or more URLs from which it can be retrieved'.format(name, abspath)) - - def _checkSha1PropertyCondition(propName, cond, inputPath): - if not cond: - absInputPath = _make_absolute(inputPath, self.suite.dir) - if exists(absInputPath): - abort('Missing "{0}" property for library {1}. Add the following line to projects file:\nlibrary@{2}@{3}={4}'.format(propName, name, name, propName, sha1OfFile(absInputPath))) - abort('Missing "{0}" property for library {1}'.format(propName, name)) - - _checkSha1PropertyCondition('sha1', sha1, path) - _checkSha1PropertyCondition('sourceSha1', not sourcePath or sourceSha1, sourcePath) - - for url in urls: - if url.endswith('/') != self.path.endswith(os.sep): - abort('Path for dependency directory must have a URL ending with "/": path=' + self.path + ' url=' + url) - - def __eq__(self, other): - if isinstance(other, Library): - if len(self.urls) == 0: - return self.path == other.path - else: - return self.urls == other.urls - else: - return NotImplemented - - def get_path(self, resolve): - path = _make_absolute(self.path, self.suite.dir) - sha1path = path + '.sha1' - - includedInJDK = getattr(self, 'includedInJDK', None) - # TODO since we don't know which JDK will be used, this check is dubious - if includedInJDK and java().javaCompliance >= JavaCompliance(includedInJDK): - return None - - bootClassPathAgent = getattr(self, 'bootClassPathAgent').lower() == 'true' if hasattr(self, 'bootClassPathAgent') else False - - return download_file_with_sha1(self.name, path, self.urls, self.sha1, sha1path, resolve, not self.optional, canSymlink=not bootClassPathAgent) - - def get_source_path(self, resolve): - if self.sourcePath is None: - return None - path = _make_absolute(self.sourcePath, self.suite.dir) - sha1path = path + '.sha1' - - return download_file_with_sha1(self.name, path, self.sourceUrls, self.sourceSha1, sha1path, resolve, len(self.sourceUrls) != 0, sources=True) - - def append_to_classpath(self, cp, resolve): - path = self.get_path(resolve) - if path and (exists(path) or not resolve): - cp.append(path) - - def all_deps(self, deps, includeLibs, includeSelf=True, includeJreLibs=False, includeAnnotationProcessors=False): - """ - Add the transitive set of dependencies for this library to the 'deps' list. - """ - if not includeLibs: - return sorted(deps) - childDeps = list(self.deps) - if self in deps: - return sorted(deps) - for name in childDeps: - assert name != self.name - dep = library(name) - if not dep in deps: - dep.all_deps(deps, includeLibs=includeLibs, includeJreLibs=includeJreLibs, includeAnnotationProcessors=includeAnnotationProcessors) - if not self in deps and includeSelf: - deps.append(self) - return sorted(deps) - -class HgConfig: - """ - Encapsulates access to Mercurial (hg) - """ - def __init__(self): - self.missing = 'no hg executable found' - self.has_hg = None - - def check(self, abortOnFail=True): - if self.has_hg is None: - try: - subprocess.check_output(['hg']) - self.has_hg = True - except OSError: - self.has_hg = False - warn(self.missing) - - if not self.has_hg: - if abortOnFail: - abort(self.missing) - else: - warn(self.missing) - - def tip(self, sDir, abortOnError=True): - try: - return subprocess.check_output(['hg', 'tip', '-R', sDir, '--template', '{node}']) - except OSError: - warn(self.missing) - except subprocess.CalledProcessError: - if abortOnError: - abort('failed to get tip revision id') - else: - return None - - def isDirty(self, sDir, abortOnError=True): - try: - return len(subprocess.check_output(['hg', 'status', '-R', sDir])) > 0 - except OSError: - warn(self.missing) - except subprocess.CalledProcessError: - if abortOnError: - abort('failed to get status') - else: - return None - - def locate(self, sDir, patterns=None, abortOnError=True): - try: - if patterns is None: - patterns = [] - elif not isinstance(patterns, list): - patterns = [patterns] - return subprocess.check_output(['hg', 'locate', '-R', sDir] + patterns).split('\n') - except OSError: - warn(self.missing) - except subprocess.CalledProcessError as e: - if e.returncode == 1: - # hg locate returns 1 if no matches were found - return [] - if abortOnError: - abort('failed to locate') - else: - return None - -def _load_suite_dict(mxDir): - - suffix = 1 - suite = None - dictName = 'suite' - - def expand(value, context): - if isinstance(value, types.DictionaryType): - for n, v in value.iteritems(): - value[n] = expand(v, context + [n]) - elif isinstance(value, types.ListType): - for i in range(len(value)): - value[i] = expand(value[i], context + [str(i)]) - else: - if not isinstance(value, types.StringTypes): - abort('value of ' + '.'.join(context) + ' is of unexpected type ' + str(type(value))) - value = expandvars(value) - if '$' in value or '%' in value: - abort('value of ' + '.'.join(context) + ' contains an undefined environment variable: ' + value) - - return value - - moduleName = 'suite' - modulePath = join(mxDir, moduleName + '.py') - while exists(modulePath): - - savedModule = sys.modules.get(moduleName) - if savedModule: - warn(modulePath + ' conflicts with ' + savedModule.__file__) - # temporarily extend the Python path - sys.path.insert(0, mxDir) - - snapshot = frozenset(sys.modules.keys()) - module = __import__(moduleName) - - if savedModule: - # restore the old module into the module name space - sys.modules[moduleName] = savedModule - else: - # remove moduleName from the module name space - sys.modules.pop(moduleName) - - # For now fail fast if extra modules were loaded. - # This can later be relaxed to simply remove the extra modules - # from the sys.modules name space if necessary. - extraModules = frozenset(sys.modules.keys()) - snapshot - assert len(extraModules) == 0, 'loading ' + modulePath + ' caused extra modules to be loaded: ' + ', '.join([m for m in extraModules]) - - # revert the Python path - del sys.path[0] - - if not hasattr(module, dictName): - abort(modulePath + ' must define a variable named "' + dictName + '"') - d = expand(getattr(module, dictName), [dictName]) - sections = ['projects', 'libraries', 'jrelibraries', 'distributions'] + (['distribution_extensions'] if suite else ['name', 'mxversion']) - unknown = frozenset(d.keys()) - frozenset(sections) - if unknown: - abort(modulePath + ' defines unsupported suite sections: ' + ', '.join(unknown)) - - if suite is None: - suite = d - else: - for s in sections: - existing = suite.get(s) - additional = d.get(s) - if additional: - if not existing: - suite[s] = additional - else: - conflicting = frozenset(additional.keys()) & frozenset(existing.keys()) - if conflicting: - abort(modulePath + ' redefines: ' + ', '.join(conflicting)) - existing.update(additional) - distExtensions = d.get('distribution_extensions') - if distExtensions: - existing = suite['distributions'] - for n, attrs in distExtensions.iteritems(): - original = existing.get(n) - if not original: - abort('cannot extend non-existing distribution ' + n) - for k, v in attrs.iteritems(): - if k != 'dependencies': - abort('Only the dependencies of distribution ' + n + ' can be extended') - if not isinstance(v, types.ListType): - abort('distribution_extensions.' + n + '.dependencies must be a list') - original['dependencies'] += v - - dictName = 'extra' - moduleName = 'suite' + str(suffix) - modulePath = join(mxDir, moduleName + '.py') - - deprecatedModulePath = join(mxDir, 'projects' + str(suffix) + '.py') - if exists(deprecatedModulePath): - abort('Please rename ' + deprecatedModulePath + ' to ' + modulePath) - - suffix = suffix + 1 - - return suite, modulePath - -class Suite: - def __init__(self, mxDir, primary, load=True): - self.dir = dirname(mxDir) - self.mxDir = mxDir - self.projects = [] - self.libs = [] - self.jreLibs = [] - self.dists = [] - self.commands = None - self.primary = primary - self.requiredMxVersion = None - self.name = _suitename(mxDir) # validated in _load_projects - if load: - # just check that there are no imports - self._load_imports() - self._load_env() - self._load_commands() - _suites[self.name] = self - - def __str__(self): - return self.name - - def _load_projects(self): - suitePyFile = join(self.mxDir, 'suite.py') - if not exists(suitePyFile): - return - - suiteDict, _ = _load_suite_dict(self.mxDir) - - if suiteDict.get('name') is not None and suiteDict.get('name') != self.name: - abort('suite name in project file does not match ' + _suitename(self.mxDir)) - - if suiteDict.has_key('mxversion'): - try: - self.requiredMxVersion = VersionSpec(suiteDict['mxversion']) - except AssertionError as ae: - abort('Exception while parsing "mxversion" in project file: ' + str(ae)) - - libsMap = suiteDict['libraries'] - jreLibsMap = suiteDict['jrelibraries'] - projsMap = suiteDict['projects'] - distsMap = suiteDict['distributions'] - - def pop_list(attrs, name, context): - v = attrs.pop(name, None) - if not v: - return [] - if not isinstance(v, list): - abort('Attribute "' + name + '" for ' + context + ' must be a list') - return v - - for name, attrs in sorted(projsMap.iteritems()): - context = 'project ' + name - srcDirs = pop_list(attrs, 'sourceDirs', context) - deps = pop_list(attrs, 'dependencies', context) - ap = pop_list(attrs, 'annotationProcessors', context) - javaCompliance = attrs.pop('javaCompliance', None) - subDir = attrs.pop('subDir', None) - if subDir is None: - d = join(self.dir, name) - else: - d = join(self.dir, subDir, name) - workingSets = attrs.pop('workingSets', None) - p = Project(self, name, srcDirs, deps, javaCompliance, workingSets, d) - p.checkstyleProj = attrs.pop('checkstyle', name) - p.native = attrs.pop('native', '') == 'true' - p.checkPackagePrefix = attrs.pop('checkPackagePrefix', 'true') == 'true' - if not p.native and p.javaCompliance is None: - abort('javaCompliance property required for non-native project ' + name) - if len(ap) > 0: - p._declaredAnnotationProcessors = ap - p.__dict__.update(attrs) - self.projects.append(p) - - for name, attrs in sorted(jreLibsMap.iteritems()): - jar = attrs.pop('jar') - # JRE libraries are optional by default - optional = attrs.pop('optional', 'true') != 'false' - l = JreLibrary(self, name, jar, optional) - self.jreLibs.append(l) - - for name, attrs in sorted(libsMap.iteritems()): - context = 'library ' + name - if "|" in name: - if name.count('|') != 2: - abort("Format error in library name: " + name + "\nsyntax: libname|os-platform|architecture") - name, platform, architecture = name.split("|") - if platform != get_os() or architecture != get_arch(): - continue - path = attrs.pop('path') - urls = pop_list(attrs, 'urls', context) - sha1 = attrs.pop('sha1', None) - sourcePath = attrs.pop('sourcePath', None) - sourceUrls = pop_list(attrs, 'sourceUrls', context) - sourceSha1 = attrs.pop('sourceSha1', None) - deps = pop_list(attrs, 'dependencies', context) - # Add support optional libraries once we have a good use case - optional = False - l = Library(self, name, path, optional, urls, sha1, sourcePath, sourceUrls, sourceSha1, deps) - l.__dict__.update(attrs) - self.libs.append(l) - - for name, attrs in sorted(distsMap.iteritems()): - context = 'distribution ' + name - path = attrs.pop('path') - sourcesPath = attrs.pop('sourcesPath', None) - deps = pop_list(attrs, 'dependencies', context) - mainClass = attrs.pop('mainClass', None) - exclDeps = pop_list(attrs, 'exclude', context) - distDeps = pop_list(attrs, 'distDependencies', context) - javaCompliance = attrs.pop('javaCompliance', None) - d = Distribution(self, name, path, sourcesPath, deps, mainClass, exclDeps, distDeps, javaCompliance) - d.__dict__.update(attrs) - self.dists.append(d) - - # Create a distribution for each project that defines annotation processors - for p in self.projects: - annotationProcessors = None - for srcDir in p.source_dirs(): - configFile = join(srcDir, 'META-INF', 'services', 'javax.annotation.processing.Processor') - if exists(configFile): - with open(configFile) as fp: - annotationProcessors = [ap.strip() for ap in fp] - if len(annotationProcessors) != 0: - for ap in annotationProcessors: - if not ap.startswith(p.name): - abort(ap + ' in ' + configFile + ' does not start with ' + p.name) - if annotationProcessors: - dname = p.name.replace('.', '_').upper() - apDir = join(p.dir, 'ap') - path = join(apDir, p.name + '.jar') - sourcesPath = None - deps = [p.name] - mainClass = None - exclDeps = [] - distDeps = [] - javaCompliance = None - d = Distribution(self, dname, path, sourcesPath, deps, mainClass, exclDeps, distDeps, javaCompliance, True) - d.subDir = os.path.relpath(os.path.dirname(p.dir), self.dir) - self.dists.append(d) - p.definedAnnotationProcessors = annotationProcessors - p.definedAnnotationProcessorsDist = d - d.definingProject = p - - # Restrict exported annotation processors to those explicitly defined by the project - def _refineAnnotationProcessorServiceConfig(dist): - aps = dist.definingProject.definedAnnotationProcessors - apsJar = dist.path - config = 'META-INF/services/javax.annotation.processing.Processor' - with zipfile.ZipFile(apsJar, 'r') as zf: - currentAps = zf.read(config).split() - if currentAps != aps: - logv('[updating ' + config + ' in ' + apsJar + ']') - with Archiver(apsJar) as arc: - with zipfile.ZipFile(apsJar, 'r') as lp: - for arcname in lp.namelist(): - if arcname == config: - arc.zf.writestr(arcname, '\n'.join(aps)) - else: - arc.zf.writestr(arcname, lp.read(arcname)) - d.add_update_listener(_refineAnnotationProcessorServiceConfig) - self.dists.append(d) - - if self.name is None: - abort('Missing "suite=<name>" in ' + suitePyFile) - - def _commands_name(self): - return 'mx_' + self.name.replace('-', '_') - - def _find_commands(self, name): - commandsPath = join(self.mxDir, name + '.py') - if exists(commandsPath): - return name - else: - return None - - def _load_commands(self): - commandsName = self._find_commands(self._commands_name()) - if commandsName is None: - # backwards compatibility - commandsName = self._find_commands('commands') - if commandsName is not None: - if commandsName in sys.modules: - abort(commandsName + '.py in suite ' + self.name + ' duplicates ' + sys.modules[commandsName].__file__) - # temporarily extend the Python path - sys.path.insert(0, self.mxDir) - mod = __import__(commandsName) - - self.commands = sys.modules.pop(commandsName) - sys.modules[commandsName] = self.commands - - # revert the Python path - del sys.path[0] - - if not hasattr(mod, 'mx_init'): - abort(commandsName + '.py in suite ' + self.name + ' must define an mx_init(suite) function') - if hasattr(mod, 'mx_post_parse_cmd_line'): - self.mx_post_parse_cmd_line = mod.mx_post_parse_cmd_line - - mod.mx_init(self) - self.commands = mod - - def _load_imports(self): - if exists(join(self.mxDir, 'imports')): - abort('multiple suites are not supported in this version of mx') - - def _load_env(self): - e = join(self.mxDir, 'env') - if exists(e): - with open(e) as f: - lineNum = 0 - for line in f: - lineNum = lineNum + 1 - line = line.strip() - if len(line) != 0 and line[0] != '#': - if not '=' in line: - abort(e + ':' + str(lineNum) + ': line does not match pattern "key=value"') - key, value = line.split('=', 1) - os.environ[key.strip()] = expandvars_in_property(value.strip()) - - def _post_init(self, opts): - self._load_projects() - if self.requiredMxVersion is None: - warn("This suite does not express any required mx version. Consider adding 'mxversion=<version>' to your projects file.") - elif self.requiredMxVersion > version: - abort("This suite requires mx version " + str(self.requiredMxVersion) + " while your current mx version is " + str(version) + ". Please update mx.") - # set the global data structures, checking for conflicts unless _check_global_structures is False - for p in self.projects: - existing = _projects.get(p.name) - if existing is not None: - abort('cannot override project ' + p.name + ' in ' + p.dir + " with project of the same name in " + existing.dir) - if not p.name in _opts.ignored_projects: - _projects[p.name] = p - for l in self.libs: - existing = _libs.get(l.name) - # Check that suites that define same library are consistent - if existing is not None and existing != l: - abort('inconsistent library redefinition of ' + l.name + ' in ' + existing.suite.dir + ' and ' + l.suite.dir) - _libs[l.name] = l - for l in self.jreLibs: - existing = _jreLibs.get(l.name) - # Check that suites that define same library are consistent - if existing is not None and existing != l: - abort('inconsistent JRE library redefinition of ' + l.name + ' in ' + existing.suite.dir + ' and ' + l.suite.dir) - _jreLibs[l.name] = l - for d in self.dists: - existing = _dists.get(d.name) - if existing is not None: - # allow redefinition, so use path from existing - # abort('cannot redefine distribution ' + d.name) - warn('distribution ' + d.name + ' redefined') - d.path = existing.path - _dists[d.name] = d - - # Remove projects and libraries that (recursively) depend on an optional library - # whose artifact does not exist or on a JRE library that is not present in the - # JDK for a project. Also remove projects whose Java compliance requirement - # cannot be satisfied by the configured JDKs. - # - # Removed projects and libraries are also removed from - # distributions in they are listed as dependencies. - ommittedDeps = set() - for d in sorted_deps(includeLibs=True): - if d.isLibrary(): - if d.optional: - try: - d.optional = False - path = d.get_path(resolve=True) - except SystemExit: - path = None - finally: - d.optional = True - if not path: - logv('[omitting optional library {0} as {1} does not exist]'.format(d, d.path)) - ommittedDeps.add(d.name) - del _libs[d.name] - self.libs.remove(d) - elif d.isProject(): - if java(d.javaCompliance, cancel='some projects will be omitted which may result in errrors') is None: - logv('[omitting project {0} as Java compliance {1} cannot be satisfied by configured JDKs]'.format(d, d.javaCompliance)) - ommittedDeps.add(d.name) - del _projects[d.name] - self.projects.remove(d) - else: - for name in list(d.deps): - jreLib = _jreLibs.get(name) - if jreLib: - if not jreLib.is_present_in_jdk(java(d.javaCompliance)): - if jreLib.optional: - logv('[omitting project {0} as dependency {1} is missing]'.format(d, name)) - ommittedDeps.add(d.name) - del _projects[d.name] - self.projects.remove(d) - else: - abort('JRE library {0} required by {1} not found'.format(jreLib, d)) - elif not dependency(name, fatalIfMissing=False): - logv('[omitting project {0} as dependency {1} is missing]'.format(d, name)) - ommittedDeps.add(d.name) - del _projects[d.name] - self.projects.remove(d) - for dist in _dists.itervalues(): - for name in list(dist.deps): - if name in ommittedDeps: - logv('[omitting {0} from distribution {1}]'.format(name, dist)) - dist.deps.remove(name) - - if hasattr(self, 'mx_post_parse_cmd_line'): - self.mx_post_parse_cmd_line(opts) - -class XMLElement(xml.dom.minidom.Element): - def writexml(self, writer, indent="", addindent="", newl=""): - writer.write(indent + "<" + self.tagName) - - attrs = self._get_attributes() - a_names = attrs.keys() - a_names.sort() - - for a_name in a_names: - writer.write(" %s=\"" % a_name) - xml.dom.minidom._write_data(writer, attrs[a_name].value) - writer.write("\"") - if self.childNodes: - if not self.ownerDocument.padTextNodeWithoutSiblings and len(self.childNodes) == 1 and isinstance(self.childNodes[0], xml.dom.minidom.Text): - # if the only child of an Element node is a Text node, then the - # text is printed without any indentation or new line padding - writer.write(">") - self.childNodes[0].writexml(writer) - writer.write("</%s>%s" % (self.tagName, newl)) - else: - writer.write(">%s" % (newl)) - for node in self.childNodes: - node.writexml(writer, indent + addindent, addindent, newl) - writer.write("%s</%s>%s" % (indent, self.tagName, newl)) - else: - writer.write("/>%s" % (newl)) - -class XMLDoc(xml.dom.minidom.Document): - - def __init__(self): - xml.dom.minidom.Document.__init__(self) - self.current = self - self.padTextNodeWithoutSiblings = False - - def createElement(self, tagName): - # overwritten to create XMLElement - e = XMLElement(tagName) - e.ownerDocument = self - return e - - def comment(self, txt): - self.current.appendChild(self.createComment(txt)) - - def open(self, tag, attributes=None, data=None): - if attributes is None: - attributes = {} - element = self.createElement(tag) - for key, value in attributes.items(): - element.setAttribute(key, value) - self.current.appendChild(element) - self.current = element - if data is not None: - element.appendChild(self.createTextNode(data)) - return self - - def close(self, tag): - assert self.current != self - assert tag == self.current.tagName, str(tag) + ' != ' + self.current.tagName - self.current = self.current.parentNode - return self - - def element(self, tag, attributes=None, data=None): - if attributes is None: - attributes = {} - return self.open(tag, attributes, data).close(tag) - - def xml(self, indent='', newl='', escape=False, standalone=None): - assert self.current == self - result = self.toprettyxml(indent, newl, encoding="UTF-8") - if escape: - entities = {'"': """, "'": "'", '\n': ' '} - result = xml.sax.saxutils.escape(result, entities) - if standalone is not None: - result = result.replace('encoding="UTF-8"?>', 'encoding="UTF-8" standalone="' + str(standalone) + '"?>') - return result - -def get_jython_os(): - from java.lang import System as System - os_name = System.getProperty('os.name').lower() - if System.getProperty('isCygwin'): - return 'cygwin' - elif os_name.startswith('mac'): - return 'darwin' - elif os_name.startswith('linux'): - return 'linux' - elif os_name.startswith('sunos'): - return 'solaris' - elif os_name.startswith('win'): - return 'windows' - else: - abort('Unknown operating system ' + os_name) - -def get_os(): - """ - Get a canonical form of sys.platform. - """ - if is_jython(): - return get_jython_os() - elif sys.platform.startswith('darwin'): - return 'darwin' - elif sys.platform.startswith('linux'): - return 'linux' - elif sys.platform.startswith('sunos'): - return 'solaris' - elif sys.platform.startswith('win32'): - return 'windows' - elif sys.platform.startswith('cygwin'): - return 'cygwin' - else: - abort('Unknown operating system ' + sys.platform) - -def _cygpathU2W(p): - """ - Translate a path from unix-style to windows-style. - This method has no effects on other platforms than cygwin. - """ - if p is None or get_os() != "cygwin": - return p - return subprocess.check_output(['cygpath', '-a', '-w', p]).strip() - -def _cygpathW2U(p): - """ - Translate a path from windows-style to unix-style. - This method has no effects on other platforms than cygwin. - """ - if p is None or get_os() != "cygwin": - return p - return subprocess.check_output(['cygpath', '-a', '-u', p]).strip() - -def _separatedCygpathU2W(p): - """ - Translate a group of paths, separated by a path separator. - unix-style to windows-style. - This method has no effects on other platforms than cygwin. - """ - if p is None or p == "" or get_os() != "cygwin": - return p - return ';'.join(map(_cygpathU2W, p.split(os.pathsep))) - -def _separatedCygpathW2U(p): - """ - Translate a group of paths, separated by a path separator. - windows-style to unix-style. - This method has no effects on other platforms than cygwin. - """ - if p is None or p == "" or get_os() != "cygwin": - return p - return os.pathsep.join(map(_cygpathW2U, p.split(';'))) - -def get_arch(): - machine = platform.uname()[4] - if machine in ['amd64', 'AMD64', 'x86_64', 'i86pc']: - return 'amd64' - if machine in ['sun4v', 'sun4u']: - return 'sparcv9' - if machine == 'i386' and get_os() == 'darwin': - try: - # Support for Snow Leopard and earlier version of MacOSX - if subprocess.check_output(['sysctl', '-n', 'hw.cpu64bit_capable']).strip() == '1': - return 'amd64' - except OSError: - # sysctl is not available - pass - abort('unknown or unsupported architecture: os=' + get_os() + ', machine=' + machine) - -def _loadSuite(mxDir, primary=False): - """ - Load a suite from 'mxDir'. - """ - for s in _suites.itervalues(): - if s.mxDir == mxDir: - return s - # create the new suite - s = Suite(mxDir, primary) - return s - -def suites(opt_limit_to_suite=False): - """ - Get the list of all loaded suites. - """ - return _suites.values() - -def suite(name, fatalIfMissing=True): - """ - Get the suite for a given name. - """ - s = _suites.get(name) - if s is None and fatalIfMissing: - abort('suite named ' + name + ' not found') - return s - - -def projects_from_names(projectNames): - """ - Get the list of projects corresponding to projectNames; all projects if None - """ - if projectNames is None: - return projects() - else: - return [project(name) for name in projectNames] - -def projects(opt_limit_to_suite=False): - """ - Get the list of all loaded projects limited by --suite option if opt_limit_to_suite == True - """ - sortedProjects = sorted(_projects.values(), key=lambda p: p.name) - if opt_limit_to_suite: - return _projects_opt_limit_to_suites(sortedProjects) - else: - return sortedProjects - -def projects_opt_limit_to_suites(): - """ - Get the list of all loaded projects optionally limited by --suite option - """ - return projects(True) - -def _projects_opt_limit_to_suites(projects): - return projects - -def annotation_processors(): - """ - Get the list of all loaded projects that define an annotation processor. - """ - global _annotationProcessors - if _annotationProcessors is None: - aps = set() - for p in projects(): - for ap in p.annotation_processors(): - if project(ap, False): - aps.add(ap) - _annotationProcessors = list(aps) - return _annotationProcessors - -def distribution(name, fatalIfMissing=True): - """ - Get the distribution for a given name. This will abort if the named distribution does - not exist and 'fatalIfMissing' is true. - """ - d = _dists.get(name) - if d is None and fatalIfMissing: - abort('distribution named ' + name + ' not found') - return d - -def dependency(name, fatalIfMissing=True): - """ - Get the project or library for a given name. This will abort if a project or library does - not exist for 'name' and 'fatalIfMissing' is true. - """ - d = _projects.get(name) - if d is None: - d = _libs.get(name) - if d is None: - d = _jreLibs.get(name) - if d is None and fatalIfMissing: - if name in _opts.ignored_projects: - abort('project named ' + name + ' is ignored') - abort('project or library named ' + name + ' not found') - return d - -def project(name, fatalIfMissing=True): - """ - Get the project for a given name. This will abort if the named project does - not exist and 'fatalIfMissing' is true. - """ - p = _projects.get(name) - if p is None and fatalIfMissing: - if name in _opts.ignored_projects: - abort('project named ' + name + ' is ignored') - abort('project named ' + name + ' not found') - return p - -def library(name, fatalIfMissing=True): - """ - Gets the library for a given name. This will abort if the named library does - not exist and 'fatalIfMissing' is true. - """ - l = _libs.get(name) - if l is None and fatalIfMissing: - if _projects.get(name): - abort(name + ' is a project, not a library') - abort('library named ' + name + ' not found') - return l - -def _as_classpath(deps, resolve): - cp = [] - if _opts.cp_prefix is not None: - cp = [_opts.cp_prefix] - for d in deps: - d.append_to_classpath(cp, resolve) - if _opts.cp_suffix is not None: - cp += [_opts.cp_suffix] - return os.pathsep.join(cp) - -def classpath(names=None, resolve=True, includeSelf=True, includeBootClasspath=False): - """ - Get the class path for a list of given dependencies and distributions, resolving each entry in the - path (e.g. downloading a missing library) if 'resolve' is true. - """ - if names is None: - deps = sorted_deps(includeLibs=True) - dists = list(_dists.values()) - else: - deps = [] - dists = [] - if isinstance(names, types.StringTypes): - names = [names] - for n in names: - dep = dependency(n, fatalIfMissing=False) - if dep: - dep.all_deps(deps, True, includeSelf) - else: - dist = distribution(n) - if not dist: - abort('project, library or distribution named ' + n + ' not found') - dists.append(dist) - - if len(dists): - distsDeps = set() - for d in dists: - distsDeps.update(d.sorted_deps()) - - # remove deps covered by a dist that will be on the class path - deps = [d for d in deps if d not in distsDeps] - - result = _as_classpath(deps, resolve) - - # prepend distributions - if len(dists): - distsCp = os.pathsep.join(dist.path for dist in dists) - if len(result): - result = distsCp + os.pathsep + result - else: - result = distsCp - - if includeBootClasspath: - result = os.pathsep.join([java().bootclasspath(), result]) - - return result - -def classpath_walk(names=None, resolve=True, includeSelf=True, includeBootClasspath=False): - """ - Walks the resources available in a given classpath, yielding a tuple for each resource - where the first member of the tuple is a directory path or ZipFile object for a - classpath entry and the second member is the qualified path of the resource relative - to the classpath entry. - """ - cp = classpath(names, resolve, includeSelf, includeBootClasspath) - for entry in cp.split(os.pathsep): - if not exists(entry): - continue - if isdir(entry): - for root, dirs, files in os.walk(entry): - for d in dirs: - entryPath = join(root[len(entry) + 1:], d) - yield entry, entryPath - for f in files: - entryPath = join(root[len(entry) + 1:], f) - yield entry, entryPath - elif entry.endswith('.jar') or entry.endswith('.zip'): - with zipfile.ZipFile(entry, 'r') as zf: - for zi in zf.infolist(): - entryPath = zi.filename - yield zf, entryPath - -def sorted_deps(projectNames=None, includeLibs=False, includeJreLibs=False, includeAnnotationProcessors=False): - """ - Gets projects and libraries sorted such that dependencies - are before the projects that depend on them. Unless 'includeLibs' is - true, libraries are omitted from the result. - """ - projects = projects_from_names(projectNames) - - return sorted_project_deps(projects, includeLibs=includeLibs, includeJreLibs=includeJreLibs, includeAnnotationProcessors=includeAnnotationProcessors) - -def sorted_dists(): - """ - Gets distributions sorted such that each distribution comes after - any distributions it depends upon. - """ - dists = [] - def add_dist(dist): - if not dist in dists: - for depDist in [distribution(name) for name in dist.distDependencies]: - add_dist(depDist) - if not dist in dists: - dists.append(dist) - - for d in _dists.itervalues(): - add_dist(d) - return dists - -def sorted_project_deps(projects, includeLibs=False, includeJreLibs=False, includeAnnotationProcessors=False): - deps = [] - for p in projects: - p.all_deps(deps, includeLibs=includeLibs, includeJreLibs=includeJreLibs, includeAnnotationProcessors=includeAnnotationProcessors) - return deps - -class ArgParser(ArgumentParser): - # Override parent to append the list of available commands - def format_help(self): - return ArgumentParser.format_help(self) + _format_commands() - - - def __init__(self): - self.java_initialized = False - # this doesn't resolve the right way, but can't figure out how to override _handle_conflict_resolve in _ActionsContainer - ArgumentParser.__init__(self, prog='mx', conflict_handler='resolve') - - self.add_argument('-v', action='store_true', dest='verbose', help='enable verbose output') - self.add_argument('-V', action='store_true', dest='very_verbose', help='enable very verbose output') - self.add_argument('-w', action='store_true', dest='warn', help='enable warning messages') - self.add_argument('-p', '--primary-suite-path', help='set the primary suite directory', metavar='<path>') - self.add_argument('--dbg', type=int, dest='java_dbg_port', help='make Java processes wait on <port> for a debugger', metavar='<port>') - self.add_argument('-d', action='store_const', const=8000, dest='java_dbg_port', help='alias for "-dbg 8000"') - self.add_argument('--attach', dest='attach', help='Connect to existing server running at [<address>:]<port>') - self.add_argument('--backup-modified', action='store_true', help='backup generated files if they pre-existed and are modified') - self.add_argument('--cp-pfx', dest='cp_prefix', help='class path prefix', metavar='<arg>') - self.add_argument('--cp-sfx', dest='cp_suffix', help='class path suffix', metavar='<arg>') - self.add_argument('--J', dest='java_args', help='Java VM arguments (e.g. --J @-dsa)', metavar='@<args>') - self.add_argument('--Jp', action='append', dest='java_args_pfx', help='prefix Java VM arguments (e.g. --Jp @-dsa)', metavar='@<args>', default=[]) - self.add_argument('--Ja', action='append', dest='java_args_sfx', help='suffix Java VM arguments (e.g. --Ja @-dsa)', metavar='@<args>', default=[]) - self.add_argument('--user-home', help='users home directory', metavar='<path>', default=os.path.expanduser('~')) - self.add_argument('--java-home', help='primary JDK directory (must be JDK 7 or later)', metavar='<path>') - self.add_argument('--extra-java-homes', help='secondary JDK directories separated by "' + os.pathsep + '"', metavar='<path>') - self.add_argument('--strict-compliance', action='store_true', dest='strict_compliance', help='Projects with an explicit compliance will only be built if a JDK exactly matching the compliance is available', default=False) - self.add_argument('--ignore-project', action='append', dest='ignored_projects', help='name of project to ignore', metavar='<name>', default=[]) - self.add_argument('--kill-with-sigquit', action='store_true', dest='killwithsigquit', help='send sigquit first before killing child processes') - if get_os() != 'windows': - # Time outs are (currently) implemented with Unix specific functionality - self.add_argument('--timeout', help='timeout (in seconds) for command', type=int, default=0, metavar='<secs>') - self.add_argument('--ptimeout', help='timeout (in seconds) for subprocesses', type=int, default=0, metavar='<secs>') - - def _parse_cmd_line(self, args=None): - if args is None: - args = sys.argv[1:] - - self.add_argument('commandAndArgs', nargs=REMAINDER, metavar='command args...') - - opts = self.parse_args() - - global _opts - _opts = opts - - # Give the timeout options a default value to avoid the need for hasattr() tests - opts.__dict__.setdefault('timeout', 0) - opts.__dict__.setdefault('ptimeout', 0) - - if opts.very_verbose: - opts.verbose = True - - if opts.user_home is None or opts.user_home == '': - abort('Could not find user home. Use --user-home option or ensure HOME environment variable is set.') - - if opts.java_home: - os.environ['JAVA_HOME'] = opts.java_home - os.environ['HOME'] = opts.user_home - - if os.environ.get('STRICT_COMPLIANCE'): - _opts.strict_compliance = True - - opts.ignored_projects = opts.ignored_projects + os.environ.get('IGNORED_PROJECTS', '').split(',') - - commandAndArgs = opts.__dict__.pop('commandAndArgs') - return opts, commandAndArgs - - def _handle_conflict_resolve(self, action, conflicting_actions): - self._handle_conflict_error(action, conflicting_actions) - -def _format_commands(): - msg = '\navailable commands:\n\n' - for cmd in sorted(_commands.iterkeys()): - c, _ = _commands[cmd][:2] - doc = c.__doc__ - if doc is None: - doc = '' - msg += ' {0:<20} {1}\n'.format(cmd, doc.split('\n', 1)[0]) - return msg + '\n' - -_canceled_java_requests = set() - -def java(requiredCompliance=None, purpose=None, cancel=None): - """ - Get a JavaConfig object containing Java commands launch details. - If requiredCompliance is None, the compliance level specified by --java-home/JAVA_HOME - is returned. Otherwise, the JavaConfig exactly matching requiredCompliance is returned - or None if there is no exact match. - """ - - global _default_java_home - if cancel and (requiredCompliance, purpose) in _canceled_java_requests: - return None - - if not requiredCompliance: - if not _default_java_home: - _default_java_home = _find_jdk(purpose=purpose, cancel=cancel) - if not _default_java_home: - assert cancel - _canceled_java_requests.add((requiredCompliance, purpose)) - return _default_java_home - - if _opts.strict_compliance: - complianceCheck = requiredCompliance.exactMatch - desc = str(requiredCompliance) - else: - compVersion = VersionSpec(str(requiredCompliance)) - complianceCheck = lambda version: version >= compVersion - desc = '>=' + str(requiredCompliance) - - for java in _extra_java_homes: - if complianceCheck(java.version): - return java - - jdk = _find_jdk(versionCheck=complianceCheck, versionDescription=desc, purpose=purpose, cancel=cancel) - if jdk: - assert jdk not in _extra_java_homes - _extra_java_homes.append(jdk) - else: - assert cancel - _canceled_java_requests.add((requiredCompliance, purpose)) - return jdk - -def java_version(versionCheck, versionDescription=None, purpose=None): - if _default_java_home and versionCheck(_default_java_home.version): - return _default_java_home - for java in _extra_java_homes: - if versionCheck(java.version): - return java - jdk = _find_jdk(versionCheck, versionDescription, purpose) - assert jdk not in _extra_java_homes - _extra_java_homes.append(jdk) - return jdk - -def _find_jdk(versionCheck=None, versionDescription=None, purpose=None, cancel=None): - if not versionCheck: - versionCheck = lambda v: True - assert not versionDescription or versionCheck - if not versionCheck and not purpose: - isDefaultJdk = True - else: - isDefaultJdk = False - - candidateJdks = [] - source = '' - if _opts.java_home: - candidateJdks.append(_opts.java_home) - source = '--java-home' - elif os.environ.get('JAVA_HOME'): - candidateJdks.append(os.environ.get('JAVA_HOME')) - source = 'JAVA_HOME' - - result = _find_jdk_in_candidates(candidateJdks, versionCheck, warn=True, source=source) - if result: - return result - - candidateJdks = [] - - if _opts.extra_java_homes: - candidateJdks += _opts.extra_java_homes.split(os.pathsep) - source = '--extra-java-homes' - elif os.environ.get('EXTRA_JAVA_HOMES'): - candidateJdks += os.environ.get('EXTRA_JAVA_HOMES').split(os.pathsep) - source = 'EXTRA_JAVA_HOMES' - - result = _find_jdk_in_candidates(candidateJdks, versionCheck, warn=True, source=source) - if not result: - candidateJdks = [] - source = '' - - if get_os() == 'darwin': - base = '/Library/Java/JavaVirtualMachines' - if exists(base): - candidateJdks = [join(base, n, 'Contents/Home') for n in os.listdir(base)] - elif get_os() == 'linux': - base = '/usr/lib/jvm' - if exists(base): - candidateJdks = [join(base, n) for n in os.listdir(base)] - base = '/usr/java' - if exists(base): - candidateJdks += [join(base, n) for n in os.listdir(base)] - elif get_os() == 'solaris': - base = '/usr/jdk/instances' - if exists(base): - candidateJdks = [join(base, n) for n in os.listdir(base)] - elif get_os() == 'windows': - base = r'C:\Program Files\Java' - if exists(base): - candidateJdks = [join(base, n) for n in os.listdir(base)] - - configs = _filtered_jdk_configs(candidateJdks, versionCheck) - else: - if not isDefaultJdk: - return result - configs = [result] - - if len(configs) > 1: - if not is_interactive(): - msg = "Multiple possible choices for a JDK" - if purpose: - msg += ' for' + purpose - msg += ': ' - if versionDescription: - msg += '(' + versionDescription + ')' - selected = configs[0] - msg += ". Selecting " + str(selected) - log(msg) - else: - msg = 'Please select a ' - if isDefaultJdk: - msg += 'default ' - msg += 'JDK' - if purpose: - msg += ' for' + purpose - msg += ': ' - if versionDescription: - msg += '(' + versionDescription + ')' - log(msg) - choices = configs + ['<other>'] - if cancel: - choices.append('Cancel (' + cancel + ')') - selected = select_items(choices, allowMultiple=False) - if isinstance(selected, types.StringTypes) and selected == '<other>': - selected = None - if isinstance(selected, types.StringTypes) and selected == 'Cancel (' + cancel + ')': - return None - elif len(configs) == 1: - selected = configs[0] - msg = 'Selected ' + str(selected) + ' as ' - if isDefaultJdk: - msg += 'default' - msg += 'JDK' - if versionDescription: - msg = msg + ' ' + versionDescription - if purpose: - msg += ' for' + purpose - log(msg) - else: - msg = 'Could not find any JDK' - if purpose: - msg += ' for' + purpose - msg += ' ' - if versionDescription: - msg = msg + '(' + versionDescription + ')' - log(msg) - selected = None - - while not selected: - jdkLocation = raw_input('Enter path of JDK: ') - selected = _find_jdk_in_candidates([jdkLocation], versionCheck, warn=True) - - varName = 'JAVA_HOME' if isDefaultJdk else 'EXTRA_JAVA_HOMES' - allowMultiple = not isDefaultJdk - envPath = join(_primary_suite.mxDir, 'env') - if is_interactive() and ask_yes_no('Persist this setting by adding "{0}={1}" to {2}'.format(varName, selected.jdk, envPath), 'y'): - envLines = [] - with open(envPath) as fp: - append = True - for line in fp: - if line.rstrip().startswith(varName): - _, currentValue = line.split('=', 1) - currentValue = currentValue.strip() - if not allowMultiple and currentValue: - if not ask_yes_no('{0} is already set to {1}, overwrite with {2}?'.format(varName, currentValue, selected.jdk), 'n'): - return selected - else: - line = varName + '=' + selected.jdk + os.linesep - else: - line = line.rstrip() - if currentValue: - line += os.pathsep - line += selected.jdk + os.linesep - append = False - envLines.append(line) - if append: - envLines.append(varName + '=' + selected.jdk) - - with open(envPath, 'w') as fp: - for line in envLines: - fp.write(line) - - if varName == 'JAVA_HOME': - os.environ['JAVA_HOME'] = selected.jdk - - return selected - -def is_interactive(): - return sys.__stdin__.isatty() - -def _filtered_jdk_configs(candidates, versionCheck, warn=False, source=None): - filtered = [] - for candidate in candidates: - try: - config = JavaConfig(candidate) - if versionCheck(config.version): - filtered.append(config) - except JavaConfigException as e: - if warn: - log('Path in ' + source + "' is not pointing to a JDK (" + e.message + ")") - return filtered - -def _find_jdk_in_candidates(candidates, versionCheck, warn=False, source=None): - filtered = _filtered_jdk_configs(candidates, versionCheck, warn, source) - if filtered: - return filtered[0] - return None - - -def run_java(args, nonZeroIsFatal=True, out=None, err=None, cwd=None, addDefaultArgs=True, javaConfig=None): - if not javaConfig: - javaConfig = java() - return run(javaConfig.format_cmd(args, addDefaultArgs), nonZeroIsFatal=nonZeroIsFatal, out=out, err=err, cwd=cwd) - -def _kill_process_group(pid, sig): - if not sig: - sig = signal.SIGKILL - pgid = os.getpgid(pid) - try: - os.killpg(pgid, sig) - return True - except: - log('Error killing subprocess ' + str(pgid) + ': ' + str(sys.exc_info()[1])) - return False - -def _waitWithTimeout(process, args, timeout): - def _waitpid(pid): - while True: - try: - return os.waitpid(pid, os.WNOHANG) - except OSError, e: - if e.errno == errno.EINTR: - continue - raise - - def _returncode(status): - if os.WIFSIGNALED(status): - return -os.WTERMSIG(status) - elif os.WIFEXITED(status): - return os.WEXITSTATUS(status) - else: - # Should never happen - raise RuntimeError("Unknown child exit status!") - - end = time.time() + timeout - delay = 0.0005 - while True: - (pid, status) = _waitpid(process.pid) - if pid == process.pid: - return _returncode(status) - remaining = end - time.time() - if remaining <= 0: - abort('Process timed out after {0} seconds: {1}'.format(timeout, ' '.join(args))) - delay = min(delay * 2, remaining, .05) - time.sleep(delay) - -# Makes the current subprocess accessible to the abort() function -# This is a list of tuples of the subprocess.Popen or -# multiprocessing.Process object and args. -_currentSubprocesses = [] - -def _addSubprocess(p, args): - entry = (p, args) - _currentSubprocesses.append(entry) - return entry - -def _removeSubprocess(entry): - if entry and entry in _currentSubprocesses: - try: - _currentSubprocesses.remove(entry) - except: - pass - -def waitOn(p): - if get_os() == 'windows': - # on windows use a poll loop, otherwise signal does not get handled - retcode = None - while retcode == None: - retcode = p.poll() - time.sleep(0.05) - else: - retcode = p.wait() - return retcode - -def run(args, nonZeroIsFatal=True, out=None, err=None, cwd=None, timeout=None, env=None): - """ - Run a command in a subprocess, wait for it to complete and return the exit status of the process. - If the exit status is non-zero and `nonZeroIsFatal` is true, then mx is exited with - the same exit status. - Each line of the standard output and error streams of the subprocess are redirected to - out and err if they are callable objects. - """ - - assert isinstance(args, types.ListType), "'args' must be a list: " + str(args) - for arg in args: - assert isinstance(arg, types.StringTypes), 'argument is not a string: ' + str(arg) - - if env is None: - env = os.environ.copy() - - # Ideally the command line could be communicated directly in an environment - # variable. However, since environment variables share the same resource - # space as the command line itself (on Unix at least), this would cause the - # limit to be exceeded too easily. - with tempfile.NamedTemporaryFile(suffix='', prefix='mx_subprocess_command.', mode='w', delete=False) as fp: - subprocessCommandFile = fp.name - for arg in args: - # TODO: handle newlines in args once there's a use case - assert '\n' not in arg - print >> fp, arg - env['MX_SUBPROCESS_COMMAND_FILE'] = subprocessCommandFile - - if _opts.verbose: - if _opts.very_verbose: - log('Environment variables:') - for key in sorted(env.keys()): - log(' ' + key + '=' + env[key]) - log(' '.join(map(pipes.quote, args))) - - if timeout is None and _opts.ptimeout != 0: - timeout = _opts.ptimeout - - sub = None - try: - # On Unix, the new subprocess should be in a separate group so that a timeout alarm - # can use os.killpg() to kill the whole subprocess group - preexec_fn = None - creationflags = 0 - if not is_jython(): - if get_os() == 'windows': - creationflags = subprocess.CREATE_NEW_PROCESS_GROUP - else: - preexec_fn = os.setsid - def redirect(stream, f): - for line in iter(stream.readline, ''): - f(line) - stream.close() - stdout = out if not callable(out) else subprocess.PIPE - stderr = err if not callable(err) else subprocess.PIPE - p = subprocess.Popen(args, cwd=cwd, stdout=stdout, stderr=stderr, preexec_fn=preexec_fn, creationflags=creationflags, env=env) - sub = _addSubprocess(p, args) - joiners = [] - if callable(out): - t = Thread(target=redirect, args=(p.stdout, out)) - # Don't make the reader thread a daemon otherwise output can be droppped - t.start() - joiners.append(t) - if callable(err): - t = Thread(target=redirect, args=(p.stderr, err)) - # Don't make the reader thread a daemon otherwise output can be droppped - t.start() - joiners.append(t) - while any([t.is_alive() for t in joiners]): - # Need to use timeout otherwise all signals (including CTRL-C) are blocked - # see: http://bugs.python.org/issue1167930 - for t in joiners: - t.join(10) - if timeout is None or timeout == 0: - retcode = waitOn(p) - else: - if get_os() == 'windows': - abort('Use of timeout not (yet) supported on Windows') - retcode = _waitWithTimeout(p, args, timeout) - except OSError as e: - log('Error executing \'' + ' '.join(args) + '\': ' + str(e)) - if _opts.verbose: - raise e - abort(e.errno) - except KeyboardInterrupt: - abort(1) - finally: - _removeSubprocess(sub) - os.remove(subprocessCommandFile) - - if retcode and nonZeroIsFatal: - if _opts.verbose: - if _opts.very_verbose: - raise subprocess.CalledProcessError(retcode, ' '.join(args)) - else: - log('[exit code: ' + str(retcode) + ']') - abort(retcode) - - return retcode - -def exe_suffix(name): - """ - Gets the platform specific suffix for an executable - """ - if get_os() == 'windows': - return name + '.exe' - return name - -def add_lib_prefix(name): - """ - Adds the platform specific library prefix to a name - """ - os = get_os() - if os == 'linux' or os == 'solaris' or os == 'darwin': - return 'lib' + name - return name - -def add_lib_suffix(name): - """ - Adds the platform specific library suffix to a name - """ - os = get_os() - if os == 'windows': - return name + '.dll' - if os == 'linux' or os == 'solaris': - return name + '.so' - if os == 'darwin': - return name + '.dylib' - return name - -""" -Utility for filtering duplicate lines. -""" -class DuplicateSuppressingStream: - """ - Creates an object that will suppress duplicate lines sent to 'out'. - The lines considered for suppression are those that contain one of the - strings in 'restrictTo' if it is not None. - """ - def __init__(self, restrictTo=None, out=sys.stdout): - self.restrictTo = restrictTo - self.seen = set() - self.out = out - self.currentFilteredLineCount = 0 - self.currentFilteredTime = None - - def isSuppressionCandidate(self, line): - if self.restrictTo: - for p in self.restrictTo: - if p in line: - return True - return False - else: - return True - - def write(self, line): - if self.isSuppressionCandidate(line): - if line in self.seen: - self.currentFilteredLineCount += 1 - if self.currentFilteredTime: - if time.time() - self.currentFilteredTime > 1 * 60: - self.out.write(" Filtered " + str(self.currentFilteredLineCount) + " repeated lines...\n") - self.currentFilteredTime = time.time() - else: - self.currentFilteredTime = time.time() - return - self.seen.add(line) - self.currentFilteredLineCount = 0 - self.out.write(line) - self.currentFilteredTime = None - -""" -A JavaCompliance simplifies comparing Java compliance values extracted from a JDK version string. -""" -class JavaCompliance: - def __init__(self, ver): - m = re.match(r'1\.(\d+).*', ver) - assert m is not None, 'not a recognized version string: ' + ver - self.value = int(m.group(1)) - - def __str__(self): - return '1.' + str(self.value) - - def __cmp__(self, other): - if isinstance(other, types.StringType): - other = JavaCompliance(other) - - return cmp(self.value, other.value) - - def __hash__(self): - return self.value.__hash__() - - def exactMatch(self, version): - assert isinstance(version, VersionSpec) - return len(version.parts) > 1 and version.parts[0] == 1 and version.parts[1] == self.value - -""" -A version specification as defined in JSR-56 -""" -class VersionSpec: - def __init__(self, versionString): - validChar = r'[\x21-\x25\x27-\x29\x2c\x2f-\x5e\x60-\x7f]' - separator = r'[.\-_]' - m = re.match("^" + validChar + '+(' + separator + validChar + '+)*$', versionString) - assert m is not None, 'not a recognized version string: ' + versionString - self.versionString = versionString - self.parts = [int(f) if f.isdigit() else f for f in re.split(separator, versionString)] - - def __str__(self): - return self.versionString - - def __cmp__(self, other): - return cmp(self.parts, other.parts) - -def _filter_non_existant_paths(paths): - if paths: - return os.pathsep.join([path for path in _separatedCygpathW2U(paths).split(os.pathsep) if exists(path)]) - return None - -class JavaConfigException(Exception): - def __init__(self, value): - Exception.__init__(self, value) - -""" -A JavaConfig object encapsulates info on how Java commands are run. -""" -class JavaConfig: - def __init__(self, java_home): - self.jdk = java_home - self.jar = exe_suffix(join(self.jdk, 'bin', 'jar')) - self.java = exe_suffix(join(self.jdk, 'bin', 'java')) - self.javac = exe_suffix(join(self.jdk, 'bin', 'javac')) - self.javap = exe_suffix(join(self.jdk, 'bin', 'javap')) - self.javadoc = exe_suffix(join(self.jdk, 'bin', 'javadoc')) - self.pack200 = exe_suffix(join(self.jdk, 'bin', 'pack200')) - self.toolsjar = join(self.jdk, 'lib', 'tools.jar') - self._classpaths_initialized = False - self._bootclasspath = None - self._extdirs = None - self._endorseddirs = None - - if not exists(self.java): - raise JavaConfigException('Java launcher does not exist: ' + self.java) - - def delAtAndSplit(s): - return shlex.split(s.lstrip('@')) - - self.java_args = delAtAndSplit(_opts.java_args) if _opts.java_args else [] - self.java_args_pfx = sum(map(delAtAndSplit, _opts.java_args_pfx), []) - self.java_args_sfx = sum(map(delAtAndSplit, _opts.java_args_sfx), []) - - # Prepend the -d64 VM option only if the java command supports it - try: - output = subprocess.check_output([self.java, '-d64', '-version'], stderr=subprocess.STDOUT) - self.java_args = ['-d64'] + self.java_args - except subprocess.CalledProcessError as e: - try: - output = subprocess.check_output([self.java, '-version'], stderr=subprocess.STDOUT) - except subprocess.CalledProcessError as e: - raise JavaConfigException(e.returncode + " :" + e.output) - - def _checkOutput(out): - return 'version' in out - - # hotspot can print a warning, e.g. if there's a .hotspot_compiler file in the cwd - output = output.split('\n') - version = None - for o in output: - if _checkOutput(o): - assert version is None - version = o - - self.version = VersionSpec(version.split()[2].strip('"')) - self.javaCompliance = JavaCompliance(self.version.versionString) - - attach = None - if _opts.attach is not None: - attach = 'server=n,address=' + _opts.attach - else: - if _opts.java_dbg_port is not None: - attach = 'server=y,address=' + str(_opts.java_dbg_port) - - if attach is not None: - self.java_args += ['-Xdebug', '-Xrunjdwp:transport=dt_socket,' + attach + ',suspend=y'] - - def _init_classpaths(self): - if not self._classpaths_initialized: - _, binDir = _compile_mx_class('ClasspathDump', jdk=self) - self._bootclasspath, self._extdirs, self._endorseddirs = [x if x != 'null' else None for x in subprocess.check_output([self.java, '-cp', _cygpathU2W(binDir), 'ClasspathDump'], stderr=subprocess.PIPE).split('|')] - if self.javaCompliance <= JavaCompliance('1.8'): - # All 3 system properties accessed by ClasspathDump are expected to exist - if not self._bootclasspath or not self._extdirs or not self._endorseddirs: - warn("Could not find all classpaths: boot='" + str(self._bootclasspath) + "' extdirs='" + str(self._extdirs) + "' endorseddirs='" + str(self._endorseddirs) + "'") - self._bootclasspath = _filter_non_existant_paths(self._bootclasspath) - self._extdirs = _filter_non_existant_paths(self._extdirs) - self._endorseddirs = _filter_non_existant_paths(self._endorseddirs) - self._classpaths_initialized = True - - def __repr__(self): - return "JavaConfig(" + str(self.jdk) + ")" - - def __str__(self): - return "Java " + str(self.version) + " (" + str(self.javaCompliance) + ") from " + str(self.jdk) - - def __hash__(self): - return hash(self.jdk) - - def __cmp__(self, other): - if isinstance(other, JavaConfig): - compilanceCmp = cmp(self.javaCompliance, other.javaCompliance) - if compilanceCmp: - return compilanceCmp - versionCmp = cmp(self.version, other.version) - if versionCmp: - return versionCmp - return cmp(self.jdk, other.jdk) - raise TypeError() - - def format_cmd(self, args, addDefaultArgs): - if addDefaultArgs: - return [self.java] + self.processArgs(args) - else: - return [self.java] + args - - def processArgs(self, args): - return self.java_args_pfx + self.java_args + self.java_args_sfx + args - - def bootclasspath(self): - self._init_classpaths() - return _separatedCygpathU2W(self._bootclasspath) - - - """ - Add javadoc style options for the library paths of this JDK. - """ - def javadocLibOptions(self, args): - self._init_classpaths() - if args is None: - args = [] - if self._bootclasspath: - args.append('-bootclasspath') - args.append(self._bootclasspath) - if self._extdirs: - args.append('-extdirs') - args.append(self._extdirs) - return args - - """ - Add javac style options for the library paths of this JDK. - """ - def javacLibOptions(self, args): - args = self.javadocLibOptions(args) - if self._endorseddirs: - args.append('-endorseddirs') - args.append(self._endorseddirs) - return args - - def containsJar(self, jar): - self._init_classpaths() - - if self._bootclasspath: - for e in self._bootclasspath.split(os.pathsep): - if basename(e) == jar: - return True - if self._extdirs: - for d in self._extdirs.split(os.pathsep): - if len(d) and jar in os.listdir(d): - return True - if self._endorseddirs: - for d in self._endorseddirs.split(os.pathsep): - if len(d) and jar in os.listdir(d): - return True - return False - -def check_get_env(key): - """ - Gets an environment variable, aborting with a useful message if it is not set. - """ - value = get_env(key) - if value is None: - abort('Required environment variable ' + key + ' must be set') - return value - -def get_env(key, default=None): - """ - Gets an environment variable. - """ - value = os.environ.get(key, default) - return value - -def logv(msg=None): - if _opts.verbose: - log(msg) - -def log(msg=None): - """ - Write a message to the console. - All script output goes through this method thus allowing a subclass - to redirect it. - """ - if msg is None: - print - else: - print msg - -def expand_project_in_class_path_arg(cpArg): - cp = [] - for part in cpArg.split(os.pathsep): - if part.startswith('@'): - cp += classpath(part[1:]).split(os.pathsep) - else: - cp.append(part) - return os.pathsep.join(cp) - -def expand_project_in_args(args): - for i in range(len(args)): - if args[i] == '-cp' or args[i] == '-classpath': - if i + 1 < len(args): - args[i + 1] = expand_project_in_class_path_arg(args[i + 1]) - return - - -def gmake_cmd(): - for a in ['make', 'gmake', 'gnumake']: - try: - output = subprocess.check_output([a, '--version']) - if 'GNU' in output: - return a - except: - pass - abort('Could not find a GNU make executable on the current path.') - -def expandvars_in_property(value): - result = expandvars(value) - if '$' in result or '%' in result: - abort('Property contains an undefined environment variable: ' + value) - return result - -def _send_sigquit(): - for p, args in _currentSubprocesses: - - def _isJava(): - if args: - name = args[0].split(os.sep)[-1] - return name == "java" - return False - - if p is not None and _isJava(): - if get_os() == 'windows': - log("mx: implement me! want to send SIGQUIT to my child process") - else: - _kill_process_group(p.pid, sig=signal.SIGQUIT) - time.sleep(0.1) - -def abort(codeOrMessage): - """ - Aborts the program with a SystemExit exception. - If 'codeOrMessage' is a plain integer, it specifies the system exit status; - if it is None, the exit status is zero; if it has another type (such as a string), - the object's value is printed and the exit status is one. - """ - - if _opts and _opts.killwithsigquit: - _send_sigquit() - - def is_alive(p): - if isinstance(p, subprocess.Popen): - return p.poll() is None - assert is_jython() or isinstance(p, multiprocessing.Process), p - return p.is_alive() - - for p, args in _currentSubprocesses: - if is_alive(p): - try: - if get_os() == 'windows': - p.terminate() - else: - _kill_process_group(p.pid, signal.SIGKILL) - except BaseException as e: - if is_alive(p): - log('error while killing subprocess {0} "{1}": {2}'.format(p.pid, ' '.join(args), e)) - - if _opts and _opts.verbose: - import traceback - traceback.print_stack() - raise SystemExit(codeOrMessage) - -def download(path, urls, verbose=False): - """ - Attempts to downloads content for each URL in a list, stopping after the first successful download. - If the content cannot be retrieved from any URL, the program is aborted. The downloaded content - is written to the file indicated by 'path'. - """ - d = dirname(path) - if d != '' and not exists(d): - os.makedirs(d) - - assert not path.endswith(os.sep) - - _, binDir = _compile_mx_class('URLConnectionDownload') - - verbose = [] - if sys.stderr.isatty(): - verbose.append("-v") - if run([java().java, '-cp', _cygpathU2W(binDir), 'URLConnectionDownload', _cygpathU2W(path)] + verbose + urls, nonZeroIsFatal=False) == 0: - return - - abort('Could not download to ' + path + ' from any of the following URLs:\n\n ' + - '\n '.join(urls) + '\n\nPlease use a web browser to do the download manually') - -def update_file(path, content): - """ - Updates a file with some given content if the content differs from what's in - the file already. The return value indicates if the file was updated. - """ - existed = exists(path) - try: - old = None - if existed: - with open(path, 'rb') as f: - old = f.read() - - if old == content: - return False - - if existed and _opts.backup_modified: - shutil.move(path, path + '.orig') - - with open(path, 'wb') as f: - f.write(content) - - log(('modified ' if existed else 'created ') + path) - return True - except IOError as e: - abort('Error while writing to ' + path + ': ' + str(e)) - -# Builtin commands - -def _defaultEcjPath(): - return get_env('JDT', join(_primary_suite.mxDir, 'ecj.jar')) - -class JavaCompileTask: - def __init__(self, args, proj, reason, javafilelist, jdk, outputDir, jdtJar, deps): - self.proj = proj - self.reason = reason - self.javafilelist = javafilelist - self.deps = deps - self.jdk = jdk - self.outputDir = outputDir - self.done = False - self.jdtJar = jdtJar - self.args = args - - def __str__(self): - return self.proj.name - - def logCompilation(self, compiler): - log('Compiling Java sources for {0} with {1}... [{2}]'.format(self.proj.name, compiler, self.reason)) - - def execute(self): - argfileName = join(self.proj.dir, 'javafilelist.txt') - argfile = open(argfileName, 'wb') - argfile.write('\n'.join(map(_cygpathU2W, self.javafilelist))) - argfile.close() - - processorArgs = [] - processorPath = self.proj.annotation_processors_path() - if processorPath: - genDir = self.proj.source_gen_dir() - if exists(genDir): - shutil.rmtree(genDir) - os.mkdir(genDir) - processorArgs += ['-processorpath', _separatedCygpathU2W(join(processorPath)), '-s', _cygpathU2W(genDir)] - else: - processorArgs += ['-proc:none'] - - args = self.args - jdk = self.jdk - outputDir = _cygpathU2W(self.outputDir) - compliance = str(jdk.javaCompliance) - cp = _separatedCygpathU2W(classpath(self.proj.name, includeSelf=True)) - toBeDeleted = [argfileName] - - try: - if not self.jdtJar: - mainJava = java() - if not args.error_prone: - javac = args.alt_javac if args.alt_javac else mainJava.javac - self.logCompilation('javac' if not args.alt_javac else args.alt_javac) - javacCmd = [javac, '-g', '-J-Xmx1g', '-source', compliance, '-target', compliance, '-classpath', cp, '-d', outputDir] - jdk.javacLibOptions(javacCmd) - - attach = None - if _opts.attach is not None: - attach = 'server=n,address=' + _opts.attach - else: - if _opts.java_dbg_port is not None: - attach = 'server=y,address=' + str(_opts.java_dbg_port) - - if attach is not None: - javacCmd += ['-J-Xdebug', '-J-Xrunjdwp:transport=dt_socket,' + attach + ',suspend=y'] - javacCmd += processorArgs - javacCmd += ['@' + _cygpathU2W(argfile.name)] - - if not args.warnAPI: - javacCmd.append('-XDignore.symbol.file') - run(javacCmd) - else: - self.logCompilation('javac (with error-prone)') - javaArgs = ['-Xmx1g'] - javacArgs = ['-g', '-source', compliance, '-target', compliance, '-classpath', cp, '-d', outputDir] - jdk.javacLibOptions(javacCmd) - javacArgs += processorArgs - javacArgs += ['@' + argfile.name] - if not args.warnAPI: - javacArgs.append('-XDignore.symbol.file') - run_java(javaArgs + ['-cp', os.pathsep.join([mainJava.toolsjar, args.error_prone]), 'com.google.errorprone.ErrorProneCompiler'] + javacArgs) - else: - self.logCompilation('JDT') - - jdtVmArgs = ['-Xmx1g', '-jar', _cygpathU2W(self.jdtJar)] - - jdtArgs = ['-' + compliance, - '-cp', cp, '-g', '-enableJavadoc', - '-d', outputDir] - jdk.javacLibOptions(jdtArgs) - jdtArgs += processorArgs - - jdtProperties = join(self.proj.dir, '.settings', 'org.eclipse.jdt.core.prefs') - rootJdtProperties = join(self.proj.suite.mxDir, 'eclipse-settings', 'org.eclipse.jdt.core.prefs') - if not exists(jdtProperties) or os.path.getmtime(jdtProperties) < os.path.getmtime(rootJdtProperties): - # Try to fix a missing properties file by running eclipseinit - _eclipseinit_project(self.proj) - if not exists(jdtProperties): - log('JDT properties file {0} not found'.format(jdtProperties)) - else: - with open(jdtProperties) as fp: - origContent = fp.read() - content = origContent - if self.proj.uses_annotation_processor_library(): - # unfortunately, the command line compiler doesn't let us ignore warnings for generated files only - content = content.replace('=warning', '=ignore') - elif args.jdt_warning_as_error: - content = content.replace('=warning', '=error') - if not args.jdt_show_task_tags: - content = content + '\norg.eclipse.jdt.core.compiler.problem.tasks=ignore' - if origContent != content: - jdtPropertiesTmp = jdtProperties + '.tmp' - with open(jdtPropertiesTmp, 'w') as fp: - fp.write(content) - toBeDeleted.append(jdtPropertiesTmp) - jdtArgs += ['-properties', _cygpathU2W(jdtPropertiesTmp)] - else: - jdtArgs += ['-properties', _cygpathU2W(jdtProperties)] - jdtArgs.append('@' + _cygpathU2W(argfile.name)) - - run_java(jdtVmArgs + jdtArgs) - - # Create annotation processor jar for a project that defines annotation processors - if self.proj.definedAnnotationProcessorsDist: - self.proj.definedAnnotationProcessorsDist.make_archive() - - finally: - # Do not clean up temp files if verbose as there's - # a good chance the user wants to copy and paste the - # Java compiler command directly - if not _opts.verbose: - for n in toBeDeleted: - os.remove(n) - - self.done = True - -def build(args, parser=None): - """compile the Java and C sources, linking the latter - - Compile all the Java source code using the appropriate compilers - and linkers for the various source code types.""" - - suppliedParser = parser is not None - if not suppliedParser: - parser = ArgumentParser(prog='mx build') - - parser = parser if parser is not None else ArgumentParser(prog='mx build') - parser.add_argument('-f', action='store_true', dest='force', help='force build (disables timestamp checking)') - parser.add_argument('-c', action='store_true', dest='clean', help='removes existing build output') - parser.add_argument('-p', action='store_true', dest='parallelize', help='parallelizes Java compilation if possible') - parser.add_argument('--source', dest='compliance', help='Java compliance level for projects without an explicit one') - parser.add_argument('--Wapi', action='store_true', dest='warnAPI', help='show warnings about using internal APIs') - parser.add_argument('--check-distributions', action='store_true', dest='check_distributions', help='check built distributions for overlap') - parser.add_argument('--projects', action='store', help='comma separated projects to build (omit to build all projects)') - parser.add_argument('--only', action='store', help='comma separated projects to build, without checking their dependencies (omit to build all projects)') - parser.add_argument('--no-java', action='store_false', dest='java', help='do not build Java projects') - parser.add_argument('--no-native', action='store_false', dest='native', help='do not build native projects') - parser.add_argument('--jdt-warning-as-error', action='store_true', help='convert all Eclipse batch compiler warnings to errors') - parser.add_argument('--jdt-show-task-tags', action='store_true', help='show task tags as Eclipse batch compiler warnings') - parser.add_argument('--alt-javac', dest='alt_javac', help='path to alternative javac executable', metavar='<path>') - compilerSelect = parser.add_mutually_exclusive_group() - compilerSelect.add_argument('--error-prone', dest='error_prone', help='path to error-prone.jar', metavar='<path>') - compilerSelect.add_argument('--jdt', help='path to ecj.jar, the Eclipse batch compiler', default=_defaultEcjPath(), metavar='<path>') - compilerSelect.add_argument('--force-javac', action='store_true', dest='javac', help='use javac whether ecj.jar is found or not') - - if suppliedParser: - parser.add_argument('remainder', nargs=REMAINDER, metavar='...') - - args = parser.parse_args(args) - - if is_jython(): - if args.parallelize: - logv('[multiprocessing not available in jython]') - args.parallelize = False - - jdtJar = None - if not args.javac and args.jdt is not None: - if not args.jdt.endswith('.jar'): - abort('Path for Eclipse batch compiler does not look like a jar file: ' + args.jdt) - jdtJar = args.jdt - if not exists(jdtJar): - if os.path.abspath(jdtJar) == os.path.abspath(_defaultEcjPath()) and get_env('JDT', None) is None: - # Silently ignore JDT if default location is used but does not exist - jdtJar = None - else: - abort('Eclipse batch compiler jar does not exist: ' + args.jdt) - - if args.only is not None: - # N.B. This build will not include dependencies including annotation processor dependencies - sortedProjects = [project(name) for name in args.only.split(',')] - else: - if args.projects is not None: - projectNames = args.projects.split(',') - else: - projectNames = None - - projects = _projects_opt_limit_to_suites(projects_from_names(projectNames)) - # N.B. Limiting to a suite only affects the starting set of projects. Dependencies in other suites will still be compiled - sortedProjects = sorted_project_deps(projects, includeAnnotationProcessors=True) - - if args.java and jdtJar: - ideinit([], refreshOnly=True, buildProcessorJars=False) - - tasks = {} - updatedAnnotationProcessorDists = set() - for p in sortedProjects: - if p.native: - if args.native: - log('Calling GNU make {0}...'.format(p.dir)) - - if args.clean: - run([gmake_cmd(), 'clean'], cwd=p.dir) - - run([gmake_cmd()], cwd=p.dir) - continue - else: - if not args.java: - continue - if exists(join(p.dir, 'plugin.xml')): # eclipse plugin project - continue - - # skip building this Java project if its Java compliance level is "higher" than the configured JDK - requiredCompliance = p.javaCompliance if p.javaCompliance else JavaCompliance(args.compliance) if args.compliance else None - jdk = java(requiredCompliance) - - outputDir = p.output_dir() - - sourceDirs = p.source_dirs() - buildReason = None - if args.force: - buildReason = 'forced build' - elif args.clean: - buildReason = 'clean' - - taskDeps = [] - for dep in p.all_deps([], includeLibs=False, includeAnnotationProcessors=True): - taskDep = tasks.get(dep.name) - if taskDep: - if not buildReason: - buildReason = dep.name + ' rebuilt' - taskDeps.append(taskDep) - - javafilelist = [] - nonjavafiletuples = [] - for sourceDir in sourceDirs: - for root, _, files in os.walk(sourceDir): - javafiles = [join(root, name) for name in files if name.endswith('.java')] - javafilelist += javafiles - - nonjavafiletuples += [(sourceDir, [join(root, name) for name in files if not name.endswith('.java')])] - - if not buildReason: - for javafile in javafiles: - classfile = TimeStampFile(outputDir + javafile[len(sourceDir):-len('java')] + 'class') - if not classfile.exists() or classfile.isOlderThan(javafile): - if basename(classfile.path) != 'package-info.class': - buildReason = 'class file(s) out of date' - break - - apsOutOfDate = p.update_current_annotation_processors_file() - if apsOutOfDate: - buildReason = 'annotation processor(s) changed' - - if not buildReason: - logv('[all class files for {0} are up to date - skipping]'.format(p.name)) - _handleNonJavaFiles(outputDir, p, False, nonjavafiletuples) - continue - - _handleNonJavaFiles(outputDir, p, True, nonjavafiletuples) - - if len(javafilelist) == 0: - logv('[no Java sources for {0} - skipping]'.format(p.name)) - continue - - javafilelist = sorted(javafilelist) - - task = JavaCompileTask(args, p, buildReason, javafilelist, jdk, outputDir, jdtJar, taskDeps) - if p.definedAnnotationProcessorsDist: - updatedAnnotationProcessorDists.add(p.definedAnnotationProcessorsDist) - - tasks[p.name] = task - if args.parallelize: - # Best to initialize class paths on main process - jdk.bootclasspath() - task.proc = None - else: - task.execute() - - if args.parallelize: - - def joinTasks(tasks): - failed = [] - for t in tasks: - t.proc.join() - _removeSubprocess(t.sub) - if t.proc.exitcode != 0: - failed.append(t) - return failed - - def checkTasks(tasks): - active = [] - for t in tasks: - if t.proc.is_alive(): - active.append(t) - else: - if t.proc.exitcode != 0: - return ([], joinTasks(tasks)) - return (active, []) - - def remainingDepsDepth(task): - if task._d is None: - incompleteDeps = [d for d in task.deps if d.proc is None or d.proc.is_alive()] - if len(incompleteDeps) == 0: - task._d = 0 - else: - task._d = max([remainingDepsDepth(t) for t in incompleteDeps]) + 1 - return task._d - - def compareTasks(t1, t2): - d = remainingDepsDepth(t1) - remainingDepsDepth(t2) - if d == 0: - t1Work = (1 + len(t1.proj.annotation_processors())) * len(t1.javafilelist) - t2Work = (1 + len(t2.proj.annotation_processors())) * len(t2.javafilelist) - d = t1Work - t2Work - return d - - def sortWorklist(tasks): - for t in tasks: - t._d = None - return sorted(tasks, compareTasks) - - cpus = cpu_count() - worklist = sortWorklist(tasks.values()) - active = [] - failed = [] - while len(worklist) != 0: - while True: - active, failed = checkTasks(active) - if len(failed) != 0: - assert not active, active - break - if len(active) == cpus: - # Sleep for 1 second - time.sleep(1) - else: - break - - if len(failed) != 0: - break - - def executeTask(task): - # Clear sub-process list cloned from parent process - del _currentSubprocesses[:] - task.execute() - - def depsDone(task): - for d in task.deps: - if d.proc is None or d.proc.exitcode is None: - return False - return True - - for task in worklist: - if depsDone(task): - worklist.remove(task) - task.proc = multiprocessing.Process(target=executeTask, args=(task,)) - task.proc.start() - active.append(task) - task.sub = _addSubprocess(task.proc, ['JavaCompileTask', str(task)]) - if len(active) == cpus: - break - - worklist = sortWorklist(worklist) - - failed += joinTasks(active) - if len(failed): - for t in failed: - log('Compiling {0} failed'.format(t.proj.name)) - abort('{0} Java compilation tasks failed'.format(len(failed))) - - if args.java and not args.only: - files = [] - for dist in sorted_dists(): - if dist not in updatedAnnotationProcessorDists: - archive(['@' + dist.name]) - if args.check_distributions and not dist.isProcessorDistribution: - with zipfile.ZipFile(dist.path, 'r') as zf: - files.extend([member for member in zf.namelist() if not member.startswith('META-INF')]) - dups = set([x for x in files if files.count(x) > 1]) - if len(dups) > 0: - abort('Distributions overlap! duplicates: ' + str(dups)) - - if suppliedParser: - return args - return None - -def _handleNonJavaFiles(outputDir, p, clean, nonjavafiletuples): - if exists(outputDir): - if clean: - log('Cleaning {0}...'.format(outputDir)) - shutil.rmtree(outputDir) - os.mkdir(outputDir) - else: - os.mkdir(outputDir) - genDir = p.source_gen_dir() - if genDir != '' and exists(genDir) and clean: - log('Cleaning {0}...'.format(genDir)) - for f in os.listdir(genDir): - shutil.rmtree(join(genDir, f)) - - # Copy all non Java resources or assemble Jasmin files - jasminAvailable = None - for nonjavafiletuple in nonjavafiletuples: - sourceDir = nonjavafiletuple[0] - nonjavafilelist = nonjavafiletuple[1] - - for src in nonjavafilelist: - if src.endswith('.jasm'): - className = None - with open(src) as f: - for line in f: - if line.startswith('.class '): - className = line.split()[-1] - break - - if className is not None: - jasminOutputDir = p.jasmin_output_dir() - classFile = join(jasminOutputDir, className.replace('/', os.sep) + '.class') - if exists(dirname(classFile)) and (not exists(classFile) or os.path.getmtime(classFile) < os.path.getmtime(src)): - if jasminAvailable is None: - try: - with open(os.devnull) as devnull: - subprocess.call('jasmin', stdout=devnull, stderr=subprocess.STDOUT) - jasminAvailable = True - except OSError: - jasminAvailable = False - - if jasminAvailable: - log('Assembling Jasmin file ' + src) - run(['jasmin', '-d', jasminOutputDir, src]) - else: - log('The jasmin executable could not be found - skipping ' + src) - with file(classFile, 'a'): - os.utime(classFile, None) - - else: - log('could not file .class directive in Jasmin source: ' + src) - else: - dst = join(outputDir, src[len(sourceDir) + 1:]) - if not exists(dirname(dst)): - os.makedirs(dirname(dst)) - if exists(dirname(dst)) and (not exists(dst) or os.path.getmtime(dst) < os.path.getmtime(src)): - shutil.copyfile(src, dst) - -def _chunk_files_for_command_line(files, limit=None, pathFunction=lambda f: f): - """ - Returns a generator for splitting up a list of files into chunks such that the - size of the space separated file paths in a chunk is less than a given limit. - This is used to work around system command line length limits. - """ - chunkSize = 0 - chunkStart = 0 - if limit is None: - commandLinePrefixAllowance = 3000 - if get_os() == 'windows': - # The CreateProcess function on Windows limits the length of a command line to - # 32,768 characters (http://msdn.microsoft.com/en-us/library/ms682425%28VS.85%29.aspx) - limit = 32768 - commandLinePrefixAllowance - else: - # Using just SC_ARG_MAX without extra downwards adjustment - # results in "[Errno 7] Argument list too long" on MacOS. - commandLinePrefixAllowance = 20000 - syslimit = os.sysconf('SC_ARG_MAX') - if syslimit == -1: - syslimit = 262144 # we could use sys.maxint but we prefer a more robust smaller value - limit = syslimit - commandLinePrefixAllowance - assert limit > 0 - for i in range(len(files)): - path = pathFunction(files[i]) - size = len(path) + 1 - assert size < limit - if chunkSize + size < limit: - chunkSize += size - else: - assert i > chunkStart - yield files[chunkStart:i] - chunkStart = i - chunkSize = 0 - if chunkStart == 0: - assert chunkSize < limit - yield files - -def eclipseformat(args): - """run the Eclipse Code Formatter on the Java sources - - The exit code 1 denotes that at least one file was modified.""" - - parser = ArgumentParser(prog='mx eclipseformat') - parser.add_argument('-e', '--eclipse-exe', help='location of the Eclipse executable') - parser.add_argument('-C', '--no-backup', action='store_false', dest='backup', help='do not save backup of modified files') - parser.add_argument('--projects', action='store', help='comma separated projects to process (omit to process all projects)') - - args = parser.parse_args(args) - if args.eclipse_exe is None: - args.eclipse_exe = os.environ.get('ECLIPSE_EXE') - if args.eclipse_exe is None: - abort('Could not find Eclipse executable. Use -e option or ensure ECLIPSE_EXE environment variable is set.') - - # Maybe an Eclipse installation dir was specified - look for the executable in it - if isdir(args.eclipse_exe): - args.eclipse_exe = join(args.eclipse_exe, exe_suffix('eclipse')) - warn("The eclipse-exe was a directory, now using " + args.eclipse_exe) - - if not os.path.isfile(args.eclipse_exe): - abort('File does not exist: ' + args.eclipse_exe) - if not os.access(args.eclipse_exe, os.X_OK): - abort('Not an executable file: ' + args.eclipse_exe) - - eclipseinit([], buildProcessorJars=False) - - # build list of projects to be processed - projects = sorted_deps() - if args.projects is not None: - projects = [project(name) for name in args.projects.split(',')] - - class Batch: - def __init__(self, settingsDir, javaCompliance): - self.path = join(settingsDir, 'org.eclipse.jdt.core.prefs') - self.javaCompliance = javaCompliance - with open(join(settingsDir, 'org.eclipse.jdt.ui.prefs')) as fp: - jdtUiPrefs = fp.read() - self.removeTrailingWhitespace = 'sp_cleanup.remove_trailing_whitespaces_all=true' in jdtUiPrefs - if self.removeTrailingWhitespace: - assert 'sp_cleanup.remove_trailing_whitespaces=true' in jdtUiPrefs and 'sp_cleanup.remove_trailing_whitespaces_ignore_empty=false' in jdtUiPrefs - self.cachedHash = None - - def __hash__(self): - if not self.cachedHash: - with open(self.path) as fp: - self.cachedHash = (fp.read(), self.javaCompliance, self.removeTrailingWhitespace).__hash__() - return self.cachedHash - - def __eq__(self, other): - if not isinstance(other, Batch): - return False - if self.removeTrailingWhitespace != other.removeTrailingWhitespace: - return False - if self.javaCompliance != other.javaCompliance: - return False - if self.path == other.path: - return True - with open(self.path) as fp: - with open(other.path) as ofp: - if fp.read() != ofp.read(): - return False - return True - - class FileInfo: - def __init__(self, path): - self.path = path - with open(path) as fp: - self.content = fp.read() - self.times = (os.path.getatime(path), os.path.getmtime(path)) - - def update(self, removeTrailingWhitespace): - with open(self.path) as fp: - content = fp.read() - - if self.content != content: - # Only apply *after* formatting to match the order in which the IDE does it - if removeTrailingWhitespace: - content, n = re.subn(r'[ \t]+$', '', content, flags=re.MULTILINE) - if n != 0 and self.content == content: - # undo on-disk changes made by the Eclipse formatter - with open(self.path, 'w') as fp: - fp.write(content) - - if self.content != content: - self.diff = difflib.unified_diff(self.content.splitlines(1), content.splitlines(1)) - self.content = content - return True - - # reset access and modification time of file - os.utime(self.path, self.times) - - modified = list() - batches = dict() # all sources with the same formatting settings are formatted together - for p in projects: - if p.native: - continue - sourceDirs = p.source_dirs() - - batch = Batch(join(p.dir, '.settings'), p.javaCompliance) - - if not exists(batch.path): - if _opts.verbose: - log('[no Eclipse Code Formatter preferences at {0} - skipping]'.format(batch.path)) - continue - javafiles = [] - for sourceDir in sourceDirs: - for root, _, files in os.walk(sourceDir): - for f in [join(root, name) for name in files if name.endswith('.java')]: - javafiles.append(FileInfo(f)) - if len(javafiles) == 0: - logv('[no Java sources in {0} - skipping]'.format(p.name)) - continue - - res = batches.setdefault(batch, javafiles) - if res is not javafiles: - res.extend(javafiles) - - log("we have: " + str(len(batches)) + " batches") - for batch, javafiles in batches.iteritems(): - for chunk in _chunk_files_for_command_line(javafiles, pathFunction=lambda f: f.path): - run([args.eclipse_exe, - '-nosplash', - '-application', - 'org.eclipse.jdt.core.JavaCodeFormatter', - '-vm', java(batch.javaCompliance).java, - '-config', batch.path] - + [f.path for f in chunk]) - for fi in chunk: - if fi.update(batch.removeTrailingWhitespace): - modified.append(fi) - - log('{0} files were modified'.format(len(modified))) - - if len(modified) != 0: - arcbase = _primary_suite.dir - if args.backup: - backup = os.path.abspath('eclipseformat.backup.zip') - zf = zipfile.ZipFile(backup, 'w', zipfile.ZIP_DEFLATED) - for fi in modified: - name = os.path.relpath(fi.path, arcbase) - log(' - {0}'.format(name)) - log('Changes:') - log(''.join(fi.diff)) - if args.backup: - arcname = name.replace(os.sep, '/') - zf.writestr(arcname, fi.content) - if args.backup: - zf.close() - log('Wrote backup of {0} modified files to {1}'.format(len(modified), backup)) - return 1 - return 0 - -def processorjars(): - for s in suites(True): - _processorjars_suite(s) - -def _processorjars_suite(s): - projs = [p for p in s.projects if p.definedAnnotationProcessors is not None] - if len(projs) <= 0: - return [] - - pnames = [p.name for p in projs] - build(['--jdt-warning-as-error', '--projects', ",".join(pnames)]) - return [p.definedAnnotationProcessorsDist.path for p in s.projects if p.definedAnnotationProcessorsDist is not None] - -def pylint(args): - """run pylint (if available) over Python source files (found by 'hg locate' or by tree walk with -walk)""" - - parser = ArgumentParser(prog='mx pylint') - parser.add_argument('--walk', action='store_true', help='use tree walk find .py files') - args = parser.parse_args(args) - - rcfile = join(dirname(__file__), '.pylintrc') - if not exists(rcfile): - log('pylint configuration file does not exist: ' + rcfile) - return - - try: - output = subprocess.check_output(['pylint', '--version'], stderr=subprocess.STDOUT) - m = re.match(r'.*pylint (\d+)\.(\d+)\.(\d+).*', output, re.DOTALL) - if not m: - log('could not determine pylint version from ' + output) - return - major, minor, micro = (int(m.group(1)), int(m.group(2)), int(m.group(3))) - if major != 1 or minor != 1: - log('require pylint version = 1.1.x (got {0}.{1}.{2})'.format(major, minor, micro)) - return - except BaseException: - log('pylint is not available') - return - - def findfiles_by_walk(): - result = [] - for suite in suites(True): - for root, dirs, files in os.walk(suite.dir): - for f in files: - if f.endswith('.py'): - pyfile = join(root, f) - result.append(pyfile) - if 'bin' in dirs: - dirs.remove('bin') - if 'lib' in dirs: - # avoids downloaded .py files - dirs.remove('lib') - return result - - def findfiles_by_hg(): - result = [] - for suite in suites(True): - versioned = subprocess.check_output(['hg', 'locate', '-f'], stderr=subprocess.STDOUT, cwd=suite.dir).split(os.linesep) - for f in versioned: - if f.endswith('.py') and exists(f): - result.append(f) - return result - - # Perhaps we should just look in suite.mxDir directories for .py files? - if args.walk: - pyfiles = findfiles_by_walk() - else: - pyfiles = findfiles_by_hg() - - env = os.environ.copy() - - pythonpath = dirname(__file__) - for suite in suites(True): - pythonpath = os.pathsep.join([pythonpath, suite.mxDir]) - - env['PYTHONPATH'] = pythonpath - - for pyfile in pyfiles: - log('Running pylint on ' + pyfile + '...') - run(['pylint', '--reports=n', '--rcfile=' + rcfile, pyfile], env=env) - -""" -Utility for creating and updating a zip file atomically. -""" -class Archiver: - def __init__(self, path): - self.path = path - - def __enter__(self): - if self.path: - if not isdir(dirname(self.path)): - os.makedirs(dirname(self.path)) - fd, tmp = tempfile.mkstemp(suffix='', prefix=basename(self.path) + '.', dir=dirname(self.path)) - self.tmpFd = fd - self.tmpPath = tmp - self.zf = zipfile.ZipFile(tmp, 'w') - else: - self.tmpFd = None - self.tmpPath = None - self.zf = None - return self - - def __exit__(self, exc_type, exc_value, traceback): - if self.zf: - self.zf.close() - os.close(self.tmpFd) - # Correct the permissions on the temporary file which is created with restrictive permissions - os.chmod(self.tmpPath, 0o666 & ~currentUmask) - # Atomic on Unix - shutil.move(self.tmpPath, self.path) - -def _archive(args): - archive(args) - return 0 - -def archive(args): - """create jar files for projects and distributions""" - parser = ArgumentParser(prog='mx archive') - parser.add_argument('names', nargs=REMAINDER, metavar='[<project>|@<distribution>]...') - args = parser.parse_args(args) - - archives = [] - for name in args.names: - if name.startswith('@'): - dname = name[1:] - d = distribution(dname) - d.make_archive() - archives.append(d.path) - else: - p = project(name) - archives.append(p.make_archive()) - - logv("generated archives: " + str(archives)) - return archives - -def canonicalizeprojects(args): - """check all project specifications for canonical dependencies - - The exit code of this command reflects how many projects have non-canonical dependencies.""" - - nonCanonical = [] - for s in suites(True): - for p in s.projects: - if p.checkPackagePrefix: - for pkg in p.defined_java_packages(): - if not pkg.startswith(p.name): - abort('package in {0} does not have prefix matching project name: {1}'.format(p, pkg)) - - ignoredDeps = set([name for name in p.deps if project(name, False) is not None]) - for pkg in p.imported_java_packages(): - for name in p.deps: - dep = project(name, False) - if dep is None: - ignoredDeps.discard(name) - else: - if pkg in dep.defined_java_packages(): - ignoredDeps.discard(name) - if pkg in dep.extended_java_packages(): - ignoredDeps.discard(name) - if len(ignoredDeps) != 0: - candidates = set() - # Compute dependencies based on projects required by p - for d in sorted_deps(): - if not d.defined_java_packages().isdisjoint(p.imported_java_packages()): - candidates.add(d) - # Remove non-canonical candidates - for c in list(candidates): - candidates.difference_update(c.all_deps([], False, False)) - candidates = [d.name for d in candidates] - - abort('{0} does not use any packages defined in these projects: {1}\nComputed project dependencies: {2}'.format( - p, ', '.join(ignoredDeps), ','.join(candidates))) - - excess = frozenset(p.deps) - set(p.canonical_deps()) - if len(excess) != 0: - nonCanonical.append(p) - if len(nonCanonical) != 0: - for p in nonCanonical: - canonicalDeps = p.canonical_deps() - if len(canonicalDeps) != 0: - log('Canonical dependencies for project ' + p.name + ' are: [') - for d in canonicalDeps: - log(' "' + d + '",') - log(' ],') - else: - log('Canonical dependencies for project ' + p.name + ' are: []') - return len(nonCanonical) - -class TimeStampFile: - def __init__(self, path): - self.path = path - self.timestamp = os.path.getmtime(path) if exists(path) else None - - def isOlderThan(self, arg): - if not self.timestamp: - return True - if isinstance(arg, TimeStampFile): - if arg.timestamp is None: - return False - else: - return arg.timestamp > self.timestamp - elif isinstance(arg, types.ListType): - files = arg - else: - files = [arg] - for f in files: - if os.path.getmtime(f) > self.timestamp: - return True - return False - - def exists(self): - return exists(self.path) - - def touch(self): - if exists(self.path): - os.utime(self.path, None) - else: - if not isdir(dirname(self.path)): - os.makedirs(dirname(self.path)) - file(self.path, 'a') - -def checkstyle(args): - """run Checkstyle on the Java sources - - Run Checkstyle over the Java sources. Any errors or warnings - produced by Checkstyle result in a non-zero exit code.""" - - parser = ArgumentParser(prog='mx checkstyle') - - parser.add_argument('-f', action='store_true', dest='force', help='force checking (disables timestamp checking)') - args = parser.parse_args(args) - - totalErrors = 0 - for p in projects_opt_limit_to_suites(): - if p.native: - continue - sourceDirs = p.source_dirs() - - config = join(project(p.checkstyleProj).dir, '.checkstyle_checks.xml') - if not exists(config): - logv('[No Checkstyle configuration found for {0} - skipping]'.format(p)) - continue - - # skip checking this Java project if its Java compliance level is "higher" than the configured JDK - jdk = java(p.javaCompliance) - assert jdk - - for sourceDir in sourceDirs: - javafilelist = [] - for root, _, files in os.walk(sourceDir): - javafilelist += [join(root, name) for name in files if name.endswith('.java') and name != 'package-info.java'] - if len(javafilelist) == 0: - logv('[no Java sources in {0} - skipping]'.format(sourceDir)) - continue - - timestamp = TimeStampFile(join(p.suite.mxDir, 'checkstyle-timestamps', sourceDir[len(p.suite.dir) + 1:].replace(os.sep, '_') + '.timestamp')) - mustCheck = False - if not args.force and timestamp.exists(): - mustCheck = timestamp.isOlderThan(javafilelist) - else: - mustCheck = True - - if not mustCheck: - if _opts.verbose: - log('[all Java sources in {0} already checked - skipping]'.format(sourceDir)) - continue - - exclude = join(p.dir, '.checkstyle.exclude') - if exists(exclude): - with open(exclude) as f: - # Convert patterns to OS separators - patterns = [name.rstrip().replace('/', os.sep) for name in f.readlines()] - def match(name): - for p in patterns: - if p in name: - if _opts.verbose: - log('excluding: ' + name) - return True - return False - - javafilelist = [name for name in javafilelist if not match(name)] - - auditfileName = join(p.dir, 'checkstyleOutput.txt') - log('Running Checkstyle on {0} using {1}...'.format(sourceDir, config)) - - try: - for chunk in _chunk_files_for_command_line(javafilelist): - try: - run_java(['-Xmx1g', '-jar', library('CHECKSTYLE').get_path(True), '-f', 'xml', '-c', config, '-o', auditfileName] + chunk, nonZeroIsFatal=False) - finally: - if exists(auditfileName): - errors = [] - source = [None] - def start_element(name, attrs): - if name == 'file': - source[0] = attrs['name'] - elif name == 'error': - errors.append('{0}:{1}: {2}'.format(source[0], attrs['line'], attrs['message'])) - - xp = xml.parsers.expat.ParserCreate() - xp.StartElementHandler = start_element - with open(auditfileName) as fp: - xp.ParseFile(fp) - if len(errors) != 0: - map(log, errors) - totalErrors = totalErrors + len(errors) - else: - timestamp.touch() - finally: - if exists(auditfileName): - os.unlink(auditfileName) - return totalErrors - -def clean(args, parser=None): - """remove all class files, images, and executables - - Removes all files created by a build, including Java class files, executables, and - generated images. - """ - - suppliedParser = parser is not None - - parser = parser if suppliedParser else ArgumentParser(prog='mx clean') - parser.add_argument('--no-native', action='store_false', dest='native', help='do not clean native projects') - parser.add_argument('--no-java', action='store_false', dest='java', help='do not clean Java projects') - parser.add_argument('--no-dist', action='store_false', dest='dist', help='do not delete distributions') - - args = parser.parse_args(args) - - def _rmtree(dirPath): - path = dirPath - if get_os() == 'windows': - path = unicode("\\\\?\\" + dirPath) - shutil.rmtree(path) - - def _rmIfExists(name): - if name and os.path.isfile(name): - os.unlink(name) - - for p in projects_opt_limit_to_suites(): - if p.native: - if args.native: - run([gmake_cmd(), '-C', p.dir, 'clean']) - else: - if args.java: - genDir = p.source_gen_dir() - if genDir != '' and exists(genDir): - log('Clearing {0}...'.format(genDir)) - for f in os.listdir(genDir): - _rmtree(join(genDir, f)) - - - outputDir = p.output_dir() - if outputDir != '' and exists(outputDir): - log('Removing {0}...'.format(outputDir)) - _rmtree(outputDir) - - for configName in ['netbeans-config.zip', 'eclipse-config.zip']: - config = TimeStampFile(join(p.suite.mxDir, configName)) - if config.exists(): - os.unlink(config.path) - - if args.java: - if args.dist: - for d in _dists.keys(): - log('Removing distribution {0}...'.format(d)) - _rmIfExists(distribution(d).path) - _rmIfExists(distribution(d).sourcesPath) - - if suppliedParser: - return args - -def about(args): - """show the 'man page' for mx""" - print __doc__ - -def help_(args): - """show help for a given command - -With no arguments, print a list of commands and short help for each command. - -Given a command name, print help for that command.""" - if len(args) == 0: - _argParser.print_help() - return - - name = args[0] - if not _commands.has_key(name): - hits = [c for c in _commands.iterkeys() if c.startswith(name)] - if len(hits) == 1: - name = hits[0] - elif len(hits) == 0: - abort('mx: unknown command \'{0}\'\n{1}use "mx help" for more options'.format(name, _format_commands())) - else: - abort('mx: command \'{0}\' is ambiguous\n {1}'.format(name, ' '.join(hits))) - - value = _commands[name] - (func, usage) = value[:2] - doc = func.__doc__ - if len(value) > 2: - docArgs = value[2:] - fmtArgs = [] - for d in docArgs: - if isinstance(d, Callable): - fmtArgs += [d()] - else: - fmtArgs += [str(d)] - doc = doc.format(*fmtArgs) - print 'mx {0} {1}\n\n{2}\n'.format(name, usage, doc) - -def projectgraph(args, suite=None): - """create graph for project structure ("mx projectgraph | dot -Tpdf -oprojects.pdf" or "mx projectgraph --igv")""" - - parser = ArgumentParser(prog='mx projectgraph') - parser.add_argument('--igv', action='store_true', help='output to IGV listening on 127.0.0.1:4444') - parser.add_argument('--igv-format', action='store_true', help='output graph in IGV format') - - args = parser.parse_args(args) - - if args.igv or args.igv_format: - ids = {} - nextToIndex = {} - igv = XMLDoc() - igv.open('graphDocument') - igv.open('group') - igv.open('properties') - igv.element('p', {'name' : 'name'}, 'GraalProjectDependencies') - igv.close('properties') - igv.open('graph', {'name' : 'dependencies'}) - igv.open('nodes') - for p in sorted_deps(includeLibs=True, includeJreLibs=True): - ident = len(ids) - ids[p.name] = str(ident) - igv.open('node', {'id' : str(ident)}) - igv.open('properties') - igv.element('p', {'name' : 'name'}, p.name) - igv.close('properties') - igv.close('node') - igv.close('nodes') - igv.open('edges') - for p in projects(): - fromIndex = 0 - for dep in p.canonical_deps(): - toIndex = nextToIndex.get(dep, 0) - nextToIndex[dep] = toIndex + 1 - igv.element('edge', {'from' : ids[p.name], 'fromIndex' : str(fromIndex), 'to' : ids[dep], 'toIndex' : str(toIndex), 'label' : 'dependsOn'}) - fromIndex = fromIndex + 1 - igv.close('edges') - igv.close('graph') - igv.close('group') - igv.close('graphDocument') - - if args.igv: - s = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - s.connect(('127.0.0.1', 4444)) - s.send(igv.xml()) - else: - print igv.xml(indent=' ', newl='\n') - return - - print 'digraph projects {' - print 'rankdir=BT;' - print 'node [shape=rect];' - for p in projects(): - for dep in p.canonical_deps(): - print '"' + p.name + '"->"' + dep + '";' - if hasattr(p, '_declaredAnnotationProcessors'): - for ap in p._declaredAnnotationProcessors: - print '"' + p.name + '"->"' + ap + '" [style="dashed"];' - print '}' - -def _source_locator_memento(deps): - slm = XMLDoc() - slm.open('sourceLookupDirector') - slm.open('sourceContainers', {'duplicates' : 'false'}) - - javaCompliance = None - for dep in deps: - if dep.isLibrary(): - if hasattr(dep, 'eclipse.container'): - memento = XMLDoc().element('classpathContainer', {'path' : getattr(dep, 'eclipse.container')}).xml(standalone='no') - slm.element('classpathContainer', {'memento' : memento, 'typeId':'org.eclipse.jdt.launching.sourceContainer.classpathContainer'}) - elif dep.get_source_path(resolve=True): - memento = XMLDoc().element('archive', {'detectRoot' : 'true', 'path' : dep.get_source_path(resolve=True)}).xml(standalone='no') - slm.element('container', {'memento' : memento, 'typeId':'org.eclipse.debug.core.containerType.externalArchive'}) - elif dep.isProject(): - memento = XMLDoc().element('javaProject', {'name' : dep.name}).xml(standalone='no') - slm.element('container', {'memento' : memento, 'typeId':'org.eclipse.jdt.launching.sourceContainer.javaProject'}) - if javaCompliance is None or dep.javaCompliance > javaCompliance: - javaCompliance = dep.javaCompliance - - if javaCompliance: - memento = XMLDoc().element('classpathContainer', {'path' : 'org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-' + str(javaCompliance)}).xml(standalone='no') - slm.element('classpathContainer', {'memento' : memento, 'typeId':'org.eclipse.jdt.launching.sourceContainer.classpathContainer'}) - else: - memento = XMLDoc().element('classpathContainer', {'path' : 'org.eclipse.jdt.launching.JRE_CONTAINER'}).xml(standalone='no') - slm.element('classpathContainer', {'memento' : memento, 'typeId':'org.eclipse.jdt.launching.sourceContainer.classpathContainer'}) - - slm.close('sourceContainers') - slm.close('sourceLookupDirector') - return slm - -def make_eclipse_attach(suite, hostname, port, name=None, deps=None): - """ - Creates an Eclipse launch configuration file for attaching to a Java process. - """ - if deps is None: - deps = [] - slm = _source_locator_memento(deps) - launch = XMLDoc() - launch.open('launchConfiguration', {'type' : 'org.eclipse.jdt.launching.remoteJavaApplication'}) - launch.element('stringAttribute', {'key' : 'org.eclipse.debug.core.source_locator_id', 'value' : 'org.eclipse.jdt.launching.sourceLocator.JavaSourceLookupDirector'}) - launch.element('stringAttribute', {'key' : 'org.eclipse.debug.core.source_locator_memento', 'value' : '%s'}) - launch.element('booleanAttribute', {'key' : 'org.eclipse.jdt.launching.ALLOW_TERMINATE', 'value' : 'true'}) - launch.open('mapAttribute', {'key' : 'org.eclipse.jdt.launching.CONNECT_MAP'}) - launch.element('mapEntry', {'key' : 'hostname', 'value' : hostname}) - launch.element('mapEntry', {'key' : 'port', 'value' : port}) - launch.close('mapAttribute') - launch.element('stringAttribute', {'key' : 'org.eclipse.jdt.launching.PROJECT_ATTR', 'value' : ''}) - launch.element('stringAttribute', {'key' : 'org.eclipse.jdt.launching.VM_CONNECTOR_ID', 'value' : 'org.eclipse.jdt.launching.socketAttachConnector'}) - launch.close('launchConfiguration') - launch = launch.xml(newl='\n', standalone='no') % slm.xml(escape=True, standalone='no') - - if name is None: - if len(suites()) == 1: - suitePrefix = '' - else: - suitePrefix = suite.name + '-' - name = suitePrefix + 'attach-' + hostname + '-' + port - eclipseLaunches = join(suite.mxDir, 'eclipse-launches') - if not exists(eclipseLaunches): - os.makedirs(eclipseLaunches) - launchFile = join(eclipseLaunches, name + '.launch') - return update_file(launchFile, launch), launchFile - -def make_eclipse_launch(javaArgs, jre, name=None, deps=None): - """ - Creates an Eclipse launch configuration file for running/debugging a Java command. - """ - if deps is None: - deps = [] - mainClass = None - vmArgs = [] - appArgs = [] - cp = None - argsCopy = list(reversed(javaArgs)) - while len(argsCopy) != 0: - a = argsCopy.pop() - if a == '-jar': - mainClass = '-jar' - appArgs = list(reversed(argsCopy)) - break - if a == '-cp' or a == '-classpath': - assert len(argsCopy) != 0 - cp = argsCopy.pop() - vmArgs.append(a) - vmArgs.append(cp) - elif a.startswith('-'): - vmArgs.append(a) - else: - mainClass = a - appArgs = list(reversed(argsCopy)) - break - - if mainClass is None: - log('Cannot create Eclipse launch configuration without main class or jar file: java ' + ' '.join(javaArgs)) - return False - - if name is None: - if mainClass == '-jar': - name = basename(appArgs[0]) - if len(appArgs) > 1 and not appArgs[1].startswith('-'): - name = name + '_' + appArgs[1] - else: - name = mainClass - name = time.strftime('%Y-%m-%d-%H%M%S_' + name) - - if cp is not None: - for e in cp.split(os.pathsep): - for s in suites(): - deps += [p for p in s.projects if e == p.output_dir()] - deps += [l for l in s.libs if e == l.get_path(False)] - - slm = _source_locator_memento(deps) - - launch = XMLDoc() - launch.open('launchConfiguration', {'type' : 'org.eclipse.jdt.launching.localJavaApplication'}) - launch.element('stringAttribute', {'key' : 'org.eclipse.debug.core.source_locator_id', 'value' : 'org.eclipse.jdt.launching.sourceLocator.JavaSourceLookupDirector'}) - launch.element('stringAttribute', {'key' : 'org.eclipse.debug.core.source_locator_memento', 'value' : '%s'}) - launch.element('stringAttribute', {'key' : 'org.eclipse.jdt.launching.JRE_CONTAINER', 'value' : 'org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/' + jre}) - launch.element('stringAttribute', {'key' : 'org.eclipse.jdt.launching.MAIN_TYPE', 'value' : mainClass}) - launch.element('stringAttribute', {'key' : 'org.eclipse.jdt.launching.PROGRAM_ARGUMENTS', 'value' : ' '.join(appArgs)}) - launch.element('stringAttribute', {'key' : 'org.eclipse.jdt.launching.PROJECT_ATTR', 'value' : ''}) - launch.element('stringAttribute', {'key' : 'org.eclipse.jdt.launching.VM_ARGUMENTS', 'value' : ' '.join(vmArgs)}) - launch.close('launchConfiguration') - launch = launch.xml(newl='\n', standalone='no') % slm.xml(escape=True, standalone='no') - - eclipseLaunches = join('mx', 'eclipse-launches') - if not exists(eclipseLaunches): - os.makedirs(eclipseLaunches) - return update_file(join(eclipseLaunches, name + '.launch'), launch) - -def eclipseinit(args, buildProcessorJars=True, refreshOnly=False): - """(re)generate Eclipse project configurations and working sets""" - for s in suites(True): - _eclipseinit_suite(args, s, buildProcessorJars, refreshOnly) - - generate_eclipse_workingsets() - -def _check_ide_timestamp(suite, configZip, ide): - """return True if and only if the projects file, eclipse-settings files, and mx itself are all older than configZip""" - suitePyFiles = [join(suite.mxDir, e) for e in os.listdir(suite.mxDir) if e.startswith('suite') and e.endswith('.py')] - if configZip.isOlderThan(suitePyFiles): - return False - # Assume that any mx change might imply changes to the generated IDE files - if configZip.isOlderThan(__file__): - return False - - if ide == 'eclipse': - eclipseSettingsDir = join(suite.mxDir, 'eclipse-settings') - if exists(eclipseSettingsDir): - for name in os.listdir(eclipseSettingsDir): - path = join(eclipseSettingsDir, name) - if configZip.isOlderThan(path): - return False - return True - -def _eclipseinit_project(p, files=None, libFiles=None): - assert java(p.javaCompliance) - - if not exists(p.dir): - os.makedirs(p.dir) - - out = XMLDoc() - out.open('classpath') - - for src in p.srcDirs: - srcDir = join(p.dir, src) - if not exists(srcDir): - os.mkdir(srcDir) - out.element('classpathentry', {'kind' : 'src', 'path' : src}) - - processorPath = p.annotation_processors_path() - if processorPath: - genDir = p.source_gen_dir() - if not exists(genDir): - os.mkdir(genDir) - out.open('classpathentry', {'kind' : 'src', 'path' : 'src_gen'}) - if p.uses_annotation_processor_library(): - # ignore warnings produced by third-party annotation processors - out.open('attributes') - out.element('attribute', {'name' : 'ignore_optional_problems', 'value' : 'true'}) - out.close('attributes') - out.close('classpathentry') - - if files: - files.append(genDir) - - # Every Java program depends on a JRE - out.element('classpathentry', {'kind' : 'con', 'path' : 'org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-' + str(p.javaCompliance)}) - - if exists(join(p.dir, 'plugin.xml')): # eclipse plugin project - out.element('classpathentry', {'kind' : 'con', 'path' : 'org.eclipse.pde.core.requiredPlugins'}) - - containerDeps = set() - libraryDeps = set() - projectDeps = set() - - for dep in p.all_deps([], True): - if dep == p: - continue - if dep.isLibrary(): - if hasattr(dep, 'eclipse.container'): - container = getattr(dep, 'eclipse.container') - containerDeps.add(container) - libraryDeps -= set(dep.all_deps([], True)) - else: - libraryDeps.add(dep) - elif dep.isProject(): - projectDeps.add(dep) - - for dep in sorted(containerDeps): - out.element('classpathentry', {'exported' : 'true', 'kind' : 'con', 'path' : dep}) - - for dep in sorted(libraryDeps): - path = dep.path - dep.get_path(resolve=True) - - # Relative paths for "lib" class path entries have various semantics depending on the Eclipse - # version being used (e.g. see https://bugs.eclipse.org/bugs/show_bug.cgi?id=274737) so it's - # safest to simply use absolute paths. - path = _make_absolute(path, p.suite.dir) - - attributes = {'exported' : 'true', 'kind' : 'lib', 'path' : path} - - sourcePath = dep.get_source_path(resolve=True) - if sourcePath is not None: - attributes['sourcepath'] = sourcePath - out.element('classpathentry', attributes) - if libFiles: - libFiles.append(path) - - for dep in sorted(projectDeps): - out.element('classpathentry', {'combineaccessrules' : 'false', 'exported' : 'true', 'kind' : 'src', 'path' : '/' + dep.name}) - - out.element('classpathentry', {'kind' : 'output', 'path' : getattr(p, 'eclipse.output', 'bin')}) - out.close('classpath') - classpathFile = join(p.dir, '.classpath') - update_file(classpathFile, out.xml(indent='\t', newl='\n')) - if files: - files.append(classpathFile) - - csConfig = join(project(p.checkstyleProj).dir, '.checkstyle_checks.xml') - if exists(csConfig): - out = XMLDoc() - - dotCheckstyle = join(p.dir, ".checkstyle") - checkstyleConfigPath = '/' + p.checkstyleProj + '/.checkstyle_checks.xml' - out.open('fileset-config', {'file-format-version' : '1.2.0', 'simple-config' : 'true'}) - out.open('local-check-config', {'name' : 'Checks', 'location' : checkstyleConfigPath, 'type' : 'project', 'description' : ''}) - out.element('additional-data', {'name' : 'protect-config-file', 'value' : 'false'}) - out.close('local-check-config') - out.open('fileset', {'name' : 'all', 'enabled' : 'true', 'check-config-name' : 'Checks', 'local' : 'true'}) - out.element('file-match-pattern', {'match-pattern' : '.', 'include-pattern' : 'true'}) - out.close('fileset') - out.open('filter', {'name' : 'all', 'enabled' : 'true', 'check-config-name' : 'Checks', 'local' : 'true'}) - out.element('filter-data', {'value' : 'java'}) - out.close('filter') - - exclude = join(p.dir, '.checkstyle.exclude') - if exists(exclude): - out.open('filter', {'name' : 'FilesFromPackage', 'enabled' : 'true'}) - with open(exclude) as f: - for line in f: - if not line.startswith('#'): - line = line.strip() - exclDir = join(p.dir, line) - assert isdir(exclDir), 'excluded source directory listed in ' + exclude + ' does not exist or is not a directory: ' + exclDir - out.element('filter-data', {'value' : line}) - out.close('filter') - - out.close('fileset-config') - update_file(dotCheckstyle, out.xml(indent=' ', newl='\n')) - if files: - files.append(dotCheckstyle) - else: - # clean up existing .checkstyle file - dotCheckstyle = join(p.dir, ".checkstyle") - if exists(dotCheckstyle): - os.unlink(dotCheckstyle) - - out = XMLDoc() - out.open('projectDescription') - out.element('name', data=p.name) - out.element('comment', data='') - out.element('projects', data='') - out.open('buildSpec') - out.open('buildCommand') - out.element('name', data='org.eclipse.jdt.core.javabuilder') - out.element('arguments', data='') - out.close('buildCommand') - if exists(csConfig): - out.open('buildCommand') - out.element('name', data='net.sf.eclipsecs.core.CheckstyleBuilder') - out.element('arguments', data='') - out.close('buildCommand') - if exists(join(p.dir, 'plugin.xml')): # eclipse plugin project - for buildCommand in ['org.eclipse.pde.ManifestBuilder', 'org.eclipse.pde.SchemaBuilder']: - out.open('buildCommand') - out.element('name', data=buildCommand) - out.element('arguments', data='') - out.close('buildCommand') - - if p.definedAnnotationProcessorsDist: - # Create a launcher that will (re)build the annotation processor - # jar any time one of its sources is modified. - dist = p.definedAnnotationProcessorsDist - - distProjects = [d for d in dist.sorted_deps(transitive=True) if d.isProject()] - relevantResources = [] - for p in distProjects: - for srcDir in p.source_dirs(): - relevantResources.append(join(p.name, os.path.relpath(srcDir, p.dir))) - relevantResources.append(join(p.name, os.path.relpath(p.output_dir(), p.dir))) - - # The path should always be p.name/dir independent of where the workspace actually is. - # So we use the parent folder of the project, whatever that is, to generate such a relative path. - logicalWorkspaceRoot = os.path.dirname(p.dir) - refreshFile = os.path.relpath(p.definedAnnotationProcessorsDist.path, logicalWorkspaceRoot) - _genEclipseBuilder(out, p, 'CreateAnnotationProcessorJar', 'archive @' + dist.name, refresh=True, refreshFile=refreshFile, relevantResources=relevantResources, async=True, xmlIndent='', xmlStandalone='no') - - out.close('buildSpec') - out.open('natures') - out.element('nature', data='org.eclipse.jdt.core.javanature') - if exists(csConfig): - out.element('nature', data='net.sf.eclipsecs.core.CheckstyleNature') - if exists(join(p.dir, 'plugin.xml')): # eclipse plugin project - out.element('nature', data='org.eclipse.pde.PluginNature') - out.close('natures') - out.close('projectDescription') - projectFile = join(p.dir, '.project') - update_file(projectFile, out.xml(indent='\t', newl='\n')) - if files: - files.append(projectFile) - - settingsDir = join(p.dir, ".settings") - if not exists(settingsDir): - os.mkdir(settingsDir) - - # collect the defaults from mxtool - defaultEclipseSettingsDir = join(dirname(__file__), 'eclipse-settings') - esdict = {} - if exists(defaultEclipseSettingsDir): - for name in os.listdir(defaultEclipseSettingsDir): - if isfile(join(defaultEclipseSettingsDir, name)): - esdict[name] = os.path.abspath(join(defaultEclipseSettingsDir, name)) - - # check for suite overrides - eclipseSettingsDir = join(p.suite.mxDir, 'eclipse-settings') - if exists(eclipseSettingsDir): - for name in os.listdir(eclipseSettingsDir): - if isfile(join(eclipseSettingsDir, name)): - esdict[name] = os.path.abspath(join(eclipseSettingsDir, name)) - - # check for project overrides - projectSettingsDir = join(p.dir, 'eclipse-settings') - if exists(projectSettingsDir): - for name in os.listdir(projectSettingsDir): - if isfile(join(projectSettingsDir, name)): - esdict[name] = os.path.abspath(join(projectSettingsDir, name)) - - # copy a possibly modified file to the project's .settings directory - for name, path in esdict.iteritems(): - # ignore this file altogether if this project has no annotation processors - if name == "org.eclipse.jdt.apt.core.prefs" and not processorPath: - continue - - with open(path) as f: - content = f.read() - content = content.replace('${javaCompliance}', str(p.javaCompliance)) - if processorPath: - content = content.replace('org.eclipse.jdt.core.compiler.processAnnotations=disabled', 'org.eclipse.jdt.core.compiler.processAnnotations=enabled') - update_file(join(settingsDir, name), content) - if files: - files.append(join(settingsDir, name)) - - if processorPath: - out = XMLDoc() - out.open('factorypath') - out.element('factorypathentry', {'kind' : 'PLUGIN', 'id' : 'org.eclipse.jst.ws.annotations.core', 'enabled' : 'true', 'runInBatchMode' : 'false'}) - for e in processorPath.split(os.pathsep): - out.element('factorypathentry', {'kind' : 'EXTJAR', 'id' : e, 'enabled' : 'true', 'runInBatchMode' : 'false'}) - out.close('factorypath') - update_file(join(p.dir, '.factorypath'), out.xml(indent='\t', newl='\n')) - if files: - files.append(join(p.dir, '.factorypath')) - -def _eclipseinit_suite(args, suite, buildProcessorJars=True, refreshOnly=False): - configZip = TimeStampFile(join(suite.mxDir, 'eclipse-config.zip')) - configLibsZip = join(suite.mxDir, 'eclipse-config-libs.zip') - if refreshOnly and not configZip.exists(): - return - - if _check_ide_timestamp(suite, configZip, 'eclipse'): - logv('[Eclipse configurations are up to date - skipping]') - return - - files = [] - libFiles = [] - if buildProcessorJars: - files += _processorjars_suite(suite) - - for p in suite.projects: - if p.native: - continue - _eclipseinit_project(p, files, libFiles) - - _, launchFile = make_eclipse_attach(suite, 'localhost', '8000', deps=sorted_deps(projectNames=None, includeLibs=True)) - files.append(launchFile) - - # Create an Eclipse project for each distribution that will create/update the archive - # for the distribution whenever any (transitively) dependent project of the - # distribution is updated. - for dist in suite.dists: - projectDir = dist.get_ide_project_dir() - if not projectDir: - continue - if not exists(projectDir): - os.makedirs(projectDir) - distProjects = [d for d in dist.sorted_deps(transitive=True) if d.isProject()] - relevantResources = [] - for p in distProjects: - for srcDir in p.source_dirs(): - relevantResources.append(join(p.name, os.path.relpath(srcDir, p.dir))) - relevantResources.append(join(p.name, os.path.relpath(p.output_dir(), p.dir))) - out = XMLDoc() - out.open('projectDescription') - out.element('name', data=dist.name) - out.element('comment', data='Updates ' + dist.path + ' if a project dependency of ' + dist.name + ' is updated') - out.open('projects') - for p in distProjects: - out.element('project', data=p.name) - for d in dist.distDependencies: - out.element('project', data=d) - out.close('projects') - out.open('buildSpec') - dist.dir = projectDir - dist.javaCompliance = max([p.javaCompliance for p in distProjects]) - _genEclipseBuilder(out, dist, 'Create' + dist.name + 'Dist', 'archive @' + dist.name, relevantResources=relevantResources, logToFile=True, refresh=False, async=True) - out.close('buildSpec') - out.open('natures') - out.element('nature', data='org.eclipse.jdt.core.javanature') - out.close('natures') - out.close('projectDescription') - projectFile = join(projectDir, '.project') - update_file(projectFile, out.xml(indent='\t', newl='\n')) - files.append(projectFile) - - _zip_files(files, suite.dir, configZip.path) - _zip_files(libFiles, suite.dir, configLibsZip) - -def _zip_files(files, baseDir, zipPath): - fd, tmp = tempfile.mkstemp(suffix='', prefix=basename(zipPath), dir=baseDir) - try: - zf = zipfile.ZipFile(tmp, 'w') - for f in sorted(set(files)): - relpath = os.path.relpath(f, baseDir) - arcname = relpath.replace(os.sep, '/') - zf.write(f, arcname) - zf.close() - os.close(fd) - # Atomic on Unix - shutil.move(tmp, zipPath) - # Correct the permissions on the temporary file which is created with restrictive permissions - os.chmod(zipPath, 0o666 & ~currentUmask) - finally: - if exists(tmp): - os.remove(tmp) - -def _genEclipseBuilder(dotProjectDoc, p, name, mxCommand, refresh=True, refreshFile=None, relevantResources=None, async=False, logToConsole=False, logToFile=False, appendToLogFile=True, xmlIndent='\t', xmlStandalone=None): - externalToolDir = join(p.dir, '.externalToolBuilders') - launchOut = XMLDoc() - consoleOn = 'true' if logToConsole else 'false' - launchOut.open('launchConfiguration', {'type' : 'org.eclipse.ui.externaltools.ProgramBuilderLaunchConfigurationType'}) - launchOut.element('booleanAttribute', {'key' : 'org.eclipse.debug.core.capture_output', 'value': consoleOn}) - launchOut.open('mapAttribute', {'key' : 'org.eclipse.debug.core.environmentVariables'}) - launchOut.element('mapEntry', {'key' : 'JAVA_HOME', 'value' : _default_java_home.jdk}) - launchOut.element('mapEntry', {'key' : 'EXTRA_JAVA_HOMES', 'value' : os.pathsep.join([extraJavaHome.jdk for extraJavaHome in _extra_java_homes])}) - launchOut.close('mapAttribute') - - if refresh: - if refreshFile is None: - refreshScope = '${project}' - else: - refreshScope = '${working_set:<?xml version="1.0" encoding="UTF-8"?><resources><item path="' + refreshFile + '" type="1"/></resources>}' - - launchOut.element('booleanAttribute', {'key' : 'org.eclipse.debug.core.ATTR_REFRESH_RECURSIVE', 'value': 'false'}) - launchOut.element('stringAttribute', {'key' : 'org.eclipse.debug.core.ATTR_REFRESH_SCOPE', 'value': refreshScope}) - - if relevantResources is not None: - resources = '${working_set:<?xml version="1.0" encoding="UTF-8"?><resources>' - for relevantResource in relevantResources: - resources += '<item path="' + relevantResource + '" type="2" />' - resources += '</resources>}' - launchOut.element('stringAttribute', {'key' : 'org.eclipse.ui.externaltools.ATTR_BUILD_SCOPE', 'value': resources}) - - - launchOut.element('booleanAttribute', {'key' : 'org.eclipse.debug.ui.ATTR_CONSOLE_OUTPUT_ON', 'value': consoleOn}) - launchOut.element('booleanAttribute', {'key' : 'org.eclipse.debug.ui.ATTR_LAUNCH_IN_BACKGROUND', 'value': 'true' if async else 'false'}) - if logToFile: - logFile = join(externalToolDir, name + '.log') - launchOut.element('stringAttribute', {'key' : 'org.eclipse.debug.ui.ATTR_CAPTURE_IN_FILE', 'value': logFile}) - launchOut.element('booleanAttribute', {'key' : 'org.eclipse.debug.ui.ATTR_APPEND_TO_FILE', 'value': 'true' if appendToLogFile else 'false'}) - - # expect to find the OS command to invoke mx in the same directory - baseDir = dirname(os.path.abspath(__file__)) - - cmd = 'mx.sh' - if get_os() == 'windows': - cmd = 'mx.cmd' - cmdPath = join(baseDir, cmd) - if not os.path.exists(cmdPath): - # backwards compatibility for when the commands lived in parent of mxtool - cmdPath = join(dirname(baseDir), cmd) - if not os.path.exists(cmdPath): - abort('cannot locate ' + cmd) - - launchOut.element('stringAttribute', {'key' : 'org.eclipse.ui.externaltools.ATTR_LOCATION', 'value': cmdPath}) - launchOut.element('stringAttribute', {'key' : 'org.eclipse.ui.externaltools.ATTR_RUN_BUILD_KINDS', 'value': 'full,incremental,auto,'}) - launchOut.element('stringAttribute', {'key' : 'org.eclipse.ui.externaltools.ATTR_TOOL_ARGUMENTS', 'value': mxCommand}) - launchOut.element('booleanAttribute', {'key' : 'org.eclipse.ui.externaltools.ATTR_TRIGGERS_CONFIGURED', 'value': 'true'}) - launchOut.element('stringAttribute', {'key' : 'org.eclipse.ui.externaltools.ATTR_WORKING_DIRECTORY', 'value': p.suite.dir}) - - - launchOut.close('launchConfiguration') - - if not exists(externalToolDir): - os.makedirs(externalToolDir) - update_file(join(externalToolDir, name + '.launch'), launchOut.xml(indent=xmlIndent, standalone=xmlStandalone, newl='\n')) - - dotProjectDoc.open('buildCommand') - dotProjectDoc.element('name', data='org.eclipse.ui.externaltools.ExternalToolBuilder') - dotProjectDoc.element('triggers', data='auto,full,incremental,') - dotProjectDoc.open('arguments') - dotProjectDoc.open('dictionary') - dotProjectDoc.element('key', data='LaunchConfigHandle') - dotProjectDoc.element('value', data='<project>/.externalToolBuilders/' + name + '.launch') - dotProjectDoc.close('dictionary') - dotProjectDoc.open('dictionary') - dotProjectDoc.element('key', data='incclean') - dotProjectDoc.element('value', data='true') - dotProjectDoc.close('dictionary') - dotProjectDoc.close('arguments') - dotProjectDoc.close('buildCommand') - -def generate_eclipse_workingsets(): - """ - Populate the workspace's working set configuration with working sets generated from project data for the primary suite - If the workspace already contains working set definitions, the existing ones will be retained and extended. - In case mx/env does not contain a WORKSPACE definition pointing to the workspace root directory, a parent search from the primary suite directory is performed. - If no workspace root directory can be identified, the primary suite directory is used and the user has to place the workingsets.xml file by hand. - """ - - # identify the location where to look for workingsets.xml - wsfilename = 'workingsets.xml' - wsloc = '.metadata/.plugins/org.eclipse.ui.workbench' - if os.environ.has_key('WORKSPACE'): - expected_wsroot = os.environ['WORKSPACE'] - else: - expected_wsroot = _primary_suite.dir - - wsroot = _find_eclipse_wsroot(expected_wsroot) - if wsroot is None: - # failed to find it - wsroot = expected_wsroot - - wsdir = join(wsroot, wsloc) - if not exists(wsdir): - wsdir = wsroot - logv('Could not find Eclipse metadata directory. Please place ' + wsfilename + ' in ' + wsloc + ' manually.') - wspath = join(wsdir, wsfilename) - - # gather working set info from project data - workingSets = dict() - for p in projects(): - if p.workingSets is None: - continue - for w in p.workingSets.split(","): - if not workingSets.has_key(w): - workingSets[w] = [p.name] - else: - workingSets[w].append(p.name) - - if exists(wspath): - wsdoc = _copy_workingset_xml(wspath, workingSets) - else: - wsdoc = _make_workingset_xml(workingSets) - - update_file(wspath, wsdoc.xml(newl='\n')) - -def _find_eclipse_wsroot(wsdir): - md = join(wsdir, '.metadata') - if exists(md): - return wsdir - split = os.path.split(wsdir) - if split[0] == wsdir: # root directory - return None - else: - return _find_eclipse_wsroot(split[0]) - -def _make_workingset_xml(workingSets): - wsdoc = XMLDoc() - wsdoc.open('workingSetManager') - - for w in sorted(workingSets.keys()): - _workingset_open(wsdoc, w) - for p in workingSets[w]: - _workingset_element(wsdoc, p) - wsdoc.close('workingSet') - - wsdoc.close('workingSetManager') - return wsdoc - -def _copy_workingset_xml(wspath, workingSets): - target = XMLDoc() - target.open('workingSetManager') - - parser = xml.parsers.expat.ParserCreate() - - class ParserState(object): - def __init__(self): - self.current_ws_name = 'none yet' - self.current_ws = None - self.seen_ws = list() - self.seen_projects = list() - self.aggregate_ws = False - self.nested_ws = False - - ps = ParserState() - - # parsing logic - def _ws_start(name, attributes): - if name == 'workingSet': - if attributes.has_key('name'): - ps.current_ws_name = attributes['name'] - if attributes.has_key('aggregate') and attributes['aggregate'] == 'true': - ps.aggregate_ws = True - ps.current_ws = None - elif workingSets.has_key(ps.current_ws_name): - ps.current_ws = workingSets[ps.current_ws_name] - ps.seen_ws.append(ps.current_ws_name) - ps.seen_projects = list() - else: - ps.current_ws = None - target.open(name, attributes) - parser.StartElementHandler = _ws_item - - def _ws_end(name): - closeAndResetHandler = False - if name == 'workingSet': - if ps.aggregate_ws: - if ps.nested_ws: - ps.nested_ws = False - else: - ps.aggregate_ws = False - closeAndResetHandler = True - else: - if not ps.current_ws is None: - for p in ps.current_ws: - if not p in ps.seen_projects: - _workingset_element(target, p) - closeAndResetHandler = True - if closeAndResetHandler: - target.close('workingSet') - parser.StartElementHandler = _ws_start - elif name == 'workingSetManager': - # process all working sets that are new to the file - for w in sorted(workingSets.keys()): - if not w in ps.seen_ws: - _workingset_open(target, w) - for p in workingSets[w]: - _workingset_element(target, p) - target.close('workingSet') - - def _ws_item(name, attributes): - if name == 'item': - if ps.current_ws is None: - target.element(name, attributes) - elif not attributes.has_key('elementID') and attributes.has_key('factoryID') and attributes.has_key('path') and attributes.has_key('type'): - target.element(name, attributes) - p_name = attributes['path'][1:] # strip off the leading '/' - ps.seen_projects.append(p_name) - else: - p_name = attributes['elementID'][1:] # strip off the leading '=' - _workingset_element(target, p_name) - ps.seen_projects.append(p_name) - elif name == 'workingSet': - ps.nested_ws = True - target.element(name, attributes) - - # process document - parser.StartElementHandler = _ws_start - parser.EndElementHandler = _ws_end - with open(wspath, 'r') as wsfile: - parser.ParseFile(wsfile) - - target.close('workingSetManager') - return target - -def _workingset_open(wsdoc, ws): - wsdoc.open('workingSet', {'editPageID': 'org.eclipse.jdt.ui.JavaWorkingSetPage', 'factoryID': 'org.eclipse.ui.internal.WorkingSetFactory', 'id': 'wsid_' + ws, 'label': ws, 'name': ws}) - -def _workingset_element(wsdoc, p): - wsdoc.element('item', {'elementID': '=' + p, 'factoryID': 'org.eclipse.jdt.ui.PersistableJavaElementFactory'}) - -def netbeansinit(args, refreshOnly=False, buildProcessorJars=True): - """(re)generate NetBeans project configurations""" - - for suite in suites(True): - _netbeansinit_suite(args, suite, refreshOnly, buildProcessorJars) - -def _netbeansinit_project(p, jdks=None, files=None, libFiles=None): - if not exists(join(p.dir, 'nbproject')): - os.makedirs(join(p.dir, 'nbproject')) - - jdk = java(p.javaCompliance) - assert jdk - - if jdks: - jdks.add(jdk) - - out = XMLDoc() - out.open('project', {'name' : p.name, 'default' : 'default', 'basedir' : '.'}) - out.element('description', data='Builds, tests, and runs the project ' + p.name + '.') - out.element('import', {'file' : 'nbproject/build-impl.xml'}) - out.open('target', {'name' : '-post-init'}) - out.open('pathconvert', {'property' : 'comma.javac.classpath', 'pathsep' : ','}) - out.element('path', {'path' : '${javac.classpath}'}) - out.close('pathconvert') - - out.open('restrict', {'id' : 'missing.javac.classpath'}) - out.element('filelist', {'dir' : '${basedir}', 'files' : '${comma.javac.classpath}'}) - out.open('not') - out.element('exists') - out.close('not') - out.close('restrict') - - out.element('property', {'name' : 'missing.javac.classpath', 'refid' : 'missing.javac.classpath'}) - - out.open('condition', {'property' : 'no.dependencies', 'value' : 'true'}) - out.element('equals', {'arg1' : '${missing.javac.classpath}', 'arg2' : ''}) - out.close('condition') - - out.element('property', {'name' : 'no.dependencies', 'value' : 'false'}) - - out.open('condition', {'property' : 'no.deps'}) - out.element('equals', {'arg1' : '${no.dependencies}', 'arg2' : 'true'}) - out.close('condition') - - out.close('target') - out.open('target', {'name' : 'compile'}) - out.open('exec', {'executable' : sys.executable, 'failonerror' : 'true'}) - out.element('env', {'key' : 'JAVA_HOME', 'value' : jdk.jdk}) - out.element('arg', {'value' : os.path.abspath(__file__)}) - out.element('arg', {'value' : 'build'}) - out.element('arg', {'value' : '--only'}) - out.element('arg', {'value' : p.name}) - out.element('arg', {'value' : '--force-javac'}) - out.element('arg', {'value' : '--no-native'}) - out.close('exec') - out.close('target') - out.open('target', {'name' : 'jar', 'depends' : 'compile'}) - out.close('target') - out.open('target', {'name' : 'run', 'depends' : 'compile'}) - out.open('exec', {'executable' : sys.executable, 'failonerror' : 'true'}) - out.element('env', {'key' : 'JAVA_HOME', 'value' : jdk.jdk}) - out.element('arg', {'value' : os.path.abspath(__file__)}) - out.element('arg', {'value' : 'unittest'}) - out.element('arg', {'value' : p.name}) - out.close('exec') - out.close('target') - out.open('target', {'name' : 'debug', 'depends' : 'init,compile'}) - out.open('nbjpdastart', {'addressproperty' : 'jpda.address', 'name' : p.name, }) - out.open('classpath') - out.element('path', {'path' : '${javac.classpath}'}) - out.close('classpath') - out.open('sourcepath') - out.element('pathelement', {'location' : 'src'}) - out.close('sourcepath') - out.close('nbjpdastart') - out.open('exec', {'executable' : sys.executable, 'failonerror' : 'true'}) - out.element('env', {'key' : 'JAVA_HOME', 'value' : jdk.jdk}) - out.element('arg', {'value' : os.path.abspath(__file__)}) - out.element('arg', {'value' : '-d'}) - out.element('arg', {'value' : '--attach'}) - out.element('arg', {'value' : '${jpda.address}'}) - out.element('arg', {'value' : 'unittest'}) - out.element('arg', {'value' : p.name}) - out.close('exec') - out.close('target') - out.open('target', {'name' : 'javadoc'}) - out.open('exec', {'executable' : sys.executable, 'failonerror' : 'true'}) - out.element('env', {'key' : 'JAVA_HOME', 'value' : jdk.jdk}) - out.element('arg', {'value' : os.path.abspath(__file__)}) - out.element('arg', {'value' : 'javadoc'}) - out.element('arg', {'value' : '--projects'}) - out.element('arg', {'value' : p.name}) - out.close('exec') - out.element('nbbrowse', {'file' : 'javadoc/index.html'}) - out.close('target') - out.close('project') - update_file(join(p.dir, 'build.xml'), out.xml(indent='\t', newl='\n')) - if files: - files.append(join(p.dir, 'build.xml')) - - out = XMLDoc() - out.open('project', {'xmlns' : 'http://www.netbeans.org/ns/project/1'}) - out.element('type', data='org.netbeans.modules.java.j2seproject') - out.open('configuration') - out.open('data', {'xmlns' : 'http://www.netbeans.org/ns/j2se-project/3'}) - out.element('name', data=p.name) - out.element('explicit-platform', {'explicit-source-supported' : 'true'}) - out.open('source-roots') - out.element('root', {'id' : 'src.dir'}) - if len(p.annotation_processors()) > 0: - out.element('root', {'id' : 'src.ap-source-output.dir', 'name' : 'Generated Packages'}) - out.close('source-roots') - out.open('test-roots') - out.close('test-roots') - out.close('data') - - firstDep = True - for dep in p.all_deps([], includeLibs=False, includeAnnotationProcessors=True): - if dep == p: - continue - - if dep.isProject(): - n = dep.name.replace('.', '_') - if firstDep: - out.open('references', {'xmlns' : 'http://www.netbeans.org/ns/ant-project-references/1'}) - firstDep = False - - out.open('reference') - out.element('foreign-project', data=n) - out.element('artifact-type', data='jar') - out.element('script', data='build.xml') - out.element('target', data='jar') - out.element('clean-target', data='clean') - out.element('id', data='jar') - out.close('reference') - - if not firstDep: - out.close('references') - - out.close('configuration') - out.close('project') - update_file(join(p.dir, 'nbproject', 'project.xml'), out.xml(indent=' ', newl='\n')) - if files: - files.append(join(p.dir, 'nbproject', 'project.xml')) - - out = StringIO.StringIO() - jdkPlatform = 'JDK_' + str(jdk.version) - - annotationProcessorEnabled = "false" - annotationProcessorSrcFolder = "" - if len(p.annotation_processors()) > 0: - annotationProcessorEnabled = "true" - genSrcDir = p.source_gen_dir() - if not exists(genSrcDir): - os.makedirs(genSrcDir) - annotationProcessorSrcFolder = "src.ap-source-output.dir=" + genSrcDir - - content = """ -annotation.processing.enabled=""" + annotationProcessorEnabled + """ -annotation.processing.enabled.in.editor=""" + annotationProcessorEnabled + """ -annotation.processing.processors.list= -annotation.processing.run.all.processors=true -application.title=""" + p.name + """ -application.vendor=mx -build.classes.dir=${build.dir} -build.classes.excludes=**/*.java,**/*.form -# This directory is removed when the project is cleaned: -build.dir=bin -build.generated.sources.dir=${build.dir}/generated-sources -# Only compile against the classpath explicitly listed here: -build.sysclasspath=ignore -build.test.classes.dir=${build.dir}/test/classes -build.test.results.dir=${build.dir}/test/results -# Uncomment to specify the preferred debugger connection transport: -#debug.transport=dt_socket -debug.classpath=\\ -${run.classpath} -debug.test.classpath=\\ -${run.test.classpath} -# This directory is removed when the project is cleaned: -dist.dir=dist -dist.jar=${dist.dir}/""" + p.name + """.jar -dist.javadoc.dir=${dist.dir}/javadoc -endorsed.classpath= -excludes= -includes=** -jar.compress=false -# Space-separated list of extra javac options -javac.compilerargs=-XDignore.symbol.file -javac.deprecation=false -javac.source=""" + str(p.javaCompliance) + """ -javac.target=""" + str(p.javaCompliance) + """ -javac.test.classpath=\\ -${javac.classpath}:\\ -${build.classes.dir} -javadoc.additionalparam= -javadoc.author=false -javadoc.encoding=${source.encoding} -javadoc.noindex=false -javadoc.nonavbar=false -javadoc.notree=false -javadoc.private=false -javadoc.splitindex=true -javadoc.use=true -javadoc.version=false -javadoc.windowtitle= -main.class=com.oracle.truffle.api.impl.Accessor -manifest.file=manifest.mf -meta.inf.dir=${src.dir}/META-INF -mkdist.disabled=false -platforms.""" + jdkPlatform + """.home=""" + jdk.jdk + """ -platform.active=""" + jdkPlatform + """ -run.classpath=\\ -${javac.classpath}:\\ -${build.classes.dir} -# Space-separated list of JVM arguments used when running the project -# (you may also define separate properties like run-sys-prop.name=value instead of -Dname=value -# or test-sys-prop.name=value to set system properties for unit tests): -run.jvmargs= -run.test.classpath=\\ -${javac.test.classpath}:\\ -${build.test.classes.dir} -test.src.dir=./test -""" + annotationProcessorSrcFolder + """ -source.encoding=UTF-8""".replace(':', os.pathsep).replace('/', os.sep) - print >> out, content - - mainSrc = True - for src in p.srcDirs: - srcDir = join(p.dir, src) - if not exists(srcDir): - os.mkdir(srcDir) - ref = 'file.reference.' + p.name + '-' + src - print >> out, ref + '=' + src - if mainSrc: - print >> out, 'src.dir=${' + ref + '}' - mainSrc = False - else: - print >> out, 'src.' + src + '.dir=${' + ref + '}' - - javacClasspath = [] - - deps = p.all_deps([], True) - annotationProcessorOnlyDeps = [] - if len(p.annotation_processors()) > 0: - for ap in p.annotation_processors(): - apDep = dependency(ap) - if not apDep in deps: - deps.append(apDep) - annotationProcessorOnlyDeps.append(apDep) - - annotationProcessorReferences = [] - - for dep in deps: - if dep == p: - continue - - if dep.isLibrary(): - path = dep.get_path(resolve=True) - if path: - if os.sep == '\\': - path = path.replace('\\', '\\\\') - ref = 'file.reference.' + dep.name + '-bin' - print >> out, ref + '=' + path - if libFiles: - libFiles.append(path) - - elif dep.isProject(): - n = dep.name.replace('.', '_') - relDepPath = os.path.relpath(dep.dir, p.dir).replace(os.sep, '/') - ref = 'reference.' + n + '.jar' - print >> out, 'project.' + n + '=' + relDepPath - print >> out, ref + '=${project.' + n + '}/dist/' + dep.name + '.jar' - - if not dep in annotationProcessorOnlyDeps: - javacClasspath.append('${' + ref + '}') - else: - annotationProcessorReferences.append('${' + ref + '}') - - print >> out, 'javac.classpath=\\\n ' + (os.pathsep + '\\\n ').join(javacClasspath) - print >> out, 'javac.processorpath=' + (os.pathsep + '\\\n ').join(['${javac.classpath}'] + annotationProcessorReferences) - print >> out, 'javac.test.processorpath=' + (os.pathsep + '\\\n ').join(['${javac.test.classpath}'] + annotationProcessorReferences) - - update_file(join(p.dir, 'nbproject', 'project.properties'), out.getvalue()) - out.close() - if files: - files.append(join(p.dir, 'nbproject', 'project.properties')) - -def _netbeansinit_suite(args, suite, refreshOnly=False, buildProcessorJars=True): - configZip = TimeStampFile(join(suite.mxDir, 'netbeans-config.zip')) - configLibsZip = join(suite.mxDir, 'eclipse-config-libs.zip') - if refreshOnly and not configZip.exists(): - return - - if _check_ide_timestamp(suite, configZip, 'netbeans'): - logv('[NetBeans configurations are up to date - skipping]') - return - - files = [] - libFiles = [] - jdks = set() - for p in suite.projects: - if p.native: - continue - - if exists(join(p.dir, 'plugin.xml')): # eclipse plugin project - continue - - _netbeansinit_project(p, jdks, files, libFiles) - - log('If using NetBeans:') - # http://stackoverflow.com/questions/24720665/cant-resolve-jdk-internal-package - log(' 1. Edit etc/netbeans.conf in your NetBeans installation and modify netbeans_default_options variable to include "-J-DCachingArchiveProvider.disableCtSym=true"') - log(' 2. Ensure that the following platform(s) are defined (Tools -> Java Platforms):') - for jdk in jdks: - log(' JDK_' + str(jdk.version)) - log(' 3. Open/create a Project Group for the directory containing the projects (File -> Project Group -> New Group... -> Folder of Projects)') - - _zip_files(files, suite.dir, configZip.path) - _zip_files(libFiles, suite.dir, configLibsZip) - -def intellijinit(args, refreshOnly=False): - """(re)generate Intellij project configurations""" - - for suite in suites(True): - _intellij_suite(args, suite, refreshOnly) - -def _intellij_suite(args, suite, refreshOnly=False): - - libraries = set() - - ideaProjectDirectory = join(suite.dir, '.idea') - - if not exists(ideaProjectDirectory): - os.mkdir(ideaProjectDirectory) - nameFile = join(ideaProjectDirectory, '.name') - update_file(nameFile, "Graal") - modulesXml = XMLDoc() - modulesXml.open('project', attributes={'version': '4'}) - modulesXml.open('component', attributes={'name': 'ProjectModuleManager'}) - modulesXml.open('modules') - - - def _intellij_exclude_if_exists(xml, p, name): - path = join(p.dir, name) - if exists(path): - xml.element('excludeFolder', attributes={'url':'file://$MODULE_DIR$/' + name}) - - annotationProcessorProfiles = {} - - def _complianceToIntellijLanguageLevel(compliance): - return 'JDK_1_' + str(compliance.value) - - # create the modules (1 module = 1 Intellij project) - for p in suite.projects: - if p.native: - continue - - assert java(p.javaCompliance) - - if not exists(p.dir): - os.makedirs(p.dir) - - annotationProcessorProfileKey = tuple(p.annotation_processors()) - - if not annotationProcessorProfileKey in annotationProcessorProfiles: - annotationProcessorProfiles[annotationProcessorProfileKey] = [p] - else: - annotationProcessorProfiles[annotationProcessorProfileKey].append(p) - - intellijLanguageLevel = _complianceToIntellijLanguageLevel(p.javaCompliance) - - moduleXml = XMLDoc() - moduleXml.open('module', attributes={'type': 'JAVA_MODULE', 'version': '4'}) - - moduleXml.open('component', attributes={'name': 'NewModuleRootManager', 'LANGUAGE_LEVEL': intellijLanguageLevel, 'inherit-compiler-output': 'false'}) - moduleXml.element('output', attributes={'url': 'file://$MODULE_DIR$/bin'}) - moduleXml.element('exclude-output') - - moduleXml.open('content', attributes={'url': 'file://$MODULE_DIR$'}) - for src in p.srcDirs: - srcDir = join(p.dir, src) - if not exists(srcDir): - os.mkdir(srcDir) - moduleXml.element('sourceFolder', attributes={'url':'file://$MODULE_DIR$/' + src, 'isTestSource': 'false'}) - - if len(p.annotation_processors()) > 0: - genDir = p.source_gen_dir() - if not exists(genDir): - os.mkdir(genDir) - moduleXml.element('sourceFolder', attributes={'url':'file://$MODULE_DIR$/' + os.path.relpath(genDir, p.dir), 'isTestSource': 'false'}) - - for name in ['.externalToolBuilders', '.settings', 'nbproject']: - _intellij_exclude_if_exists(moduleXml, p, name) - moduleXml.close('content') - - moduleXml.element('orderEntry', attributes={'type': 'jdk', 'jdkType': 'JavaSDK', 'jdkName': str(p.javaCompliance)}) - moduleXml.element('orderEntry', attributes={'type': 'sourceFolder', 'forTests': 'false'}) - - deps = p.all_deps([], True, includeAnnotationProcessors=True) - for dep in deps: - if dep == p: - continue - - if dep.isLibrary(): - libraries.add(dep) - moduleXml.element('orderEntry', attributes={'type': 'library', 'name': dep.name, 'level': 'project'}) - elif dep.isProject(): - moduleXml.element('orderEntry', attributes={'type': 'module', 'module-name': dep.name}) - - moduleXml.close('component') - moduleXml.close('module') - moduleFile = join(p.dir, p.name + '.iml') - update_file(moduleFile, moduleXml.xml(indent=' ', newl='\n')) - - moduleFilePath = "$PROJECT_DIR$/" + os.path.relpath(moduleFile, suite.dir) - modulesXml.element('module', attributes={'fileurl': 'file://' + moduleFilePath, 'filepath': moduleFilePath}) - - modulesXml.close('modules') - modulesXml.close('component') - modulesXml.close('project') - moduleXmlFile = join(ideaProjectDirectory, 'modules.xml') - update_file(moduleXmlFile, modulesXml.xml(indent=' ', newl='\n')) - - # TODO What about cross-suite dependencies? - - librariesDirectory = join(ideaProjectDirectory, 'libraries') - - if not exists(librariesDirectory): - os.mkdir(librariesDirectory) - - # Setup the libraries that were used above - # TODO: setup all the libraries from the suite regardless of usage? - for library in libraries: - libraryXml = XMLDoc() - - libraryXml.open('component', attributes={'name': 'libraryTable'}) - libraryXml.open('library', attributes={'name': library.name}) - libraryXml.open('CLASSES') - libraryXml.element('root', attributes={'url': 'jar://$PROJECT_DIR$/' + os.path.relpath(library.get_path(True), suite.dir) + '!/'}) - libraryXml.close('CLASSES') - libraryXml.element('JAVADOC') - if library.sourcePath: - libraryXml.open('SOURCES') - libraryXml.element('root', attributes={'url': 'jar://$PROJECT_DIR$/' + os.path.relpath(library.get_source_path(True), suite.dir) + '!/'}) - libraryXml.close('SOURCES') - else: - libraryXml.element('SOURCES') - libraryXml.close('library') - libraryXml.close('component') - - libraryFile = join(librariesDirectory, library.name + '.xml') - update_file(libraryFile, libraryXml.xml(indent=' ', newl='\n')) - - - - # Set annotation processor profiles up, and link them to modules in compiler.xml - compilerXml = XMLDoc() - compilerXml.open('project', attributes={'version': '4'}) - compilerXml.open('component', attributes={'name': 'CompilerConfiguration'}) - - compilerXml.element('option', attributes={'name': "DEFAULT_COMPILER", 'value': 'Javac'}) - compilerXml.element('resourceExtensions') - compilerXml.open('wildcardResourcePatterns') - compilerXml.element('entry', attributes={'name': '!?*.java'}) - compilerXml.close('wildcardResourcePatterns') - - if annotationProcessorProfiles: - compilerXml.open('annotationProcessing') - for processors, modules in sorted(annotationProcessorProfiles.iteritems()): - compilerXml.open('profile', attributes={'default': 'false', 'name': '-'.join(processors), 'enabled': 'true'}) - compilerXml.element('sourceOutputDir', attributes={'name': 'src_gen'}) # TODO use p.source_gen_dir() ? - compilerXml.element('outputRelativeToContentRoot', attributes={'value': 'true'}) - compilerXml.open('processorPath', attributes={'useClasspath': 'false'}) - for apName in processors: - pDep = dependency(apName) - for entry in pDep.all_deps([], True): - if entry.isLibrary(): - compilerXml.element('entry', attributes={'name': '$PROJECT_DIR$/' + os.path.relpath(entry.path, suite.dir)}) - elif entry.isProject(): - assert entry.isProject() - compilerXml.element('entry', attributes={'name': '$PROJECT_DIR$/' + os.path.relpath(entry.output_dir(), suite.dir)}) - compilerXml.close('processorPath') - for module in modules: - compilerXml.element('module', attributes={'name': module.name}) - compilerXml.close('profile') - compilerXml.close('annotationProcessing') - - compilerXml.close('component') - compilerXml.close('project') - compilerFile = join(ideaProjectDirectory, 'compiler.xml') - update_file(compilerFile, compilerXml.xml(indent=' ', newl='\n')) - - # Wite misc.xml for global JDK config - miscXml = XMLDoc() - miscXml.open('project', attributes={'version': '4'}) - miscXml.element('component', attributes={'name': 'ProjectRootManager', 'version': '2', 'languageLevel': _complianceToIntellijLanguageLevel(java().javaCompliance), 'project-jdk-name': str(java().javaCompliance), 'project-jdk-type': 'JavaSDK'}) - miscXml.close('project') - miscFile = join(ideaProjectDirectory, 'misc.xml') - update_file(miscFile, miscXml.xml(indent=' ', newl='\n')) - - - # TODO look into copyright settings - # TODO should add vcs.xml support - -def ideclean(args): - """remove all Eclipse and NetBeans project configurations""" - def rm(path): - if exists(path): - os.remove(path) - - for s in suites(): - rm(join(s.mxDir, 'eclipse-config.zip')) - rm(join(s.mxDir, 'netbeans-config.zip')) - shutil.rmtree(join(s.dir, '.idea'), ignore_errors=True) - - for p in projects(): - if p.native: - continue - - shutil.rmtree(join(p.dir, '.settings'), ignore_errors=True) - shutil.rmtree(join(p.dir, '.externalToolBuilders'), ignore_errors=True) - shutil.rmtree(join(p.dir, 'nbproject'), ignore_errors=True) - rm(join(p.dir, '.classpath')) - rm(join(p.dir, '.checkstyle')) - rm(join(p.dir, '.project')) - rm(join(p.dir, '.factorypath')) - rm(join(p.dir, p.name + '.iml')) - rm(join(p.dir, 'build.xml')) - rm(join(p.dir, 'eclipse-build.xml')) - try: - rm(join(p.dir, p.name + '.jar')) - except: - log("Error removing {0}".format(p.name + '.jar')) - - for d in _dists.itervalues(): - if d.get_ide_project_dir(): - shutil.rmtree(d.get_ide_project_dir(), ignore_errors=True) - -def ideinit(args, refreshOnly=False, buildProcessorJars=True): - """(re)generate Eclipse, NetBeans and Intellij project configurations""" - eclipseinit(args, refreshOnly=refreshOnly, buildProcessorJars=buildProcessorJars) - netbeansinit(args, refreshOnly=refreshOnly, buildProcessorJars=buildProcessorJars) - intellijinit(args, refreshOnly=refreshOnly) - if not refreshOnly: - fsckprojects([]) - -def fsckprojects(args): - """find directories corresponding to deleted Java projects and delete them""" - if not is_interactive(): - log('fsckprojects command must be run in an interactive shell') - return - hg = HgConfig() - for suite in suites(True): - projectDirs = [p.dir for p in suite.projects] - distIdeDirs = [d.get_ide_project_dir() for d in suite.dists if d.get_ide_project_dir() is not None] - for dirpath, dirnames, files in os.walk(suite.dir): - if dirpath == suite.dir: - # no point in traversing .hg, lib, or .workspace - dirnames[:] = [d for d in dirnames if d not in ['.hg', 'lib', '.workspace']] - elif dirpath in projectDirs: - # don't traverse subdirs of an existing project in this suite - dirnames[:] = [] - elif dirpath in distIdeDirs: - # don't traverse subdirs of an existing distributions in this suite - dirnames[:] = [] - else: - projectConfigFiles = frozenset(['.classpath', '.project', 'nbproject']) - indicators = projectConfigFiles.intersection(files) - if len(indicators) != 0: - indicators = [os.path.relpath(join(dirpath, i), suite.dir) for i in indicators] - indicatorsInHg = hg.locate(suite.dir, indicators) - # Only proceed if there are indicator files that are not under HG - if len(indicators) > len(indicatorsInHg): - if not is_interactive() or ask_yes_no(dirpath + ' looks like a removed project -- delete it', 'n'): - shutil.rmtree(dirpath) - log('Deleted ' + dirpath) - -def javadoc(args, parser=None, docDir='javadoc', includeDeps=True, stdDoclet=True): - """generate javadoc for some/all Java projects""" - - parser = ArgumentParser(prog='mx javadoc') if parser is None else parser - parser.add_argument('-d', '--base', action='store', help='base directory for output') - parser.add_argument('--unified', action='store_true', help='put javadoc in a single directory instead of one per project') - parser.add_argument('--force', action='store_true', help='(re)generate javadoc even if package-list file exists') - parser.add_argument('--projects', action='store', help='comma separated projects to process (omit to process all projects)') - parser.add_argument('--Wapi', action='store_true', dest='warnAPI', help='show warnings about using internal APIs') - parser.add_argument('--argfile', action='store', help='name of file containing extra javadoc options') - parser.add_argument('--arg', action='append', dest='extra_args', help='extra Javadoc arguments (e.g. --arg @-use)', metavar='@<arg>', default=[]) - parser.add_argument('-m', '--memory', action='store', help='-Xmx value to pass to underlying JVM') - parser.add_argument('--packages', action='store', help='comma separated packages to process (omit to process all packages)') - parser.add_argument('--exclude-packages', action='store', help='comma separated packages to exclude') - - args = parser.parse_args(args) - - # build list of projects to be processed - if args.projects is not None: - candidates = [project(name) for name in args.projects.split(',')] - else: - candidates = projects_opt_limit_to_suites() - - # optionally restrict packages within a project - packages = [] - if args.packages is not None: - packages = [name for name in args.packages.split(',')] - - exclude_packages = [] - if args.exclude_packages is not None: - exclude_packages = [name for name in args.exclude_packages.split(',')] - - def outDir(p): - if args.base is None: - return join(p.dir, docDir) - return join(args.base, p.name, docDir) - - def check_package_list(p): - return not exists(join(outDir(p), 'package-list')) - - def assess_candidate(p, projects): - if p in projects: - return False - if args.force or args.unified or check_package_list(p): - projects.append(p) - return True - return False - - projects = [] - for p in candidates: - if not p.native: - if includeDeps: - deps = p.all_deps([], includeLibs=False, includeSelf=False) - for d in deps: - assess_candidate(d, projects) - if not assess_candidate(p, projects): - logv('[package-list file exists - skipping {0}]'.format(p.name)) - - - def find_packages(sourceDirs, pkgs=None): - if pkgs is None: - pkgs = set() - for sourceDir in sourceDirs: - for root, _, files in os.walk(sourceDir): - if len([name for name in files if name.endswith('.java')]) != 0: - pkg = root[len(sourceDir) + 1:].replace(os.sep, '.') - if len(packages) == 0 or pkg in packages: - if len(exclude_packages) == 0 or not pkg in exclude_packages: - pkgs.add(pkg) - return pkgs - - extraArgs = [a.lstrip('@') for a in args.extra_args] - if args.argfile is not None: - extraArgs += ['@' + args.argfile] - memory = '2g' - if args.memory is not None: - memory = args.memory - memory = '-J-Xmx' + memory - - if not args.unified: - for p in projects: - # The project must be built to ensure javadoc can find class files for all referenced classes - build(['--no-native', '--projects', p.name]) - - pkgs = find_packages(p.source_dirs(), set()) - deps = p.all_deps([], includeLibs=False, includeSelf=False) - links = ['-link', 'http://docs.oracle.com/javase/' + str(p.javaCompliance.value) + '/docs/api/'] - out = outDir(p) - for d in deps: - depOut = outDir(d) - links.append('-link') - links.append(os.path.relpath(depOut, out)) - cp = classpath(p.name, includeSelf=True) - sp = os.pathsep.join(p.source_dirs()) - overviewFile = join(p.dir, 'overview.html') - delOverviewFile = False - if not exists(overviewFile): - with open(overviewFile, 'w') as fp: - print >> fp, '<html><body>Documentation for the <code>' + p.name + '</code> project.</body></html>' - delOverviewFile = True - nowarnAPI = [] - if not args.warnAPI: - nowarnAPI.append('-XDignore.symbol.file') - - # windowTitle onloy applies to the standard doclet processor - windowTitle = [] - if stdDoclet: - windowTitle = ['-windowtitle', p.name + ' javadoc'] - try: - log('Generating {2} for {0} in {1}'.format(p.name, out, docDir)) - projectJava = java(p.javaCompliance) - - # Once https://bugs.openjdk.java.net/browse/JDK-8041628 is fixed, - # this should be reverted to: - # javadocExe = java().javadoc - # we can then also respect _opts.relatex_compliance - javadocExe = projectJava.javadoc - - run([javadocExe, memory, - '-XDignore.symbol.file', - '-classpath', cp, - '-quiet', - '-d', out, - '-overview', overviewFile, - '-sourcepath', sp, - '-source', str(projectJava.javaCompliance)] + - projectJava.javadocLibOptions([]) + - ([] if projectJava.javaCompliance < JavaCompliance('1.8') else ['-Xdoclint:none']) + - links + - extraArgs + - nowarnAPI + - windowTitle + - list(pkgs)) - log('Generated {2} for {0} in {1}'.format(p.name, out, docDir)) - finally: - if delOverviewFile: - os.remove(overviewFile) - - else: - # The projects must be built to ensure javadoc can find class files for all referenced classes - build(['--no-native']) - - pkgs = set() - sp = [] - names = [] - for p in projects: - find_packages(p.source_dirs(), pkgs) - sp += p.source_dirs() - names.append(p.name) - - links = ['-link', 'http://docs.oracle.com/javase/' + str(java().javaCompliance.value) + '/docs/api/'] - out = join(_primary_suite.dir, docDir) - if args.base is not None: - out = join(args.base, docDir) - cp = classpath() - sp = os.pathsep.join(sp) - nowarnAPI = [] - if not args.warnAPI: - nowarnAPI.append('-XDignore.symbol.file') - log('Generating {2} for {0} in {1}'.format(', '.join(names), out, docDir)) - run([java().javadoc, memory, - '-classpath', cp, - '-quiet', - '-d', out, - '-sourcepath', sp] + - ([] if java().javaCompliance < JavaCompliance('1.8') else ['-Xdoclint:none']) + - links + - extraArgs + - nowarnAPI + - list(pkgs)) - log('Generated {2} for {0} in {1}'.format(', '.join(names), out, docDir)) - -def site(args): - """creates a website containing javadoc and the project dependency graph""" - - parser = ArgumentParser(prog='site') - parser.add_argument('-d', '--base', action='store', help='directory for generated site', required=True, metavar='<dir>') - parser.add_argument('--tmp', action='store', help='directory to use for intermediate results', metavar='<dir>') - parser.add_argument('--name', action='store', help='name of overall documentation', required=True, metavar='<name>') - parser.add_argument('--overview', action='store', help='path to the overview content for overall documentation', required=True, metavar='<path>') - parser.add_argument('--projects', action='store', help='comma separated projects to process (omit to process all projects)') - parser.add_argument('--jd', action='append', help='extra Javadoc arguments (e.g. --jd @-use)', metavar='@<arg>', default=[]) - parser.add_argument('--exclude-packages', action='store', help='comma separated packages to exclude', metavar='<pkgs>') - parser.add_argument('--dot-output-base', action='store', help='base file name (relative to <dir>/all) for project dependency graph .svg and .jpg files generated by dot (omit to disable dot generation)', metavar='<path>') - parser.add_argument('--title', action='store', help='value used for -windowtitle and -doctitle javadoc args for overall documentation (default: "<name>")', metavar='<title>') - args = parser.parse_args(args) - - args.base = os.path.abspath(args.base) - tmpbase = args.tmp if args.tmp else tempfile.mkdtemp(prefix=basename(args.base) + '.', dir=dirname(args.base)) - unified = join(tmpbase, 'all') - - exclude_packages_arg = [] - if args.exclude_packages is not None: - exclude_packages_arg = ['--exclude-packages', args.exclude_packages] - - projects = sorted_deps() - projects_arg = [] - if args.projects is not None: - projects_arg = ['--projects', args.projects] - projects = [project(name) for name in args.projects.split(',')] - - extra_javadoc_args = [] - for a in args.jd: - extra_javadoc_args.append('--arg') - extra_javadoc_args.append('@' + a) - - try: - # Create javadoc for each project - javadoc(['--base', tmpbase] + exclude_packages_arg + projects_arg + extra_javadoc_args) - - # Create unified javadoc for all projects - with open(args.overview) as fp: - content = fp.read() - idx = content.rfind('</body>') - if idx != -1: - args.overview = join(tmpbase, 'overview_with_projects.html') - with open(args.overview, 'w') as fp2: - print >> fp2, content[0:idx] - print >> fp2, """<div class="contentContainer"> -<table class="overviewSummary" border="0" cellpadding="3" cellspacing="0" summary="Projects table"> -<caption><span>Projects</span><span class="tabEnd"> </span></caption> -<tr><th class="colFirst" scope="col">Project</th><th class="colLast" scope="col"> </th></tr> -<tbody>""" - color = 'row' - for p in projects: - print >> fp2, '<tr class="{1}Color"><td class="colFirst"><a href="../{0}/javadoc/index.html",target = "_top">{0}</a></td><td class="colLast"> </td></tr>'.format(p.name, color) - color = 'row' if color == 'alt' else 'alt' - - print >> fp2, '</tbody></table></div>' - print >> fp2, content[idx:] - - title = args.title if args.title is not None else args.name - javadoc(['--base', tmpbase, - '--unified', - '--arg', '@-windowtitle', '--arg', '@' + title, - '--arg', '@-doctitle', '--arg', '@' + title, - '--arg', '@-overview', '--arg', '@' + args.overview] + exclude_packages_arg + projects_arg + extra_javadoc_args) - - if exists(unified): - shutil.rmtree(unified) - os.rename(join(tmpbase, 'javadoc'), unified) - - # Generate dependency graph with Graphviz - if args.dot_output_base is not None: - dotErr = None - try: - if not 'version' in subprocess.check_output(['dot', '-V'], stderr=subprocess.STDOUT): - dotErr = 'dot -V does not print a string containing "version"' - except subprocess.CalledProcessError as e: - dotErr = 'error calling "dot -V": {0}'.format(e) - except OSError as e: - dotErr = 'error calling "dot -V": {0}'.format(e) - - if dotErr != None: - abort('cannot generate dependency graph: ' + dotErr) - - dot = join(tmpbase, 'all', str(args.dot_output_base) + '.dot') - svg = join(tmpbase, 'all', str(args.dot_output_base) + '.svg') - jpg = join(tmpbase, 'all', str(args.dot_output_base) + '.jpg') - html = join(tmpbase, 'all', str(args.dot_output_base) + '.html') - with open(dot, 'w') as fp: - dim = len(projects) - print >> fp, 'digraph projects {' - print >> fp, 'rankdir=BT;' - print >> fp, 'size = "' + str(dim) + ',' + str(dim) + '";' - print >> fp, 'node [shape=rect, fontcolor="blue"];' - # print >> fp, 'edge [color="green"];' - for p in projects: - print >> fp, '"' + p.name + '" [URL = "../' + p.name + '/javadoc/index.html", target = "_top"]' - for dep in p.canonical_deps(): - if dep in [proj.name for proj in projects]: - print >> fp, '"' + p.name + '" -> "' + dep + '"' - depths = dict() - for p in projects: - d = p.max_depth() - depths.setdefault(d, list()).append(p.name) - print >> fp, '}' - - run(['dot', '-Tsvg', '-o' + svg, '-Tjpg', '-o' + jpg, dot]) - - # Post-process generated SVG to remove title elements which most browsers - # render as redundant (and annoying) tooltips. - with open(svg, 'r') as fp: - content = fp.read() - content = re.sub('<title>.*</title>', '', content) - content = re.sub('xlink:title="[^"]*"', '', content) - with open(svg, 'w') as fp: - fp.write(content) - - # Create HTML that embeds the svg file in an <object> frame - with open(html, 'w') as fp: - print >> fp, '<html><body><object data="{0}.svg" type="image/svg+xml"></object></body></html>'.format(args.dot_output_base) - - if exists(args.base): - shutil.rmtree(args.base) - if args.tmp: - shutil.copytree(tmpbase, args.base) - else: - shutil.move(tmpbase, args.base) - - print 'Created website - root is ' + join(args.base, 'all', 'index.html') - - finally: - if not args.tmp and exists(tmpbase): - shutil.rmtree(tmpbase) - -def _kwArg(kwargs): - if len(kwargs) > 0: - return kwargs.pop(0) - return None - -def findclass(args, logToConsole=True, matcher=lambda string, classname: string in classname): - """find all classes matching a given substring""" - matches = [] - for entry, filename in classpath_walk(includeBootClasspath=True): - if filename.endswith('.class'): - if isinstance(entry, zipfile.ZipFile): - classname = filename.replace('/', '.') - else: - classname = filename.replace(os.sep, '.') - classname = classname[:-len('.class')] - for a in args: - if matcher(a, classname): - matches.append(classname) - if logToConsole: - log(classname) - return matches - -def select_items(items, descriptions=None, allowMultiple=True): - """ - Presents a command line interface for selecting one or more (if allowMultiple is true) items. - - """ - if len(items) <= 1: - return items - else: - assert is_interactive() - numlen = str(len(str(len(items)))) - if allowMultiple: - log(('[{0:>' + numlen + '}] <all>').format(0)) - for i in range(0, len(items)): - if descriptions is None: - log(('[{0:>' + numlen + '}] {1}').format(i + 1, items[i])) - else: - assert len(items) == len(descriptions) - wrapper = textwrap.TextWrapper(subsequent_indent=' ') - log('\n'.join(wrapper.wrap(('[{0:>' + numlen + '}] {1} - {2}').format(i + 1, items[i], descriptions[i])))) - while True: - if allowMultiple: - s = raw_input('Enter number(s) of selection (separate multiple choices with spaces): ').split() - else: - s = [raw_input('Enter number of selection: ')] - try: - s = [int(x) for x in s] - except: - log('Selection contains non-numeric characters: "' + ' '.join(s) + '"') - continue - - if allowMultiple and 0 in s: - return items - - indexes = [] - for n in s: - if n not in range(1, len(items) + 1): - log('Invalid selection: ' + str(n)) - continue - else: - indexes.append(n - 1) - if allowMultiple: - return [items[i] for i in indexes] - if len(indexes) == 1: - return items[indexes[0]] - return None - -def exportlibs(args): - """export libraries to an archive file""" - - parser = ArgumentParser(prog='exportlibs') - parser.add_argument('-b', '--base', action='store', help='base name of archive (default: libs)', default='libs', metavar='<path>') - parser.add_argument('-a', '--include-all', action='store_true', help="include all defined libaries") - parser.add_argument('--arc', action='store', choices=['tgz', 'tbz2', 'tar', 'zip'], default='tgz', help='the type of the archive to create') - parser.add_argument('--no-sha1', action='store_false', dest='sha1', help='do not create SHA1 signature of archive') - parser.add_argument('--no-md5', action='store_false', dest='md5', help='do not create MD5 signature of archive') - parser.add_argument('--include-system-libs', action='store_true', help='include system libraries (i.e., those not downloaded from URLs)') - parser.add_argument('extras', nargs=REMAINDER, help='extra files and directories to add to archive', metavar='files...') - args = parser.parse_args(args) - - def createArchive(addMethod): - entries = {} - def add(path, arcname): - apath = os.path.abspath(path) - if not entries.has_key(arcname): - entries[arcname] = apath - logv('[adding ' + path + ']') - addMethod(path, arcname=arcname) - elif entries[arcname] != apath: - logv('[warning: ' + apath + ' collides with ' + entries[arcname] + ' as ' + arcname + ']') - else: - logv('[already added ' + path + ']') - - libsToExport = set() - if args.include_all: - for lib in _libs.itervalues(): - libsToExport.add(lib) - else: - def isValidLibrary(dep): - if dep in _libs.iterkeys(): - lib = _libs[dep] - if len(lib.urls) != 0 or args.include_system_libs: - return lib - return None - - # iterate over all project dependencies and find used libraries - for p in _projects.itervalues(): - for dep in p.deps: - r = isValidLibrary(dep) - if r: - libsToExport.add(r) - - # a library can have other libraries as dependency - size = 0 - while size != len(libsToExport): - size = len(libsToExport) - for lib in libsToExport.copy(): - for dep in lib.deps: - r = isValidLibrary(dep) - if r: - libsToExport.add(r) - - for lib in libsToExport: - add(lib.get_path(resolve=True), lib.path) - if lib.sha1: - add(lib.get_path(resolve=True) + ".sha1", lib.path + ".sha1") - if lib.sourcePath: - add(lib.get_source_path(resolve=True), lib.sourcePath) - if lib.sourceSha1: - add(lib.get_source_path(resolve=True) + ".sha1", lib.sourcePath + ".sha1") - - if args.extras: - for e in args.extras: - if os.path.isdir(e): - for root, _, filenames in os.walk(e): - for name in filenames: - f = join(root, name) - add(f, f) - else: - add(e, e) - - if args.arc == 'zip': - path = args.base + '.zip' - with zipfile.ZipFile(path, 'w') as zf: - createArchive(zf.write) - else: - path = args.base + '.tar' - mode = 'w' - if args.arc != 'tar': - sfx = args.arc[1:] - mode = mode + ':' + sfx - path = path + '.' + sfx - with tarfile.open(path, mode) as tar: - createArchive(tar.add) - log('created ' + path) - - def digest(enabled, path, factory, suffix): - if enabled: - d = factory() - with open(path, 'rb') as f: - while True: - buf = f.read(4096) - if not buf: - break - d.update(buf) - with open(path + '.' + suffix, 'w') as fp: - print >> fp, d.hexdigest() - log('created ' + path + '.' + suffix) - - digest(args.sha1, path, hashlib.sha1, 'sha1') - digest(args.md5, path, hashlib.md5, 'md5') - -def javap(args): - """disassemble classes matching given pattern with javap""" - - javapExe = java().javap - if not exists(javapExe): - abort('The javap executable does not exists: ' + javapExe) - else: - candidates = findclass(args, logToConsole=False) - if len(candidates) == 0: - log('no matches') - selection = select_items(candidates) - run([javapExe, '-private', '-verbose', '-classpath', classpath()] + selection) - -def show_projects(args): - """show all projects""" - for s in suites(): - if len(s.projects) != 0: - log(join(s.mxDir, 'suite*.py')) - for p in s.projects: - log('\t' + p.name) - -def show_suites(args): - """show all suites""" - def _show_section(name, section): - if len(section) != 0: - log(' ' + name + ':') - for e in section: - log(' ' + e.name) - - for s in suites(): - log(join(s.mxDir, 'suite*.py')) - _show_section('libraries', s.libs) - _show_section('jrelibraries', s.jreLibs) - _show_section('projects', s.projects) - _show_section('distributions', s.dists) - -def _compile_mx_class(javaClassName, classpath=None, jdk=None, myDir=None): - myDir = dirname(__file__) if myDir is None else myDir - binDir = join(myDir, 'bin' if not jdk else '.jdk' + str(jdk.version)) - javaSource = join(myDir, javaClassName + '.java') - javaClass = join(binDir, javaClassName + '.class') - if not exists(javaClass) or getmtime(javaClass) < getmtime(javaSource): - if not exists(binDir): - os.mkdir(binDir) - javac = jdk.javac if jdk else java().javac - cmd = [javac, '-d', _cygpathU2W(binDir)] - if classpath: - cmd += ['-cp', _separatedCygpathU2W(binDir + os.pathsep + classpath)] - cmd += [_cygpathU2W(javaSource)] - try: - subprocess.check_call(cmd) - except subprocess.CalledProcessError: - abort('failed to compile:' + javaSource) - - return (myDir, binDir) - -def checkcopyrights(args): - '''run copyright check on the sources''' - class CP(ArgumentParser): - def format_help(self): - return ArgumentParser.format_help(self) + self._get_program_help() - - def _get_program_help(self): - help_output = subprocess.check_output([java().java, '-cp', _cygpathU2W(binDir), 'CheckCopyright', '--help']) - return '\nother argumemnts preceded with --\n' + help_output - - myDir, binDir = _compile_mx_class('CheckCopyright') - - parser = CP(prog='mx checkcopyrights') - - parser.add_argument('--primary', action='store_true', help='limit checks to primary suite') - parser.add_argument('remainder', nargs=REMAINDER, metavar='...') - args = parser.parse_args(args) - remove_doubledash(args.remainder) - - - # ensure compiled form of code is up to date - - result = 0 - # copyright checking is suite specific as each suite may have different overrides - for s in suites(True): - if args.primary and not s.primary: - continue - custom_copyrights = _cygpathU2W(join(s.mxDir, 'copyrights')) - custom_args = [] - if exists(custom_copyrights): - custom_args = ['--custom-copyright-dir', custom_copyrights] - rc = run([java().java, '-cp', _cygpathU2W(binDir), 'CheckCopyright', '--copyright-dir', _cygpathU2W(myDir)] + custom_args + args.remainder, cwd=s.dir, nonZeroIsFatal=False) - result = result if rc == 0 else rc - return result - -def remove_doubledash(args): - if '--' in args: - args.remove('--') - -def ask_yes_no(question, default=None): - """""" - assert not default or default == 'y' or default == 'n' - if not is_interactive(): - if default: - return default - else: - abort("Can not answer '" + question + "?' if stdout is not a tty") - questionMark = '? [yn]: ' - if default: - questionMark = questionMark.replace(default, default.upper()) - answer = raw_input(question + questionMark) or default - while not answer: - answer = raw_input(question + questionMark) - return answer.lower().startswith('y') - -def add_argument(*args, **kwargs): - """ - Define how a single command-line argument. - """ - assert _argParser is not None - _argParser.add_argument(*args, **kwargs) - -def update_commands(suite, new_commands): - for key, value in new_commands.iteritems(): - if _commands.has_key(key): - warn("redefining command '" + key + "' in suite " + suite.name) - _commands[key] = value - -def warn(msg): - if _warn: - print 'WARNING: ' + msg - -# Table of commands in alphabetical order. -# Keys are command names, value are lists: [<function>, <usage msg>, <format args to doc string of function>...] -# If any of the format args are instances of Callable, then they are called with an 'env' are before being -# used in the call to str.format(). -# Suite extensions should not update this table directly, but use update_commands -_commands = { - 'about': [about, ''], - 'build': [build, '[options]'], - 'checkstyle': [checkstyle, ''], - 'canonicalizeprojects': [canonicalizeprojects, ''], - 'checkcopyrights': [checkcopyrights, '[options]'], - 'clean': [clean, ''], - 'eclipseinit': [eclipseinit, ''], - 'eclipseformat': [eclipseformat, ''], - 'exportlibs': [exportlibs, ''], - 'findclass': [findclass, ''], - 'fsckprojects': [fsckprojects, ''], - 'help': [help_, '[command]'], - 'ideclean': [ideclean, ''], - 'ideinit': [ideinit, ''], - 'intellijinit': [intellijinit, ''], - 'archive': [_archive, '[options]'], - 'projectgraph': [projectgraph, ''], - 'pylint': [pylint, ''], - 'javap': [javap, '<class name patterns>'], - 'javadoc': [javadoc, '[options]'], - 'site': [site, '[options]'], - 'netbeansinit': [netbeansinit, ''], - 'suites': [show_suites, ''], - 'projects': [show_projects, ''], -} - -_argParser = ArgParser() - -def _suitename(mxDir): - base = os.path.basename(mxDir) - parts = base.split('.') - # temporary workaround until mx.graal exists - if len(parts) == 1: - return 'truffle' - else: - return parts[1] - -def _is_suite_dir(d, mxDirName=None): - """ - Checks if d contains a suite. - If mxDirName is None, matches any suite name, otherwise checks for exactly that suite. - """ - if os.path.isdir(d): - for f in os.listdir(d): - if (mxDirName == None and (f == 'mx' or fnmatch.fnmatch(f, 'mx.*'))) or f == mxDirName: - mxDir = join(d, f) - if exists(mxDir) and isdir(mxDir) and (exists(join(mxDir, 'suite.py'))): - return mxDir - -def _check_primary_suite(): - if _primary_suite is None: - abort('no primary suite found') - else: - return _primary_suite - -def _findPrimarySuiteMxDirFrom(d): - """ search for a suite directory upwards from 'd' """ - while d: - mxDir = _is_suite_dir(d) - if mxDir is not None: - return mxDir - parent = dirname(d) - if d == parent: - return None - d = parent - - return None - -def _findPrimarySuiteMxDir(): - # check for explicit setting - if _primary_suite_path is not None: - mxDir = _is_suite_dir(_primary_suite_path) - if mxDir is not None: - return mxDir - else: - abort(_primary_suite_path + ' does not contain an mx suite') - - # try current working directory first - mxDir = _findPrimarySuiteMxDirFrom(os.getcwd()) - if mxDir is not None: - return mxDir - # backwards compatibility: search from path of this file - return _findPrimarySuiteMxDirFrom(dirname(__file__)) - -def main(): - primarySuiteMxDir = _findPrimarySuiteMxDir() - if primarySuiteMxDir: - global _primary_suite - _primary_suite = _loadSuite(primarySuiteMxDir, True) - else: - abort('no primary suite found') - - opts, commandAndArgs = _argParser._parse_cmd_line() - assert _opts == opts - - for s in suites(): - s._post_init(opts) - - if len(commandAndArgs) == 0: - _argParser.print_help() - return - - command = commandAndArgs[0] - command_args = commandAndArgs[1:] - - if not _commands.has_key(command): - hits = [c for c in _commands.iterkeys() if c.startswith(command)] - if len(hits) == 1: - command = hits[0] - elif len(hits) == 0: - abort('mx: unknown command \'{0}\'\n{1}use "mx help" for more options'.format(command, _format_commands())) - else: - abort('mx: command \'{0}\' is ambiguous\n {1}'.format(command, ' '.join(hits))) - - c, _ = _commands[command][:2] - def term_handler(signum, frame): - abort(1) - if not is_jython(): - signal.signal(signal.SIGTERM, term_handler) - - def quit_handler(signum, frame): - _send_sigquit() - if not is_jython() and get_os() != 'windows': - signal.signal(signal.SIGQUIT, quit_handler) - - try: - if opts.timeout != 0: - def alarm_handler(signum, frame): - abort('Command timed out after ' + str(opts.timeout) + ' seconds: ' + ' '.join(commandAndArgs)) - signal.signal(signal.SIGALRM, alarm_handler) - signal.alarm(opts.timeout) - retcode = c(command_args) - if retcode is not None and retcode != 0: - abort(retcode) - except KeyboardInterrupt: - # no need to show the stack trace when the user presses CTRL-C - abort(1) - -version = VersionSpec("1.0") - -currentUmask = None - -if __name__ == '__main__': - # rename this module as 'mx' so it is not imported twice by the commands.py modules - sys.modules['mx'] = sys.modules.pop('__main__') - - # Capture the current umask since there's no way to query it without mutating it. - currentUmask = os.umask(0) - os.umask(currentUmask) - - main()