truffle: mxtool/mx.py comparison

comparison mxtool/mx.py @ 12387:aff825fef0fd

Merge.

author	Christian Humer <christian.humer@gmail.com>
date	Wed, 02 Oct 2013 13:26:31 +0200
parents	039b133ded75
children	fbe1ee508936

comparison

equal deleted inserted replaced

-:91dbb0b7dc8b
+:aff825fef0fd
 The organizing principle of mx is a project suite. A suite is a directory
 containing one or more projects. It's not coincidental that this closely
 matches the layout of one or more projects in a Mercurial repository.
 The configuration information for a suite lives in an 'mx' sub-directory
-at the top level of the suite.
+at the top level of the suite. A suite is given a name by a 'suite=name'
+property in the 'mx/projects' file (if omitted the name is suite directory).
+An 'mx' subdirectory can be named as plain 'mx' or 'mx.name', where
+'name' is typically the name as the suite name.
+The latter is useful to avoid clashes in IDE project names.
 When launched, mx treats the current working directory as a suite.
 This is the primary suite. All other suites are called included suites.
 The configuration files (i.e. in the 'mx' sub-directory) of a suite are:
 by extension commands.
 Property values can use environment variables with Bash syntax (e.g. ${HOME}).
 """
-import sys, os, errno, time, subprocess, shlex, types, urllib2, contextlib, StringIO, zipfile, signal, xml.sax.saxutils, tempfile
+import sys, os, errno, time, subprocess, shlex, types, urllib2, contextlib, StringIO, zipfile, signal, xml.sax.saxutils, tempfile, fnmatch
 import textwrap
 import xml.parsers.expat
 import shutil, re, xml.dom.minidom
 from collections import Callable
 from threading import Thread
 _suites = dict()
 _annotationProcessors = None
 _mainSuite = None
 _opts = None
 _java = None
+_check_global_structures = True # can be set False to allow suites with duplicate definitions to load without aborting
 """
 A distribution is a jar or zip file containing the output from one or more Java projects.
 """
 class Distribution:
 self.sourceUrls = sourceUrls
 for url in urls:
 if url.endswith('/') != self.path.endswith(os.sep):
 abort('Path for dependency directory must have a URL ending with "/": path=' + self.path + ' url=' + url)
+def __eq__(self, other):
+if isinstance(other, Library):
+if len(self.urls) == 0:
+return self.path == other.path
+else:
+return self.urls == other.urls
+else:
+return NotImplemented
+def __ne__(self, other):
+result = self.__eq__(other)
+if result is NotImplemented:
+return result
+return not result
 def get_path(self, resolve):
 path = self.path
 if not isabs(path):
 path = join(self.suite.dir, path)
+includedInJDK = getattr(self, 'includedInJDK', None)
+if includedInJDK and java().javaCompliance >= JavaCompliance(includedInJDK):
+return None
 if resolve and self.mustExist and not exists(path):
 assert not len(self.urls) == 0, 'cannot find required library ' + self.name + ' ' + path
 print('Downloading ' + self.name + ' from ' + str(self.urls))
 download(path, self.urls)
 return path
 download(path, self.sourceUrls)
 return path
 def append_to_classpath(self, cp, resolve):
 path = self.get_path(resolve)
-if exists(path) or not resolve:
+if path and (exists(path) or not resolve):
 cp.append(path)
 def all_deps(self, deps, includeLibs, includeSelf=True, includeAnnotationProcessors=False):
 if not includeLibs or not includeSelf:
 return deps
 deps.append(self)
 return deps
 class Suite:
-def __init__(self, d, primary):
+def __init__(self, d, mxDir, primary):
 self.dir = d
+self.mxDir = mxDir
 self.projects = []
 self.libs = []
 self.dists = []
 self.includes = []
 self.commands = None
 self.primary = primary
-mxDir = join(d, 'mx')
 self._load_env(mxDir)
 self._load_commands(mxDir)
 self._load_includes(mxDir)
 self.name = d  # re-initialized in _load_projects
 projsMap = dict()
 distsMap = dict()
 projectsFile = join(mxDir, 'projects')
 if not exists(projectsFile):
 return
+def _find_suite_key():
+for items in _suites.items():
+if items[1].dir == self.dir:
+return items[0]
+raise KeyError
 with open(projectsFile) as f:
 for line in f:
 line = line.strip()
 if len(line) != 0 and line[0] != '#':
 key, value = line.split('=', 1)
 parts = key.split('@')
 if len(parts) == 1:
 if parts[0] != 'suite':
 abort('Single part property must be "suite": ' + key)
-self.name = value
+if self.name != value:
+currentKey = _find_suite_key()
+_suites.pop(currentKey)
+self.name = value
+_suites[value] = self
 continue
 if len(parts) != 3:
 abort('Property name does not have 3 parts separated by "@": ' + key)
 kind, name, attr = parts
 if kind == 'project':
 if not hasattr(mod, 'mx_init'):
 abort(commandsPath + ' must define an mx_init(env) function')
 if hasattr(mod, 'mx_post_parse_cmd_line'):
 self.mx_post_parse_cmd_line = mod.mx_post_parse_cmd_line
-mod.mx_init()
+mod.mx_init(self)
 self.commands = mod
 def _load_includes(self, mxDir):
 includes = join(mxDir, 'includes')
 if exists(includes):
 with open(includes) as f:
 for line in f:
 include = expandvars_in_property(line.strip())
 self.includes.append(include)
-_loadSuite(include, False)
+_loadSuite(os.path.abspath(include), False)
 def _load_env(self, mxDir):
 e = join(mxDir, 'env')
 if exists(e):
 with open(e) as f:
 if len(line) != 0 and line[0] != '#':
 if not '=' in line:
 abort(e + ':' + str(lineNum) + ': line does not match pattern "key=value"')
 key, value = line.split('=', 1)
 os.environ[key.strip()] = expandvars_in_property(value.strip())
 def _post_init(self, opts):
-mxDir = join(self.dir, 'mx')
+self._load_projects(self.mxDir)
-self._load_projects(mxDir)
+# set the global data structures, checking for conflicts unless _global_structures is False
 for p in self.projects:
 existing = _projects.get(p.name)
-if existing is not None:
+if existing is not None and _check_global_structures:
 abort('cannot override project  ' + p.name + ' in ' + p.dir + " with project of the same name in  " + existing.dir)
 if not p.name in _opts.ignored_projects:
 _projects[p.name] = p
 for l in self.libs:
 existing = _libs.get(l.name)
-if existing is not None:
+# Check that suites that define same library are consistent
-abort('cannot redefine library  ' + l.name)
+if existing is not None and existing != l and _check_global_structures:
+abort('inconsistent library redefinition of ' + l.name + ' in ' + existing.suite.dir + ' and ' + l.suite.dir)
 _libs[l.name] = l
 for d in self.dists:
 existing = _dists.get(d.name)
-if existing is not None:
+if existing is not None and _check_global_structures:
-abort('cannot redefine distribution  ' + d.name)
+# allow redefinition, so use path from existing
+# abort('cannot redefine distribution  ' + d.name)
+print('WARNING: distribution ' + d.name + ' redefined')
+d.path = existing.path
 _dists[d.name] = d
 if hasattr(self, 'mx_post_parse_cmd_line'):
 self.mx_post_parse_cmd_line(opts)
 class XMLElement(xml.dom.minidom.Element):
 return 'windows'
 else:
 abort('Unknown operating system ' + sys.platform)
 def _loadSuite(d, primary=False):
-mxDir = join(d, 'mx')
+"""
-if not exists(mxDir) or not isdir(mxDir):
+Load a suite from the 'mx' or 'mx.bbb' subdirectory of d, where 'bbb' is basename of d
+"""
+mxDefaultDir = join(d, 'mx')
+name = os.path.basename(d)
+mxTaggedDir = mxDefaultDir + '.' + name
+mxDir = None
+if exists(mxTaggedDir) and isdir(mxTaggedDir):
+mxDir = mxTaggedDir
+else:
+if exists(mxDefaultDir) and isdir(mxDefaultDir):
+mxDir = mxDefaultDir
+if mxDir is None:
 return None
 if len([s for s in _suites.itervalues() if s.dir == d]) == 0:
-s = Suite(d, primary)
+s = Suite(d, mxDir, primary)
-_suites[s.name] = s
+# N.B. this will be updated once the projects file has been read
+_suites[name] = s
 return s
-def suites():
+def suites(opt_limit_to_suite=False):
 """
 Get the list of all loaded suites.
 """
-return _suites.values()
+if opt_limit_to_suite and _opts.specific_suites:
+result = []
+for s in _suites.values():
+if s.name in _opts.specific_suites:
+result.append(s)
+return result
+else:
+return _suites.values()
 def suite(name, fatalIfMissing=True):
 """
 Get the suite for a given name.
 """
 s = _suites.get(name)
 if s is None and fatalIfMissing:
 abort('suite named ' + name + ' not found')
 return s
-def projects():
+def projects_from_names(projectNames):
 """
-Get the list of all loaded projects.
+Get the list of projects corresponding to projectNames; all projects if None
 """
-return _projects.values()
+if projectNames is None:
+return projects()
+else:
+return [project(name) for name in projectNames]
+def projects(opt_limit_to_suite=False):
+"""
+Get the list of all loaded projects limited by --suite option if opt_limit_to_suite == True
+"""
+if opt_limit_to_suite:
+return _projects_opt_limit_to_suites(_projects.values())
+else:
+return _projects.values()
+def projects_opt_limit_to_suites():
+"""
+Get the list of all loaded projects optionally limited by --suite option
+"""
+return projects(True)
+def _projects_opt_limit_to_suites(projects):
+if not _opts.specific_suites:
+return projects
+else:
+result = []
+for p in projects:
+s = p.suite
+if s.name in _opts.specific_suites:
+result.append(p)
+return result
 def annotation_processors():
 """
 Get the list of all loaded projects that define an annotation processor.
 """
 """
 Gets projects and libraries sorted such that dependencies
 are before the projects that depend on them. Unless 'includeLibs' is
 true, libraries are omitted from the result.
 """
+projects = projects_from_names(projectNames)
+return sorted_project_deps(projects, includeLibs=includeLibs, includeAnnotationProcessors=includeAnnotationProcessors)
+def sorted_project_deps(projects, includeLibs=False, includeAnnotationProcessors=False):
 deps = []
-if projectNames is None:
-projects = _projects.values()
-else:
-projects = [project(name) for name in projectNames]
 for p in projects:
 p.all_deps(deps, includeLibs=includeLibs, includeAnnotationProcessors=includeAnnotationProcessors)
 return deps
 def _handle_missing_java_home():
 return ArgumentParser.format_help(self) + _format_commands()
 def __init__(self):
 self.java_initialized = False
-ArgumentParser.__init__(self, prog='mx')
+# this doesn't resolve the right way, but can't figure out how to override _handle_conflict_resolve in _ActionsContainer
+ArgumentParser.__init__(self, prog='mx', conflict_handler='resolve')
 self.add_argument('-v', action='store_true', dest='verbose', help='enable verbose output')
 self.add_argument('-V', action='store_true', dest='very_verbose', help='enable very verbose output')
 self.add_argument('--dbg', type=int, dest='java_dbg_port', help='make Java processes wait on <port> for a debugger', metavar='<port>')
 self.add_argument('-d', action='store_const', const=8000, dest='java_dbg_port', help='alias for "-dbg 8000"')
 self.add_argument('--Jp', action='append', dest='java_args_pfx', help='prefix Java VM arguments (e.g. --Jp @-dsa)', metavar='@<args>', default=[])
 self.add_argument('--Ja', action='append', dest='java_args_sfx', help='suffix Java VM arguments (e.g. --Ja @-dsa)', metavar='@<args>', default=[])
 self.add_argument('--user-home', help='users home directory', metavar='<path>', default=os.path.expanduser('~'))
 self.add_argument('--java-home', help='bootstrap JDK installation directory (must be JDK 6 or later)', metavar='<path>')
 self.add_argument('--ignore-project', action='append', dest='ignored_projects', help='name of project to ignore', metavar='<name>', default=[])
+self.add_argument('--suite', action='append', dest='specific_suites', help='limit command to given suite', default=[])
 if get_os() != 'windows':
 # Time outs are (currently) implemented with Unix specific functionality
 self.add_argument('--timeout', help='timeout (in seconds) for command', type=int, default=0, metavar='<secs>')
 self.add_argument('--ptimeout', help='timeout (in seconds) for subprocesses', type=int, default=0, metavar='<secs>')
 opts.ignored_projects = opts.ignored_projects + os.environ.get('IGNORED_PROJECTS', '').split(',')
 commandAndArgs = opts.__dict__.pop('commandAndArgs')
 return opts, commandAndArgs
+def _handle_conflict_resolve(self, action, conflicting_actions):
+self._handle_conflict_error(action, conflicting_actions)
 def _format_commands():
 msg = '\navailable commands:\n\n'
 for cmd in sorted(_commands.iterkeys()):
 c, _ = _commands[cmd][:2]
 # Silently ignore JDT if default location is used but not ecj.jar exists there
 jdtJar = None
 built = set()
-projects = None
-if args.projects is not None:
-projects = args.projects.split(',')
 if args.only is not None:
+# N.B. This build will not include dependencies including annotation processor dependencies
 sortedProjects = [project(name) for name in args.only.split(',')]
 else:
-sortedProjects = sorted_deps(projects, includeAnnotationProcessors=True)
+if args.projects is not None:
+projectNames = args.projects.split(',')
+else:
+projectNames = None
+projects = _projects_opt_limit_to_suites(projects_from_names(projectNames))
+# N.B. Limiting to a suite only affects the starting set of projects. Dependencies in other suites will still be compiled
+sortedProjects = sorted_project_deps(projects, includeAnnotationProcessors=True)
+if args.java:
+ideinit([], refreshOnly=True, buildProcessorJars=False)
 for p in sortedProjects:
 if p.native:
 if args.native:
 log('Calling GNU make {0}...'.format(p.dir))
 pnames = [p.name for p in projs]
 build(['--projects', ",".join(pnames)])
 archive(pnames)
+def pylint(args):
+"""run pylint (if available) over Python source files (found by 'hg locate' or by tree walk with -walk)"""
+parser = ArgumentParser(prog='mx pylint')
+parser.add_argument('--walk', action='store_true', help='use tree walk find .py files')
+args = parser.parse_args(args)
+rcfile = join(dirname(__file__), '.pylintrc')
+if not exists(rcfile):
+log('pylint configuration file does not exist: ' + rcfile)
+return
+try:
+output = subprocess.check_output(['pylint', '--version'], stderr=subprocess.STDOUT)
+m = re.match(r'.*pylint (\d+)\.(\d+)\.(\d+).*', output, re.DOTALL)
+if not m:
+log('could not determine pylint version from ' + output)
+return
+major, minor, micro = (int(m.group(1)), int(m.group(2)), int(m.group(3)))
+if major < 1:
+log('require pylint version >= 1 (got {0}.{1}.{2})'.format(major, minor, micro))
+return
+except BaseException:
+log('pylint is not available')
+return
+def findfiles_by_walk():
+result = []
+for suite in suites(True):
+for root, dirs, files in os.walk(suite.dir):
+for f in files:
+if f.endswith('.py'):
+pyfile = join(root, f)
+result.append(pyfile)
+if 'bin' in dirs:
+dirs.remove('bin')
+if 'lib' in dirs:
+# avoids downloaded .py files
+dirs.remove('lib')
+return result
+def findfiles_by_hg():
+result = []
+for suite in suites(True):
+versioned = subprocess.check_output(['hg', 'locate', '-f'], stderr=subprocess.STDOUT, cwd=suite.dir).split(os.linesep)
+for f in versioned:
+if f.endswith('.py') and exists(f):
+result.append(f)
+return result
+# Perhaps we should just look in suite.mxDir directories for .py files?
+if args.walk:
+pyfiles = findfiles_by_walk()
+else:
+pyfiles = findfiles_by_hg()
+env = os.environ.copy()
+pythonpath = dirname(__file__)
+for suite in suites(True):
+pythonpath = os.pathsep.join([pythonpath, suite.mxDir])
+env['PYTHONPATH'] = pythonpath
+for pyfile in pyfiles:
+log('Running pylint on ' + pyfile + '...')
+run(['pylint', '--reports=n', '--rcfile=' + rcfile, pyfile], env=env, nonZeroIsFatal=False)
 def archive(args):
 """create jar files for projects and distributions"""
 parser = ArgumentParser(prog='mx archive')
 parser.add_argument('names', nargs=REMAINDER, metavar='[<project>|@<distribution>]...')
 args = parser.parse_args(args)
 if dep.isLibrary():
 l = dep
 # merge library jar into distribution jar
 logv('[' + d.path + ': adding library ' + l.name + ']')
 lpath = l.get_path(resolve=True)
-with zipfile.ZipFile(lpath, 'r') as lp:
+if lpath:
-for arcname in lp.namelist():
+with zipfile.ZipFile(lpath, 'r') as lp:
-if arcname.startswith('META-INF/services/'):
+for arcname in lp.namelist():
-f = arcname[len('META-INF/services/'):].replace('/', os.sep)
+if arcname.startswith('META-INF/services/') and not arcname == 'META-INF/services/':
-with open(join(services, f), 'a') as outfile:
+f = arcname[len('META-INF/services/'):].replace('/', os.sep)
-for line in lp.read(arcname).splitlines():
+with open(join(services, f), 'a') as outfile:
-outfile.write(line)
+for line in lp.read(arcname).splitlines():
-else:
+outfile.write(line)
-overwriteCheck(zf, arcname, lpath + '!' + arcname)
+else:
-zf.writestr(arcname, lp.read(arcname))
+overwriteCheck(zf, arcname, lpath + '!' + arcname)
+zf.writestr(arcname, lp.read(arcname))
 else:
 p = dep
 # skip a  Java project if its Java compliance level is "higher" than the configured JDK
 if java().javaCompliance < p.javaCompliance:
 log('Excluding {0} from {2} (Java compliance level {1} required)'.format(p.name, p.javaCompliance, d.path))
 The exit code of this command reflects how many files were updated."""
 changedFiles = 0
 for s in suites():
-projectsFile = join(s.dir, 'mx', 'projects')
+projectsFile = join(s.mxDir, 'projects')
 if not exists(projectsFile):
 continue
 with open(projectsFile) as f:
 out = StringIO.StringIO()
 pattern = re.compile('project@([^@]+)@dependencies=.*')
 content = out.getvalue()
 if update_file(projectsFile, content):
 changedFiles += 1
 return changedFiles
+class TimeStampFile:
+def __init__(self, path):
+self.path = path
+self.timestamp = os.path.getmtime(path) if exists(path) else None
+def outOfDate(self, arg):
+if not self.timestamp:
+return True
+if isinstance(arg, types.ListType):
+files = arg
+else:
+files = [arg]
+for f in files:
+if os.path.getmtime(f) > self.timestamp:
+return True
+return False
+def exists(self):
+return exists(self.path)
+def touch(self):
+if exists(self.path):
+os.utime(self.path, None)
+else:
+if not isdir(dirname(self.path)):
+os.makedirs(dirname(self.path))
+file(self.path, 'a')
 def checkstyle(args):
 """run Checkstyle on the Java sources
 Run Checkstyle over the Java sources. Any errors or warnings
 produced by Checkstyle result in a non-zero exit code."""
 parser.add_argument('-f', action='store_true', dest='force', help='force checking (disables timestamp checking)')
 args = parser.parse_args(args)
 totalErrors = 0
-for p in sorted_deps():
+for p in projects_opt_limit_to_suites():
 if p.native:
 continue
 sourceDirs = p.source_dirs()
 dotCheckstyle = join(p.dir, '.checkstyle')
 javafilelist += [join(root, name) for name in files if name.endswith('.java') and name != 'package-info.java']
 if len(javafilelist) == 0:
 logv('[no Java sources in {0} - skipping]'.format(sourceDir))
 continue
-timestampFile = join(p.suite.dir, 'mx', 'checkstyle-timestamps', sourceDir[len(p.suite.dir) + 1:].replace(os.sep, '_') + '.timestamp')
+timestamp = TimeStampFile(join(p.suite.mxDir, 'checkstyle-timestamps', sourceDir[len(p.suite.dir) + 1:].replace(os.sep, '_') + '.timestamp'))
-if not exists(dirname(timestampFile)):
-os.makedirs(dirname(timestampFile))
 mustCheck = False
-if not args.force and exists(timestampFile):
+if not args.force and timestamp.exists():
-timestamp = os.path.getmtime(timestampFile)
+mustCheck = timestamp.outOfDate(javafilelist)
-for f in javafilelist:
-if os.path.getmtime(f) > timestamp:
-mustCheck = True
-break
 else:
 mustCheck = True
 if not mustCheck:
 if _opts.verbose:
 if name == 'file':
 source[0] = attrs['name']
 elif name == 'error':
 errors.append('{}:{}: {}'.format(source[0], attrs['line'], attrs['message']))
-p = xml.parsers.expat.ParserCreate()
+xp = xml.parsers.expat.ParserCreate()
-p.StartElementHandler = start_element
+xp.StartElementHandler = start_element
 with open(auditfileName) as fp:
-p.ParseFile(fp)
+xp.ParseFile(fp)
 if len(errors) != 0:
 map(log, errors)
 totalErrors = totalErrors + len(errors)
 else:
-if exists(timestampFile):
+timestamp.touch()
-os.utime(timestampFile, None)
-else:
-file(timestampFile, 'a')
 finally:
 if exists(auditfileName):
 os.unlink(auditfileName)
 return totalErrors
 parser.add_argument('--no-native', action='store_false', dest='native', help='do not clean native projects')
 parser.add_argument('--no-java', action='store_false', dest='java', help='do not clean Java projects')
 args = parser.parse_args(args)
-for p in projects():
+for p in projects_opt_limit_to_suites():
 if p.native:
 if args.native:
 run([gmake_cmd(), '-C', p.dir, 'clean'])
 else:
 if args.java:
 slm.close('sourceContainers')
 slm.close('sourceLookupDirector')
 return slm
-def make_eclipse_attach(hostname, port, name=None, deps=None):
+def make_eclipse_attach(suite, hostname, port, name=None, deps=None):
 """
 Creates an Eclipse launch configuration file for attaching to a Java process.
 """
 if deps is None:
 deps = []
 launch.element('stringAttribute', {'key' : 'org.eclipse.jdt.launching.VM_CONNECTOR_ID', 'value' : 'org.eclipse.jdt.launching.socketAttachConnector'})
 launch.close('launchConfiguration')
 launch = launch.xml(newl='\n', standalone='no') % slm.xml(escape=True, standalone='no')
 if name is None:
-name = 'attach-' + hostname + '-' + port
+if len(suites()) == 1:
-eclipseLaunches = join('mx', 'eclipse-launches')
+suitePrefix = ''
+else:
+suitePrefix = suite.name + '-'
+name = suitePrefix + 'attach-' + hostname + '-' + port
+eclipseLaunches = join(suite.mxDir, 'eclipse-launches')
 if not exists(eclipseLaunches):
 os.makedirs(eclipseLaunches)
 return update_file(join(eclipseLaunches, name + '.launch'), launch)
 def make_eclipse_launch(javaArgs, jre, name=None, deps=None):
 eclipseLaunches = join('mx', 'eclipse-launches')
 if not exists(eclipseLaunches):
 os.makedirs(eclipseLaunches)
 return update_file(join(eclipseLaunches, name + '.launch'), launch)
-def eclipseinit(args, suite=None, buildProcessorJars=True):
+def eclipseinit(args, buildProcessorJars=True, refreshOnly=False):
 """(re)generate Eclipse project configurations and working sets"""
+for s in suites(True):
-if suite is None:
+_eclipseinit_suite(args, s, buildProcessorJars, refreshOnly)
-suite = _mainSuite
+generate_eclipse_workingsets()
+def _eclipseinit_suite(args, suite, buildProcessorJars=True, refreshOnly=False):
+projectsFile = join(suite.mxDir, 'projects')
+timestamp = TimeStampFile(join(suite.mxDir, 'eclipseinit.timestamp'))
+if refreshOnly and not timestamp.exists():
+return
+if not timestamp.outOfDate(projectsFile):
+logv('[Eclipse configurations are up to date - skipping]')
+return
 if buildProcessorJars:
+## todo suite specific
 processorjars()
 projToDist = dict()
 for dist in _dists.values():
 distDeps = sorted_deps(dist.deps)
 for p in distDeps:
 projToDist[p.name] = (dist, [dep.name for dep in distDeps])
-for p in projects():
+for p in suite.projects:
 if p.native:
 continue
 if not exists(p.dir):
 os.makedirs(p.dir)
 out.element('classpathentry', {'exported' : 'true', 'kind' : 'con', 'path' : getattr(dep, 'eclipse.container')})
 elif hasattr(dep, 'eclipse.project'):
 out.element('classpathentry', {'combineaccessrules' : 'false', 'exported' : 'true', 'kind' : 'src', 'path' : '/' + getattr(dep, 'eclipse.project')})
 else:
 path = dep.path
-if dep.mustExist:
+dep.get_path(resolve=True)
-dep.get_path(resolve=True)
+if not path or (not exists(path) and not dep.mustExist):
-if not isabs(path):
+continue
-# Relative paths for "lib" class path entries have various semantics depending on the Eclipse
-# version being used (e.g. see https://bugs.eclipse.org/bugs/show_bug.cgi?id=274737) so it's
+if not isabs(path):
-# safest to simply use absolute paths.
+# Relative paths for "lib" class path entries have various semantics depending on the Eclipse
-path = join(suite.dir, path)
+# version being used (e.g. see https://bugs.eclipse.org/bugs/show_bug.cgi?id=274737) so it's
+# safest to simply use absolute paths.
-attributes = {'exported' : 'true', 'kind' : 'lib', 'path' : path}
+path = join(p.suite.dir, path)
-sourcePath = dep.get_source_path(resolve=True)
+attributes = {'exported' : 'true', 'kind' : 'lib', 'path' : path}
-if sourcePath is not None:
-attributes['sourcepath'] = sourcePath
+sourcePath = dep.get_source_path(resolve=True)
-out.element('classpathentry', attributes)
+if sourcePath is not None:
+attributes['sourcepath'] = sourcePath
+out.element('classpathentry', attributes)
 else:
 out.element('classpathentry', {'combineaccessrules' : 'false', 'exported' : 'true', 'kind' : 'src', 'path' : '/' + dep.name})
 out.element('classpathentry', {'kind' : 'output', 'path' : getattr(p, 'eclipse.output', 'bin')})
 out.close('classpath')
 settingsDir = join(p.dir, ".settings")
 if not exists(settingsDir):
 os.mkdir(settingsDir)
-eclipseSettingsDir = join(suite.dir, 'mx', 'eclipse-settings')
+eclipseSettingsDir = join(p.suite.mxDir, 'eclipse-settings')
 if exists(eclipseSettingsDir):
 for name in os.listdir(eclipseSettingsDir):
 if name == "org.eclipse.jdt.apt.core.prefs" and not len(p.annotation_processors()) > 0:
 continue
 path = join(eclipseSettingsDir, name)
 for dep in dependency(ap).all_deps([], True):
 if dep.isLibrary():
 if not hasattr(dep, 'eclipse.container') and not hasattr(dep, 'eclipse.project'):
 if dep.mustExist:
 path = dep.get_path(resolve=True)
-if not isabs(path):
+if path:
-# Relative paths for "lib" class path entries have various semantics depending on the Eclipse
+if not isabs(path):
-# version being used (e.g. see https://bugs.eclipse.org/bugs/show_bug.cgi?id=274737) so it's
+# Relative paths for "lib" class path entries have various semantics depending on the Eclipse
-# safest to simply use absolute paths.
+# version being used (e.g. see https://bugs.eclipse.org/bugs/show_bug.cgi?id=274737) so it's
-path = join(suite.dir, path)
+# safest to simply use absolute paths.
-out.element('factorypathentry', {'kind' : 'EXTJAR', 'id' : path, 'enabled' : 'true', 'runInBatchMode' : 'false'})
+path = join(p.suite.dir, path)
+out.element('factorypathentry', {'kind' : 'EXTJAR', 'id' : path, 'enabled' : 'true', 'runInBatchMode' : 'false'})
 else:
 out.element('factorypathentry', {'kind' : 'WKSPJAR', 'id' : '/' + dep.name + '/' + dep.name + '.jar', 'enabled' : 'true', 'runInBatchMode' : 'false'})
 out.close('factorypath')
 update_file(join(p.dir, '.factorypath'), out.xml(indent='\t', newl='\n'))
-make_eclipse_attach('localhost', '8000', deps=projects())
+make_eclipse_attach(suite, 'localhost', '8000', deps=projects())
-generate_eclipse_workingsets(suite)
+timestamp.touch()
 def _isAnnotationProcessorDependency(p):
 """
 Determines if a given project is part of an annotation processor.
 """
 dotProjectDoc.element('value', data='true')
 dotProjectDoc.close('dictionary')
 dotProjectDoc.close('arguments')
 dotProjectDoc.close('buildCommand')
-def generate_eclipse_workingsets(suite):
+def generate_eclipse_workingsets():
 """
-Populate the workspace's working set configuration with working sets generated from project data.
+Populate the workspace's working set configuration with working sets generated from project data for the primary suite
 If the workspace already contains working set definitions, the existing ones will be retained and extended.
-In case mx/env does not contain a WORKSPACE definition pointing to the workspace root directory, the Graal project root directory will be assumed.
+In case mx/env does not contain a WORKSPACE definition pointing to the workspace root directory, a parent search from the primary suite directory is performed.
-If no workspace root directory can be identified, the Graal project root directory is used and the user has to place the workingsets.xml file by hand.
+If no workspace root directory can be identified, the primary suite directory is used and the user has to place the workingsets.xml file by hand.
 """
 # identify the location where to look for workingsets.xml
 wsfilename = 'workingsets.xml'
-wsroot = suite.dir
+wsloc = '.metadata/.plugins/org.eclipse.ui.workbench'
 if os.environ.has_key('WORKSPACE'):
-wsroot = os.environ['WORKSPACE']
+expected_wsroot = os.environ['WORKSPACE']
-wsdir = join(wsroot, '.metadata/.plugins/org.eclipse.ui.workbench')
+else:
+expected_wsroot = _mainSuite.dir
+wsroot = _find_eclipse_wsroot(expected_wsroot)
+if wsroot is None:
+# failed to find it
+wsroot = expected_wsroot
+wsdir = join(wsroot, wsloc)
 if not exists(wsdir):
 wsdir = wsroot
+log('Could not find Eclipse metadata directory. Please place ' + wsfilename + ' in ' + wsloc + ' manually.')
 wspath = join(wsdir, wsfilename)
 # gather working set info from project data
 workingSets = dict()
 for p in projects():
 wsdoc = _copy_workingset_xml(wspath, workingSets)
 else:
 wsdoc = _make_workingset_xml(workingSets)
 update_file(wspath, wsdoc.xml(newl='\n'))
+def _find_eclipse_wsroot(wsdir):
+md = join(wsdir, '.metadata')
+if exists(md):
+return wsdir
+split = os.path.split(wsdir)
+if split[0] == wsdir: # root directory
+return None
+else:
+return _find_eclipse_wsroot(split[0])
+def _foobar(val):
+print(val)
 def _make_workingset_xml(workingSets):
 wsdoc = XMLDoc()
 wsdoc.open('workingSetManager')
 wsdoc.open('workingSet', {'editPageID': 'org.eclipse.jdt.ui.JavaWorkingSetPage', 'factoryID': 'org.eclipse.ui.internal.WorkingSetFactory', 'id': 'wsid_' + ws, 'label': ws, 'name': ws})
 def _workingset_element(wsdoc, p):
 wsdoc.element('item', {'elementID': '=' + p, 'factoryID': 'org.eclipse.jdt.ui.PersistableJavaElementFactory'})
-def netbeansinit(args, suite=None):
+def netbeansinit(args, refreshOnly=False, buildProcessorJars=True):
 """(re)generate NetBeans project configurations"""
-if suite is None:
+for suite in suites(True):
-suite = _mainSuite
+_netbeansinit_suite(args, suite, refreshOnly, buildProcessorJars)
+def _netbeansinit_suite(args, suite, refreshOnly=False, buildProcessorJars=True):
+projectsFile = join(suite.mxDir, 'projects')
+timestamp = TimeStampFile(join(suite.mxDir, 'netbeansinit.timestamp'))
+if refreshOnly and not timestamp.exists():
+return
+if not timestamp.outOfDate(projectsFile):
+logv('[NetBeans configurations are up to date - skipping]')
+return
 updated = False
-for p in projects():
+for p in suite.projects:
 if p.native:
 continue
 if exists(join(p.dir, 'plugin.xml')):  # eclipse plugin project
 continue
 if dep.isLibrary():
 if not dep.mustExist:
 continue
 path = dep.get_path(resolve=True)
-if os.sep == '\\':
+if path:
-path = path.replace('\\', '\\\\')
+if os.sep == '\\':
-ref = 'file.reference.' + dep.name + '-bin'
+path = path.replace('\\', '\\\\')
-print >> out, ref + '=' + path
+ref = 'file.reference.' + dep.name + '-bin'
+print >> out, ref + '=' + path
 else:
 n = dep.name.replace('.', '_')
 relDepPath = os.path.relpath(dep.dir, p.dir).replace(os.sep, '/')
 ref = 'reference.' + n + '.jar'
 if updated:
 log('If using NetBeans:')
 log('  1. Ensure that a platform named "JDK_' + str(java().version) + '" is defined (Tools -> Java Platforms)')
 log('  2. Open/create a Project Group for the directory containing the projects (File -> Project Group -> New Group... -> Folder of Projects)')
-def ideclean(args, suite=None):
+timestamp.touch()
+def ideclean(args):
 """remove all Eclipse and NetBeans project configurations"""
 def rm(path):
 if exists(path):
 os.remove(path)
+for s in suites():
+rm(join(s.mxDir, 'eclipseinit.timestamp'))
+rm(join(s.mxDir, 'netbeansinit.timestamp'))
 for p in projects():
 if p.native:
 continue
 rm(join(p.dir, p.name + '.jar'))
 except:
 log("Error removing {0}".format(p.name + '.jar'))
-def ideinit(args, suite=None):
+def ideinit(args, refreshOnly=False, buildProcessorJars=True):
 """(re)generate Eclipse and NetBeans project configurations"""
-eclipseinit(args, suite)
+eclipseinit(args, refreshOnly=refreshOnly, buildProcessorJars=buildProcessorJars)
-netbeansinit(args, suite)
+netbeansinit(args, refreshOnly=refreshOnly, buildProcessorJars=buildProcessorJars)
-fsckprojects([])
+if not refreshOnly:
+fsckprojects([])
 def fsckprojects(args):
 """find directories corresponding to deleted Java projects and delete them"""
-for suite in suites():
+for suite in suites(True):
 projectDirs = [p.dir for p in suite.projects]
 for root, dirnames, files in os.walk(suite.dir):
 currentDir = join(suite.dir, root)
 if currentDir in projectDirs:
 # don't traverse subdirs of an existing project
 if len(indicators) != 0:
 if not sys.stdout.isatty() or ask_yes_no(currentDir + ' looks like a removed project -- delete it', 'n'):
 shutil.rmtree(currentDir)
 log('Deleted ' + currentDir)
-def javadoc(args, parser=None, docDir='javadoc', includeDeps=True):
+def javadoc(args, parser=None, docDir='javadoc', includeDeps=True, stdDoclet=True):
 """generate javadoc for some/all Java projects"""
 parser = ArgumentParser(prog='mx javadoc') if parser is None else parser
 parser.add_argument('-d', '--base', action='store', help='base directory for output')
 parser.add_argument('--unified', action='store_true', help='put javadoc in a single directory instead of one per project')
 parser.add_argument('--exclude-packages', action='store', help='comma separated packages to exclude')
 args = parser.parse_args(args)
 # build list of projects to be processed
-candidates = sorted_deps()
 if args.projects is not None:
 candidates = [project(name) for name in args.projects.split(',')]
+else:
+candidates = projects_opt_limit_to_suites()
 # optionally restrict packages within a project
 packages = []
 if args.packages is not None:
 packages = [name for name in args.packages.split(',')]
 print >> fp, '<html><body>Documentation for the <code>' + p.name + '</code> project.</body></html>'
 delOverviewFile = True
 nowarnAPI = []
 if not args.warnAPI:
 nowarnAPI.append('-XDignore.symbol.file')
+# windowTitle onloy applies to the standard doclet processor
+windowTitle = []
+if stdDoclet:
+windowTitle = ['-windowtitle', p.name + ' javadoc']
 try:
 log('Generating {2} for {0} in {1}'.format(p.name, out, docDir))
 run([java().javadoc, memory,
-'-windowtitle', p.name + ' javadoc',
 '-XDignore.symbol.file',
 '-classpath', cp,
 '-quiet',
 '-d', out,
 '-overview', overviewFile,
 '-sourcepath', sp] +
 links +
 extraArgs +
 nowarnAPI +
+windowTitle +
 list(pkgs))
 log('Generated {2} for {0} in {1}'.format(p.name, out, docDir))
 finally:
 if delOverviewFile:
 os.remove(overviewFile)
 run([javapExe, '-private', '-verbose', '-classpath', classpath()] + selection)
 def show_projects(args):
 """show all loaded projects"""
 for s in suites():
-projectsFile = join(s.dir, 'mx', 'projects')
+projectsFile = join(s.mxDir, 'projects')
 if exists(projectsFile):
 log(projectsFile)
 for p in s.projects:
 log('\t' + p.name)
 """
 Define how a single command-line argument.
 """
 assert _argParser is not None
 _argParser.add_argument(*args, **kwargs)
+def update_commands(suite, new_commands):
+for key, value in new_commands.iteritems():
+if _commands.has_key(key) and not suite.primary:
+pass
+# print("WARNING: attempt to redefine command '" + key + "' in suite " + suite.dir)
+else:
+_commands[key] = value
 # Table of commands in alphabetical order.
 # Keys are command names, value are lists: [<function>, <usage msg>, <format args to doc string of function>...]
 # If any of the format args are instances of Callable, then they are called with an 'env' are before being
 # used in the call to str.format().
 'help': [help_, '[command]'],
 'ideclean': [ideclean, ''],
 'ideinit': [ideinit, ''],
 'archive': [archive, '[options]'],
 'projectgraph': [projectgraph, ''],
+'pylint': [pylint, ''],
 'javap': [javap, '<class name patterns>'],
 'javadoc': [javadoc, '[options]'],
 'site': [site, '[options]'],
 'netbeansinit': [netbeansinit, ''],
 'projects': [show_projects, ''],
 _argParser = ArgParser()
 def _findPrimarySuite():
 def is_suite_dir(d):
-mxDir = join(d, 'mx')
+for f in os.listdir(d):
-if exists(mxDir) and isdir(mxDir) and exists(join(mxDir, 'projects')):
+if f == 'mx' or fnmatch.fnmatch(f, 'mx.*'):
-return dirname(mxDir)
+mxDir = join(d, f)
+if exists(mxDir) and isdir(mxDir) and exists(join(mxDir, 'projects')):
+return dirname(mxDir)
 # try current working directory first
 if is_suite_dir(os.getcwd()):
 return os.getcwd()

Mercurial > hg > truffle

comparison mxtool/mx.py @ 12387:aff825fef0fd