truffle: mxtool/mx.py comparison

comparison mxtool/mx.py @ 12477:a5d83166dca6

mxtool hg support for suites

author	Mick Jordan <mick.jordan@oracle.com>
date	Thu, 17 Oct 2013 19:59:25 -0700
parents	d08accd58925
children	28d7a11ba008

comparison

equal deleted inserted replaced

-:67566565053a
+:a5d83166dca6
 dependencies between them.
 commands.py
 Suite specific extensions to the commands available to mx.
-includes
+imports
 Other suites to be loaded. This is recursive. Each
-line in an includes file is a path to a suite directory.
+line in an imports file is the name of an imported suite.
+The suite is located using a SuiteModel (cf searchpath)
 env
 A set of environment variable definitions. These override any
 existing environment variables. Common properties set here
 include JAVA_HOME and IGNORED_PROJECTS.
-The includes and env files are typically not put under version control
+The env file is typically not put under version control
-as they usually contain local file-system paths.
+as it usually contain local file-system paths.
 The projects file is like the pom.xml file from Maven except that
 it is a properties file (not XML). Each non-comment line
 in the file specifies an attribute of a project or library. The main
 difference between a project and a library is that the former contains
 _libs = dict()
 _dists = dict()
 _suites = dict()
 _annotationProcessors = None
 _mainSuite = None
+_src_suitemodel = None
+_dst_suitemodel = None
 _opts = None
 _java = None
 _check_global_structures = True  # can be set False to allow suites with duplicate definitions to load without aborting
+_warn = False
 """
 A distribution is a jar or zip file containing the output from one or more Java projects.
 """
 if not includeLibs or not includeSelf:
 return deps
 deps.append(self)
 return deps
+class SuiteModel:
+"""
+Defines how to locate a URL/path for a suite, including imported suites.
+Conceptually a SuiteModel is defined by a primary suite URL/path and a
+map from suite name to URL/path for imported suites.
+Subclasses define a specfic implementation.
+"""
+def __init__(self):
+self.primaryDir = None
+self.suitenamemap = {}
+def _find_suite_dir(self, suitename):
+"""locates the URL/path for suitename or None if not found"""
+abort('_find_suite_dir not implemented')
+def _set_primary_dir(self, d):
+"""informs that d is the primary suite directory"""
+self._primaryDir = d
+def _importee_dir(self, importer_dir, suitename):
+"""returns the directory path for an import of suitename, given importer_dir"""
+abort('_importee_dir not implemented')
+def _nestedsuites_dirname(self):
+"""Returns the dirname that contains any nested suites if the model supports that"""
+return None
+def _mxDirName(self, name):
+# temporary workaround until mx.graal exists
+if name == 'graal':
+return 'mx'
+else:
+return 'mx.' + name
+def _search_dir(self, searchDir, mxDirName):
+for dd in os.listdir(searchDir):
+sd = _is_suite_dir(join(searchDir, dd), mxDirName)
+if sd is not None:
+return sd
+def _create_suitenamemap(self, optionspec, suitemap):
+"""Three ways to specify a suite name mapping, in order of precedence:
+1. Explicitly in optionspec.
+2. In suitemap.
+3. in MXSUITEMAP environment variable.
+"""
+if optionspec != '':
+spec = optionspec
+elif suitemap is not None:
+spec = suitemap
+elif get_env('MXSUITEMAP') is not None:
+spec = get_env('MXSUITEMAP')
+else:
+return
+pairs = spec.split(',')
+for pair in pairs:
+mappair = pair.split('=')
+self.suitenamemap[mappair[0]] = mappair[1]
+class SiblingSuiteModel(SuiteModel):
+"""All suites are siblings in the same parent directory, recorded as _suiteRootDir"""
+def __init__(self, suiteRootDir, option, suitemap):
+SuiteModel.__init__(self)
+self._suiteRootDir = suiteRootDir
+self._create_suitenamemap(option[len('sibling:'):], suitemap)
+def _find_suite_dir(self, name):
+return self._search_dir(self._suiteRootDir, self._mxDirName(name))
+def _set_primary_dir(self, d):
+SuiteModel._set_primary_dir(self, d)
+self._suiteRootDir = dirname(d)
+def _importee_dir(self, importer_dir, suitename):
+if self.suitenamemap.has_key(suitename):
+suitename = self.suitenamemap[suitename]
+return join(dirname(importer_dir), suitename)
+class NestedImportsSuiteModel(SuiteModel):
+"""Imported suites are all siblings in an 'imported_suites' directory of the primary suite"""
+def _imported_suites_dirname(self):
+return "imported_suites"
+def __init__(self, primaryDir, option, suitemap):
+SuiteModel.__init__(self)
+self._primaryDir = primaryDir
+self._create_suitenamemap(option[len('nested:'):], suitemap)
+def _find_suite_dir(self, name):
+return self._search_dir(join(self._primaryDir, self._imported_suites_dirname()), self._mxDirName(name))
+def _importee_dir(self, importer_dir, suitename):
+if self.suitenamemap.has_key(suitename):
+suitename = self.suitenamemap[suitename]
+if basename(importer_dir) == basename(self._primaryDir):
+# primary is importer
+this_imported_suites_dirname = join(importer_dir, self._imported_suites_dirname())
+if not exists(this_imported_suites_dirname):
+os.mkdir(this_imported_suites_dirname)
+return join(this_imported_suites_dirname, suitename)
+else:
+return join(dirname(importer_dir), suitename)
+def _nestedsuites_dirname(self):
+return self._imported_suites_dirname()
+class PathSuiteModel(SuiteModel):
+"""The most general model. Uses a map from suitename to URL/path provided by the user"""
+def __init__(self, path):
+SuiteModel.__init__(self)
+paths = path.split(',')
+self.suit_to_url = {}
+for path in paths:
+pair = path.split('=')
+if len(pair) > 1:
+suitename = pair[0]
+suiteurl = pair[1]
+else:
+suitename = basename(pair[0])
+suiteurl = pair[0]
+self.suit_to_url[suitename] = suiteurl
+def _find_suite_dir(self, suitename):
+if self.suit_to_url.has_key(suitename):
+return self.suit_to_url[suitename]
+else:
+return None
+def _importee_dir(self, importer_dir, suitename):
+if suitename in self.suit_to_url:
+return self.suit_to_url[suitename]
+else:
+abort('suite ' + suitename + ' not found')
+class SuiteImport:
+def __init__(self, name, version):
+self.name = name
+self.version = version
+@staticmethod
+def _parse_specification(specification):
+pair = specification.split(',')
+name = pair[0]
+if len(pair) > 1:
+version = pair[1]
+else:
+version = None
+return SuiteImport(name, version)
+@staticmethod
+def _tostring(name, version):
+return name + ',' + version
+def _self_tostring(self):
+return self.name + ',' + self.version
 class Suite:
-def __init__(self, d, mxDir, primary):
+def __init__(self, mxDir, primary, load=True):
-self.dir = d
+self.dir = dirname(mxDir)
 self.mxDir = mxDir
 self.projects = []
 self.libs = []
 self.dists = []
-self.includes = []
+self.imports = []
 self.commands = None
 self.primary = primary
-self._load_env(mxDir)
+self.name = _suitename(mxDir)  # validated in _load_projects
-self._load_commands(mxDir)
+self.version = None  # _hgtip checks current version if not None
-self._load_includes(mxDir)
+self.version = _hgtip(self, False)
-self.name = d  # re-initialized in _load_projects
+if load:
+# load suites bottom up to make sure command overriding works properly
+self._load_imports()
+self._load_env()
+self._load_commands()
+_suites[self.name] = self
 def __str__(self):
 return self.name
-def _load_projects(self, mxDir):
+def _load_projects(self):
 libsMap = dict()
 projsMap = dict()
 distsMap = dict()
-projectsFile = join(mxDir, 'projects')
+projectsFile = join(self.mxDir, 'projects')
 if not exists(projectsFile):
 return
-def _find_suite_key():
-for items in _suites.items():
-if items[1].dir == self.dir:
-return items[0]
-raise KeyError
 with open(projectsFile) as f:
 prefix = ''
 for line in f:
 line = line.strip()
 if len(parts) == 1:
 if parts[0] != 'suite':
 abort('Single part property must be "suite": ' + key)
 if self.name != value:
-currentKey = _find_suite_key()
+abort('suite name in project file does not match ' + _suitename(self.mxDir))
-_suites.pop(currentKey)
-self.name = value
-_suites[value] = self
 continue
 if len(parts) != 3:
 abort('Property name does not have 3 parts separated by "@": ' + key)
 kind, name, attr = parts
 if kind == 'project':
 self.dists.append(d)
 if self.name is None:
 abort('Missing "suite=<name>" in ' + projectsFile)
-def _load_commands(self, mxDir):
+def _commands_name(self):
-commandsPath = join(mxDir, 'commands.py')
+return 'mx_' + self.name
+def _find_commands(self, name):
+commandsPath = join(self.mxDir, name + '.py')
 if exists(commandsPath):
+return name
+else:
+return None
+def _load_commands(self):
+commandsName = self._find_commands(self._commands_name())
+if commandsName is None:
+# backwards compatibility
+commandsName = self._find_commands('commands')
+if commandsName is not None:
+if commandsName in sys.modules:
+abort(commandsName + '.py in suite ' + self.name + ' duplicates ' + sys.modules[commandsName].__file__)
 # temporarily extend the Python path
-sys.path.insert(0, mxDir)
+sys.path.insert(0, self.mxDir)
-mod = __import__('commands')
+mod = __import__(commandsName)
-self.commands = sys.modules.pop('commands')
+self.commands = sys.modules.pop(commandsName)
-sys.modules[join(mxDir, 'commands')] = self.commands
+sys.modules[commandsName] = self.commands
 # revert the Python path
 del sys.path[0]
 if not hasattr(mod, 'mx_init'):
-abort(commandsPath + ' must define an mx_init(env) function')
+abort(commandsName + '.py in suite ' + self.name + ' must define an mx_init(suite) function')
 if hasattr(mod, 'mx_post_parse_cmd_line'):
 self.mx_post_parse_cmd_line = mod.mx_post_parse_cmd_line
 mod.mx_init(self)
 self.commands = mod
-def _load_includes(self, mxDir):
+def _visit_imports(self, visitor, **extra_args):
-includes = join(mxDir, 'includes')
+"""
-if exists(includes):
+Visitor support for the imports file.
-with open(includes) as f:
+For each line of the imports file that specifies an import, the visitor function is
+called with this suite, a SuiteImport instance created from the line and any extra args
+passed to this call. In addition, if extra_args contains a key 'update_versions' that is True,
+a StringIO value is added to extra_args with key 'updated_imports', and the visitor is responsible
+for writing a (possibly) updated import line to the file, and the file is (possibly) updated after
+all imports are processed.
+N.B. There is no built-in support for avoiding visiting the same suite multiple times,
+as this function only visits the imports of a singkle suite. If a (recursive) visitor function
+wishes to visit a suite exactly once, it must manage that through extra_args.
+"""
+importsFile = join(self.mxDir, 'imports')
+if exists(importsFile):
+update_versions = extra_args.has_key('update_versions') and extra_args['update_versions']
+out = StringIO.StringIO() if update_versions else None
+extra_args['updated_imports'] = out
+with open(importsFile) as f:
 for line in f:
-include = expandvars_in_property(line.strip())
+sline = line.strip()
-self.includes.append(include)
+if len(sline) == 0 or sline.startswith('#'):
-_loadSuite(os.path.abspath(include), False)
+if out is not None:
+out.write(sline + '\n')
-def _load_env(self, mxDir):
+continue
-e = join(mxDir, 'env')
+suite_import = SuiteImport._parse_specification(line.strip())
+visitor(self, suite_import, **extra_args)
+if out is not None:
+update_file(importsFile, out.getvalue())
+@staticmethod
+def _find_and_loadsuite(suite, suite_import, **extra_args):
+"""visitor for the initial suite load"""
+importMxDir = _src_suitemodel._find_suite_dir(suite_import.name)
+if importMxDir is None:
+abort('import ' + suite_import.name + ' not found')
+suite.imports.append(suite_import)
+imported_suite = _loadSuite(importMxDir, False)
+if imported_suite.version != suite.version:
+warn('import version of ' + imported_suite.name +' does not match tip of ' + suite.version)
+def _load_imports(self):
+self._visit_imports(self._find_and_loadsuite)
+def _load_env(self):
+e = join(self.mxDir, 'env')
 if exists(e):
 with open(e) as f:
 lineNum = 0
 for line in f:
 lineNum = lineNum + 1
 abort(e + ':' + str(lineNum) + ': line does not match pattern "key=value"')
 key, value = line.split('=', 1)
 os.environ[key.strip()] = expandvars_in_property(value.strip())
 def _post_init(self, opts):
-self._load_projects(self.mxDir)
+self._load_projects()
-# set the global data structures, checking for conflicts unless _global_structures is False
+# set the global data structures, checking for conflicts unless _check_global_structures is False
 for p in self.projects:
 existing = _projects.get(p.name)
 if existing is not None and _check_global_structures:
 abort('cannot override project  ' + p.name + ' in ' + p.dir + " with project of the same name in  " + existing.dir)
 if not p.name in _opts.ignored_projects:
 for d in self.dists:
 existing = _dists.get(d.name)
 if existing is not None and _check_global_structures:
 # allow redefinition, so use path from existing
 # abort('cannot redefine distribution  ' + d.name)
-print('WARNING: distribution ' + d.name + ' redefined')
+warn('distribution ' + d.name + ' redefined')
 d.path = existing.path
 _dists[d.name] = d
 if hasattr(self, 'mx_post_parse_cmd_line'):
 self.mx_post_parse_cmd_line(opts)
 elif sys.platform.startswith('win32') or sys.platform.startswith('cygwin'):
 return 'windows'
 else:
 abort('Unknown operating system ' + sys.platform)
-def _loadSuite(d, primary=False):
+def _loadSuite(mxDir, primary=False):
 """
-Load a suite from the 'mx' or 'mx.bbb' subdirectory of d, where 'bbb' is basename of d
+Load a suite from 'mxDir'.
 """
-mxDefaultDir = join(d, 'mx')
+for s in _suites.itervalues():
-name = os.path.basename(d)
+if s.mxDir == mxDir:
-mxTaggedDir = mxDefaultDir + '.' + name
+return s
-mxDir = None
+# create the new suite
-if exists(mxTaggedDir) and isdir(mxTaggedDir):
+s = Suite(mxDir, primary)
-mxDir = mxTaggedDir
+return s
-else:
-if exists(mxDefaultDir) and isdir(mxDefaultDir):
-mxDir = mxDefaultDir
-if mxDir is None:
-return None
-if len([s for s in _suites.itervalues() if s.dir == d]) == 0:
-s = Suite(d, mxDir, primary)
-# N.B. this will be updated once the projects file has been read
-_suites[name] = s
-return s
 def suites(opt_limit_to_suite=False):
 """
 Get the list of all loaded suites.
 """
 """
 s = _suites.get(name)
 if s is None and fatalIfMissing:
 abort('suite named ' + name + ' not found')
 return s
 def projects_from_names(projectNames):
 """
 Get the list of projects corresponding to projectNames; all projects if None
 """
 log('Does not appear to be a valid JDK as ' + rtJarPath + ' does not exist')
 javaHome = None
 else:
 break
-envPath = join(_mainSuite.dir, 'mx', 'env')
+envPath = join(_mainSuite.mxDir, 'env')
 if ask_yes_no('Persist this setting by adding "JAVA_HOME=' + javaHome + '" to ' + envPath, 'y'):
 with open(envPath, 'a') as fp:
 print >> fp, 'JAVA_HOME=' + javaHome
 return javaHome
 class ArgParser(ArgumentParser):
 # Override parent to append the list of available commands
 def format_help(self):
 return ArgumentParser.format_help(self) + _format_commands()
 # this doesn't resolve the right way, but can't figure out how to override _handle_conflict_resolve in _ActionsContainer
 ArgumentParser.__init__(self, prog='mx', conflict_handler='resolve')
 self.add_argument('-v', action='store_true', dest='verbose', help='enable verbose output')
 self.add_argument('-V', action='store_true', dest='very_verbose', help='enable very verbose output')
+self.add_argument('-w', action='store_true', dest='warn', help='enable warning messages')
 self.add_argument('--dbg', type=int, dest='java_dbg_port', help='make Java processes wait on <port> for a debugger', metavar='<port>')
 self.add_argument('-d', action='store_const', const=8000, dest='java_dbg_port', help='alias for "-dbg 8000"')
 self.add_argument('--cp-pfx', dest='cp_prefix', help='class path prefix', metavar='<arg>')
 self.add_argument('--cp-sfx', dest='cp_suffix', help='class path suffix', metavar='<arg>')
 self.add_argument('--J', dest='java_args', help='Java VM arguments (e.g. --J @-dsa)', metavar='@<args>', default='-ea -Xss2m -Xmx1g')
 self.add_argument('--Ja', action='append', dest='java_args_sfx', help='suffix Java VM arguments (e.g. --Ja @-dsa)', metavar='@<args>', default=[])
 self.add_argument('--user-home', help='users home directory', metavar='<path>', default=os.path.expanduser('~'))
 self.add_argument('--java-home', help='bootstrap JDK installation directory (must be JDK 6 or later)', metavar='<path>')
 self.add_argument('--ignore-project', action='append', dest='ignored_projects', help='name of project to ignore', metavar='<name>', default=[])
 self.add_argument('--suite', action='append', dest='specific_suites', help='limit command to given suite', default=[])
+self.add_argument('--src-suitemodel', help='mechanism for locating imported suites', metavar='<arg>', default='sibling')
+self.add_argument('--dst-suitemodel', help='mechanism for placing cloned/pushed suites', metavar='<arg>', default='sibling')
+self.add_argument('--suitemap', help='explicit remapping of suite names', metavar='<args>')
 if get_os() != 'windows':
 # Time outs are (currently) implemented with Unix specific functionality
 self.add_argument('--timeout', help='timeout (in seconds) for command', type=int, default=0, metavar='<secs>')
 self.add_argument('--ptimeout', help='timeout (in seconds) for subprocesses', type=int, default=0, metavar='<secs>')
 commandAndArgs = opts.__dict__.pop('commandAndArgs')
 return opts, commandAndArgs
 def _handle_conflict_resolve(self, action, conflicting_actions):
 self._handle_conflict_error(action, conflicting_actions)
+class SMArgParser(ArgParser):
+"""Parser that just looks for suitemodel options, which must happen before suites are loaded"""
+def __init__(self):
+ArgParser.__init__(self)
+def _suitemodel(self, option, suitemap):
+if option.startswith('sibling'):
+return SiblingSuiteModel(os.getcwd(), option, suitemap)
+elif option.startswith('nested'):
+return NestedImportsSuiteModel(os.getcwd(), option, suitemap)
+elif option.startswith('path'):
+return PathSuiteModel(option[len('path:'):])
+else:
+abort('unknown suitemodel type: ' + option)
+def _parse_suitemodel_options(self):
+# the command line may contains args added by suites, so we only parse the currently known args
+opts = self.parse_known_args()[0]
+# set this early
+global _warn
+_warn = opts.warn
+global _src_suitemodel
+_src_suitemodel = self._suitemodel(opts.src_suitemodel, opts.suitemap)
+global _dst_suitemodel
+_dst_suitemodel = self._suitemodel(opts.dst_suitemodel, opts.suitemap)
 def _format_commands():
 msg = '\navailable commands:\n\n'
 for cmd in sorted(_commands.iterkeys()):
 c, _ = _commands[cmd][:2]
 if not suppliedParser:
 parser = ArgumentParser(prog='mx build')
 javaCompliance = java().javaCompliance
-defaultEcjPath = join(_mainSuite.dir, 'mx', 'ecj.jar')
+defaultEcjPath = join(_mainSuite.mxDir, 'ecj.jar')
 parser = parser if parser is not None else ArgumentParser(prog='mx build')
 parser.add_argument('-f', action='store_true', dest='force', help='force build (disables timestamp checking)')
 parser.add_argument('-c', action='store_true', dest='clean', help='removes existing build output')
 parser.add_argument('--source', dest='compliance', help='Java compliance level for projects without an explicit one', default=str(javaCompliance))
 '-d', outputDir]
 jdtArgs += processorArgs
 jdtProperties = join(p.dir, '.settings', 'org.eclipse.jdt.core.prefs')
-rootJdtProperties = join(p.suite.dir, 'mx', 'eclipse-settings', 'org.eclipse.jdt.core.prefs')
+rootJdtProperties = join(p.suite.mxDir, 'eclipse-settings', 'org.eclipse.jdt.core.prefs')
 if not exists(jdtProperties) or os.path.getmtime(jdtProperties) < os.path.getmtime(rootJdtProperties):
 # Try to fix a missing properties file by running eclipseinit
 eclipseinit([], buildProcessorJars=False)
 if not exists(jdtProperties):
 log('JDT properties file {0} not found'.format(jdtProperties))
 """process all project files to canonicalize the dependencies
 The exit code of this command reflects how many files were updated."""
 changedFiles = 0
-for s in suites():
+for s in suites(True):
 projectsFile = join(s.mxDir, 'projects')
 if not exists(projectsFile):
 continue
 with open(projectsFile) as f:
 out = StringIO.StringIO()
 if refresh:
 launchOut.element('stringAttribute', {'key' : 'org.eclipse.debug.core.ATTR_REFRESH_SCOPE', 'value': '${project}'})
 launchOut.element('booleanAttribute', {'key' : 'org.eclipse.debug.ui.ATTR_CONSOLE_OUTPUT_ON', 'value': consoleOn})
 launchOut.element('booleanAttribute', {'key' : 'org.eclipse.debug.ui.ATTR_LAUNCH_IN_BACKGROUND', 'value': 'true' if async else 'false'})
-baseDir = dirname(dirname(os.path.abspath(__file__)))
+# expect to find the OS command to invoke mx in the same directory
+baseDir = dirname(os.path.abspath(__file__))
 cmd = 'mx.sh'
 if get_os() == 'windows':
 cmd = 'mx.cmd'
-launchOut.element('stringAttribute', {'key' : 'org.eclipse.ui.externaltools.ATTR_LOCATION', 'value': join(baseDir, cmd) })
+cmdPath = join(baseDir, cmd)
+if not os.path.exists(cmdPath):
+# backwards compatibility for when the commands lived in parent of mxtool
+cmdPath = join(dirname(baseDir), cmd)
+if not os.path.exists(cmdPath):
+abort('cannot locate ' + cmd)
+launchOut.element('stringAttribute', {'key' : 'org.eclipse.ui.externaltools.ATTR_LOCATION', 'value':  cmdPath})
 launchOut.element('stringAttribute', {'key' : 'org.eclipse.ui.externaltools.ATTR_RUN_BUILD_KINDS', 'value': 'auto,full,incremental'})
 launchOut.element('stringAttribute', {'key' : 'org.eclipse.ui.externaltools.ATTR_TOOL_ARGUMENTS', 'value': mxCommand})
 launchOut.element('booleanAttribute', {'key' : 'org.eclipse.ui.externaltools.ATTR_TRIGGERS_CONFIGURED', 'value': 'true'})
 launchOut.element('stringAttribute', {'key' : 'org.eclipse.ui.externaltools.ATTR_WORKING_DIRECTORY', 'value': p.suite.dir})
 def fsckprojects(args):
 """find directories corresponding to deleted Java projects and delete them"""
 for suite in suites(True):
 projectDirs = [p.dir for p in suite.projects]
-for root, dirnames, files in os.walk(suite.dir):
+for dirpath, dirnames, files in os.walk(suite.dir):
-currentDir = join(suite.dir, root)
+if dirpath == suite.dir:
-if currentDir in projectDirs:
+# no point in traversing .hg
-# don't traverse subdirs of an existing project
+if '.hg' in dirnames:
+dirnames.remove('.hg')
+# if there are nested suites must not scan those now, as they are not in projectDirs
+if _src_suitemodel._nestedsuites_dirname() in dirnames:
+dirnames.remove(_src_suitemodel._nestedsuites_dirname())
+elif dirpath in projectDirs:
+# don't traverse subdirs of an existing project in this suite
 dirnames[:] = []
 else:
 projectConfigFiles = frozenset(['.classpath', 'nbproject'])
 indicators = projectConfigFiles.intersection(files)
 if len(indicators) != 0:
-if not sys.stdout.isatty() or ask_yes_no(currentDir + ' looks like a removed project -- delete it', 'n'):
+if not sys.stdout.isatty() or ask_yes_no(dirpath + ' looks like a removed project -- delete it', 'n'):
-shutil.rmtree(currentDir)
+shutil.rmtree(dirpath)
-log('Deleted ' + currentDir)
+log('Deleted ' + dirpath)
 def javadoc(args, parser=None, docDir='javadoc', includeDeps=True, stdDoclet=True):
 """generate javadoc for some/all Java projects"""
 parser = ArgumentParser(prog='mx javadoc') if parser is None else parser
 finally:
 if exists(tmpbase):
 shutil.rmtree(tmpbase)
+def _kwArg(kwargs):
+if len(kwargs) > 0:
+return kwargs.pop(0)
+return None
+def sclone(args):
+"""clone a suite repository, and its imported suites"""
+parser = ArgumentParser(prog='mx sclone')
+parser.add_argument('--source', help='url/path of repo containing suite', metavar='<url>')
+parser.add_argument('--dest', help='destination directory (default basename of source)', metavar='<path>')
+parser.add_argument("--no-imports", action='store_true', help='do not clone imported suites')
+parser.add_argument('nonKWArgs', nargs=REMAINDER, metavar='source [dest]...')
+args = parser.parse_args(args)
+# check for non keyword args
+if args.source is None:
+args.source = _kwArg(args.nonKWArgs)
+if args.dest is None:
+args.dest = _kwArg(args.nonKWArgs)
+if len(args.nonKWArgs) > 0:
+abort('unrecognized args: ' + ' '.join(args.nonKWArgs))
+if args.source is None:
+# must be primary suite and dest is required
+if _mainSuite is None:
+abort('--source missing and no primary suite found')
+if args.dest is None:
+abort('--dest required when --source is not given')
+source = _mainSuite.dir
+else:
+source = args.source
+if args.dest is not None:
+dest = args.dest
+else:
+dest = basename(source)
+dest = os.path.abspath(dest)
+# We can now set the primary dir for the src/dst suitemodel
+_dst_suitemodel._set_primary_dir(dest)
+_src_suitemodel._set_primary_dir(source)
+_sclone(source, dest, None, args.no_imports)
+def _sclone(source, dest, version, no_imports):
+cmd = ['hg', 'clone']
+if version is not None:
+cmd.append('-r')
+cmd.append(version)
+cmd.append(source)
+cmd.append(dest)
+run(cmd)
+mxDir = _is_suite_dir(dest)
+if mxDir is None:
+warn(source + ' is not an mx suite')
+return None
+# create a Suite (without loading) to enable imports visitor
+s = Suite(mxDir, False, load=False)
+if not no_imports:
+s._visit_imports(_scloneimports_visitor, source=source)
+return s
+def _scloneimports_visitor(s, suite_import, source, **extra_args):
+"""
+cloneimports visitor for Suite._visit_imports.
+The destination information is encapsulated by 's'
+"""
+_scloneimports(s, suite_import, source)
+def _scloneimports_suitehelper(sdir):
+mxDir = _is_suite_dir(sdir)
+if mxDir is None:
+abort(sdir + ' is not an mx suite')
+else:
+# create a Suite (without loading) to enable imports visitor
+return Suite(mxDir, False, load=False)
+def _scloneimports(s, suite_import, source):
+# clone first, then visit imports once we can locate them
+importee_source = _src_suitemodel._importee_dir(source, suite_import.name)
+importee_dest = _dst_suitemodel._importee_dir(s.dir, suite_import.name)
+if exists(importee_dest):
+importee_suite = _scloneimports_suitehelper(importee_dest)
+importee_suite._visit_imports(_scloneimports_visitor, source=importee_source)
+else:
+_sclone(importee_source, importee_dest, suite_import.version, False)
+# _clone handles the recursive visit of the new imports
+def scloneimports(args):
+"""clone the imports of an existing suite"""
+parser = ArgumentParser(prog='mx scloneimports')
+parser.add_argument('--source', help='url/path of repo containing suite', metavar='<url>')
+parser.add_argument('nonKWArgs', nargs=REMAINDER, metavar='source [dest]...')
+args = parser.parse_args(args)
+# check for non keyword args
+if args.source is None:
+args.source = _kwArg(args.nonKWArgs)
+if not os.path.isdir(args.source):
+abort(args.source + ' is not a directory')
+s = _scloneimports_suitehelper(args.source)
+default_path = _hgdefault_push(args.source)
+if default_path is None:
+abort('no default path in ' + join(args.source, '.hg', 'hgrc'))
+# We can now set the primary dir for the dst suitemodel
+# N.B. source is effectively the destination and the default_path is the (original) source
+_dst_suitemodel._set_primary_dir(args.source)
+s._visit_imports(_scloneimports_visitor, source=default_path)
+def _spush_import_visitor(s, suite_import, dest, checks, clonemissing, **extra_args):
+"""push visitor for Suite._visit_imports"""
+if dest is not None:
+dest = _dst_suitemodel._importee_dir(dest, suite_import.name)
+_spush(suite(suite_import.name), suite_import, dest, checks, clonemissing)
+def _spush_check_import_visitor(s, suite_import, **extra_args):
+"""push check visitor for Suite._visit_imports"""
+currentTip = _hgtip(suite(suite_import.name))
+if currentTip != suite_import.version:
+abort('import version of ' + suite_import.name + ' in suite ' + s.name + ' does not match tip')
+def _spush(s, suite_import, dest, checks, clonemissing):
+if checks:
+if not _hgcanpush(s):
+abort('working directory ' + s.dir + ' contains uncommitted changes, push aborted')
+# check imports first
+if checks:
+s._visit_imports(_spush_check_import_visitor)
+# ok, push imports
+s._visit_imports(_spush_import_visitor, dest=dest, checks=checks, clonemissing=clonemissing)
+dest_exists = True
+if clonemissing:
+if not os.path.exists(dest):
+dest_exists = False
+def add_version(cmd, suite_import):
+if suite_import is not None and suite_import.version is not None:
+cmd.append('-r')
+cmd.append(suite_import.version)
+if dest_exists:
+cmd = ['hg', '-R', s.dir, 'push']
+add_version(cmd, suite_import)
+if dest is not None:
+cmd.append(dest)
+rc = run(cmd, nonZeroIsFatal=False)
+if rc != 0:
+# rc of 1 not an error,  means no changes
+if rc != 1:
+abort("push failed, exit code " + str(rc))
+else:
+cmd = ['hg', 'clone']
+add_version(cmd, suite_import)
+cmd.append(s.dir)
+cmd.append(dest)
+run(cmd)
+def spush(args):
+"""push primary suite and all its imports"""
+parser = ArgumentParser(prog='mx spush')
+parser.add_argument('--dest', help='url/path of repo to push to (default as per hg push)', metavar='<path>')
+parser.add_argument('--no-checks', action='store_true', help='checks on status, versions are disabled')
+parser.add_argument('--clonemissing', action='store_true', help='clone missing imported repos at destination (forces --no-checks)')
+parser.add_argument('nonKWArgs', nargs=REMAINDER, metavar='source [dest]...')
+args = parser.parse_args(args)
+if args.dest is None:
+args.dest = _kwArg(args.nonKWArgs)
+if len(args.nonKWArgs) > 0:
+abort('unrecognized args: ' + ' '.join(args.nonKWArgs))
+if args.dest is not None and not os.path.isdir(args.dest):
+abort('destination must be a directory')
+s = _check_primary_suite()
+if args.clonemissing:
+if args.dest is None:
+abort('--dest required with --clonemissing')
+args.nochecks = True
+if args.dest is not None:
+_dst_suitemodel._set_primary_dir(args.dest)
+_spush(s, None, args.dest, not args.nochecks, args.clonemissing)
+def _supdate_import_visitor(s, suite_import, **extra_args):
+_supdate(suite(suite_import.name), suite_import)
+def _supdate(s, suite_import):
+s._visit_imports(_supdate_import_visitor)
+run(['hg', '-R', s.dir, 'update'])
+def supdate(args):
+"""update primary suite and all its imports"""
+s = _check_primary_suite()
+_supdate(s, None)
+def _scheck_imports_visitor(s, suite_import, update_versions, updated_imports):
+"""checkimportversions visitor for Suite._visit_imports"""
+_scheck_imports(suite(suite_import.name), suite_import, update_versions, updated_imports)
+def _scheck_imports(s, suite_import, update_versions, updated_imports):
+# check imports recursively
+s._visit_imports(_scheck_imports_visitor, update_versions=update_versions)
+currentTip = _hgtip(s)
+if currentTip != suite_import.version:
+print('import version of ' + s.name + ' does not match tip' + (': updating' if update_versions else ''))
+if update_versions:
+suite_import.version = currentTip
+line = suite_import._self_tostring()
+updated_imports.write(line + '\n')
+def scheckimports(args):
+"""check that suite import versions are up to date"""
+parser = ArgumentParser(prog='mx scheckimports')
+parser.add_argument('--update-versions', help='update imported version ids', action='store_true')
+args = parser.parse_args(args)
+_check_primary_suite()._visit_imports(_scheck_imports_visitor, update_versions=args.update_versions)
+def _hgtip(s, abortOnError=True):
+try:
+version = subprocess.check_output(['hg', 'tip', '-R', s.dir, '--template', '{node}'])
+if s.version is not None and s.version != version:
+abort('version of suite ' + s.name +' has changed during run')
+return version
+except subprocess.CalledProcessError:
+if abortOnError:
+abort('failed to get tip revision id')
+else:
+return None
+def _hgcanpush(s):
+try:
+output = subprocess.check_output(['hg', '-R', s.dir, 'status'])
+# super strict
+return output == ''
+except subprocess.CalledProcessError:
+return False
+def _hgdefault_push(sdir):
+with open(join(sdir, '.hg', 'hgrc')) as f:
+for line in f:
+line = line.rstrip()
+if line.startswith('default = '):
+return line[len('default = '):]
+return None
+def _spull_import_visitor(s, suite_import, update_versions, updated_imports):
+"""pull visitor for Suite._visit_imports"""
+_spull(suite(suite_import.name), update_versions, updated_imports)
+def _spull(s, update_versions, updated_imports):
+# pull imports first
+s._visit_imports(_spull_import_visitor, update_versions=update_versions)
+run(['hg', '-R', s.dir, 'pull', '-u'])
+if update_versions and updated_imports is not None:
+tip = _hgtip(s)
+updated_imports.write(SuiteImport._tostring(s.name, tip) + '\n')
+def spull(args):
+"""pull primary suite and all its imports"""
+parser = ArgumentParser(prog='mx spull')
+parser.add_argument('--update-versions', action='store_true', help='update version ids of imported suites')
+args = parser.parse_args(args)
+_spull(_check_primary_suite(), args.update_versions, None)
 def findclass(args, logToConsole=True):
 """find all classes matching a given substring"""
 matches = []
 for entry, filename in classpath_walk(includeBootClasspath=True):
 if filename.endswith('.class'):
 if isinstance(entry, zipfile.ZipFile):
 classname = filename.replace('/', '.')
 assert _argParser is not None
 _argParser.add_argument(*args, **kwargs)
 def update_commands(suite, new_commands):
 for key, value in new_commands.iteritems():
-if _commands.has_key(key) and not suite.primary:
+if _commands.has_key(key):
-pass
+warn("redefining command '" + key + "' in suite " + suite.name)
-# print("WARNING: attempt to redefine command '" + key + "' in suite " + suite.dir)
+_commands[key] = value
-else:
-_commands[key] = value
+def warn(msg):
+if _warn:
+print('WARNING: ' + msg)
 # Table of commands in alphabetical order.
 # Keys are command names, value are lists: [<function>, <usage msg>, <format args to doc string of function>...]
 # If any of the format args are instances of Callable, then they are called with an 'env' are before being
 # used in the call to str.format().
-# Extensions should update this table directly
+# Suite extensions should not update this table directly, but use update_commands
 _commands = {
 'about': [about, ''],
 'build': [build, '[options]'],
 'checkstyle': [checkstyle, ''],
 'canonicalizeprojects': [canonicalizeprojects, ''],
 'help': [help_, '[command]'],
 'ideclean': [ideclean, ''],
 'ideinit': [ideinit, ''],
 'archive': [archive, '[options]'],
 'projectgraph': [projectgraph, ''],
+'sclone': [sclone, '[options]'],
+'scheckimports': [scheckimports, ''],
+'scloneimports': [scloneimports, '[options]'],
+'spull': [spull, '[options'],
+'spush': [spush, '[options'],
+'supdate': [supdate, ''],
 'pylint': [pylint, ''],
 'javap': [javap, '<class name patterns>'],
 'javadoc': [javadoc, '[options]'],
 'site': [site, '[options]'],
 'netbeansinit': [netbeansinit, ''],
 'projects': [show_projects, ''],
 }
 _argParser = ArgParser()
-def _findPrimarySuite():
+def _suitename(mxDir):
-def is_suite_dir(d):
+base = os.path.basename(mxDir)
+parts = base.split('.')
+# temporary workaround until mx.graal exists
+if len(parts) == 1:
+return 'graal'
+else:
+return parts[1]
+def _is_suite_dir(d, mxDirName=None):
+"""
+Checks if d contains a suite.
+If mxDirName is None, matches any suite name, otherwise checks for exactly that suite.
+"""
+if os.path.isdir(d):
 for f in os.listdir(d):
-if f == 'mx' or fnmatch.fnmatch(f, 'mx.*'):
+if (mxDirName == None and (f == 'mx' or fnmatch.fnmatch(f, 'mx.*'))) or f == mxDirName:
 mxDir = join(d, f)
 if exists(mxDir) and isdir(mxDir) and exists(join(mxDir, 'projects')):
-return dirname(mxDir)
+return mxDir
+def _check_primary_suite():
-# try current working directory first
+if _mainSuite is None:
-if is_suite_dir(os.getcwd()):
+abort('no primary suite found')
-return os.getcwd()
+else:
+return _mainSuite
-# now search path of my executable
-me = sys.argv[0]
+def _needs_primary_suite(command):
-parent = dirname(me)
+return not command.startswith("sclone")
-while parent:
-if is_suite_dir(parent):
+def _findPrimarySuiteMxDir():
-return parent
+# try current working directory first, the look up the tree
-parent = dirname(parent)
+curdir = os.getcwd()
+while curdir:
+mxDir = _is_suite_dir(curdir)
+if mxDir is not None:
+return mxDir
+parent = dirname(curdir)
+if curdir == parent:
+return None
+curdir = parent
 return None
 def main():
-primarySuiteDir = _findPrimarySuite()
+SMArgParser()._parse_suitemodel_options()
-if primarySuiteDir:
+primarySuiteMxDir = _findPrimarySuiteMxDir()
+if primarySuiteMxDir:
+_src_suitemodel._set_primary_dir(dirname(primarySuiteMxDir))
 global _mainSuite
-_mainSuite = _loadSuite(primarySuiteDir, True)
+_mainSuite = _loadSuite(primarySuiteMxDir, True)
 opts, commandAndArgs = _argParser._parse_cmd_line()
+if primarySuiteMxDir is None:
+msg = 'no primary suite found'
+if len(commandAndArgs) > 0 and _needs_primary_suite(commandAndArgs[0]):
+abort(msg)
+else:
+warn(msg)
 global _opts, _java
 _opts = opts
 _java = JavaConfig(opts)

Mercurial > hg > truffle

comparison mxtool/mx.py @ 12477:a5d83166dca6