# HG changeset patch # User Mick Jordan # Date 1382482125 25200 # Node ID 5ccee20550eaaed6d6e14839c3ddcd1838790168 # Parent ef0de9485627daaeaff55d019d0b56cd73b1b68f mxtool: fixes (GRAAL-557) and code cleanups diff -r ef0de9485627 -r 5ccee20550ea mxtool/mx.py --- a/mxtool/mx.py Tue Oct 22 08:35:26 2013 -0700 +++ b/mxtool/mx.py Tue Oct 22 15:48:45 2013 -0700 @@ -156,7 +156,8 @@ _dists = dict() _suites = dict() _annotationProcessors = None -_mainSuite = None +_primary_suite_path = None +_primary_suite = None _src_suitemodel = None _dst_suitemodel = None _opts = None @@ -514,19 +515,16 @@ except OSError: self.has_hg = False warn(self.missing) - + if not self.has_hg: if abortOnFail: abort(self.missing) else: warn(self.missing) - def tip(self, s, abortOnError=True): + def tip(self, sDir, abortOnError=True): try: - version = subprocess.check_output(['hg', 'tip', '-R', s.dir, '--template', '{node}']) - if s.version is not None and s.version != version: - abort('version of suite ' + s.name +' has changed during run') - return version + return subprocess.check_output(['hg', 'tip', '-R', sDir, '--template', '{node}']) except OSError: warn(self.missing) except subprocess.CalledProcessError: @@ -564,19 +562,19 @@ self.primaryDir = None self.suitenamemap = {} - def _find_suite_dir(self, suitename): + def find_suite_dir(self, suitename): """locates the URL/path for suitename or None if not found""" - abort('_find_suite_dir not implemented') - - def _set_primary_dir(self, d): + abort('find_suite_dir not implemented') + + def set_primary_dir(self, d): """informs that d is the primary suite directory""" self._primaryDir = d - def _importee_dir(self, importer_dir, suitename): + def importee_dir(self, importer_dir, suitename): """returns the directory path for an import of suitename, given importer_dir""" - abort('_importee_dir not implemented') - - def _nestedsuites_dirname(self): + abort('importee_dir not implemented') + + def nestedsuites_dirname(self): """Returns the dirname that contains any nested suites if the model supports that""" return None @@ -613,7 +611,7 @@ self.suitenamemap[mappair[0]] = mappair[1] @staticmethod - def _set_suitemodel(option, suitemap): + def set_suitemodel(option, suitemap): if option.startswith('sibling'): return SiblingSuiteModel(os.getcwd(), option, suitemap) elif option.startswith('nested'): @@ -624,7 +622,7 @@ abort('unknown suitemodel type: ' + option) @staticmethod - def _parse_options(): + def parse_options(): # suite-specific args may match the known args so there is no way at this early stage # to use ArgParser to handle the suite model global arguments, so we just do it manually. def _get_argvalue(arg, args, i): @@ -650,13 +648,15 @@ # to get warnings on suite loading issues before command line is parsed global _warn _warn = True - + elif arg == '-p' or arg == '--primary-suite-path': + global _primary_suite_path + _primary_suite_path = os.path.abspath(_get_argvalue(arg, args, i + 1)) i = i + 1 global _src_suitemodel - _src_suitemodel = SuiteModel._set_suitemodel(src_suitemodel_arg, suitemap_arg) + _src_suitemodel = SuiteModel.set_suitemodel(src_suitemodel_arg, suitemap_arg) global _dst_suitemodel - _dst_suitemodel = SuiteModel._set_suitemodel(dst_suitemodel_arg, suitemap_arg) + _dst_suitemodel = SuiteModel.set_suitemodel(dst_suitemodel_arg, suitemap_arg) class SiblingSuiteModel(SuiteModel): @@ -666,14 +666,14 @@ self._suiteRootDir = suiteRootDir self._create_suitenamemap(option[len('sibling:'):], suitemap) - def _find_suite_dir(self, name): + def find_suite_dir(self, name): return self._search_dir(self._suiteRootDir, self._mxDirName(name)) - def _set_primary_dir(self, d): - SuiteModel._set_primary_dir(self, d) + def set_primary_dir(self, d): + SuiteModel.set_primary_dir(self, d) self._suiteRootDir = dirname(d) - def _importee_dir(self, importer_dir, suitename): + def importee_dir(self, importer_dir, suitename): if self.suitenamemap.has_key(suitename): suitename = self.suitenamemap[suitename] return join(dirname(importer_dir), suitename) @@ -688,10 +688,10 @@ self._primaryDir = primaryDir self._create_suitenamemap(option[len('nested:'):], suitemap) - def _find_suite_dir(self, name): + def find_suite_dir(self, name): return self._search_dir(join(self._primaryDir, self._imported_suites_dirname()), self._mxDirName(name)) - def _importee_dir(self, importer_dir, suitename): + def importee_dir(self, importer_dir, suitename): if self.suitenamemap.has_key(suitename): suitename = self.suitenamemap[suitename] if basename(importer_dir) == basename(self._primaryDir): @@ -703,7 +703,7 @@ else: return join(dirname(importer_dir), suitename) - def _nestedsuites_dirname(self): + def nestedsuites_dirname(self): return self._imported_suites_dirname() class PathSuiteModel(SuiteModel): @@ -722,13 +722,13 @@ suiteurl = pair[0] self.suit_to_url[suitename] = suiteurl - def _find_suite_dir(self, suitename): + def find_suite_dir(self, suitename): if self.suit_to_url.has_key(suitename): return self.suit_to_url[suitename] else: return None - def _importee_dir(self, importer_dir, suitename): + def importee_dir(self, importer_dir, suitename): if suitename in self.suit_to_url: return self.suit_to_url[suitename] else: @@ -740,7 +740,7 @@ self.version = version @staticmethod - def _parse_specification(specification): + def parse_specification(specification): pair = specification.split(',') name = pair[0] if len(pair) > 1: @@ -750,7 +750,7 @@ return SuiteImport(name, version) @staticmethod - def _tostring(name, version): + def tostring(name, version): return name + ',' + version def __str__(self): @@ -767,9 +767,6 @@ self.commands = None self.primary = primary self.name = _suitename(mxDir) # validated in _load_projects - self.version = None # _hg.tip checks current version if not None - # TODO this forces hg to be run every time mx is run - #self.version = _hg.tip(self, False) if load: # load suites bottom up to make sure command overriding works properly self._load_imports() @@ -780,6 +777,10 @@ def __str__(self): return self.name + def version(self, abortOnError=True): + # we do not cache the version + return _hg.tip(self.dir, abortOnError) + def _load_projects(self): libsMap = dict() projsMap = dict() @@ -877,7 +878,7 @@ abort('Missing "suite=" in ' + projectsFile) def _commands_name(self): - return 'mx_' + self.name + return 'mx_' + self.name.replace('-','_') def _find_commands(self, name): commandsPath = join(self.mxDir, name + '.py') @@ -912,7 +913,7 @@ mod.mx_init(self) self.commands = mod - def _visit_imports(self, visitor, **extra_args): + def visit_imports(self, visitor, **extra_args): """ Visitor support for the imports file. For each line of the imports file that specifies an import, the visitor function is @@ -922,7 +923,7 @@ for writing a (possibly) updated import line to the file, and the file is (possibly) updated after all imports are processed. N.B. There is no built-in support for avoiding visiting the same suite multiple times, - as this function only visits the imports of a singkle suite. If a (recursive) visitor function + as this function only visits the imports of a single suite. If a (recursive) visitor function wishes to visit a suite exactly once, it must manage that through extra_args. """ importsFile = join(self.mxDir, 'imports') @@ -937,25 +938,25 @@ if out is not None: out.write(sline + '\n') continue - suite_import = SuiteImport._parse_specification(line.strip()) + suite_import = SuiteImport.parse_specification(line.strip()) visitor(self, suite_import, **extra_args) if out is not None: update_file(importsFile, out.getvalue()) @staticmethod - def _find_and_loadsuite(suite, suite_import, **extra_args): + def _find_and_loadsuite(importing_suite, suite_import, **extra_args): """visitor for the initial suite load""" - importMxDir = _src_suitemodel._find_suite_dir(suite_import.name) + importMxDir = _src_suitemodel.find_suite_dir(suite_import.name) if importMxDir is None: abort('import ' + suite_import.name + ' not found') - suite.imports.append(suite_import) - imported_suite = _loadSuite(importMxDir, False) - if imported_suite.version != suite.version: - warn('import version of ' + imported_suite.name +' does not match tip of ' + suite.version) + importing_suite.imports.append(suite_import) + _loadSuite(importMxDir, False) + # we do not check at this stage whether the tip version of imported_suite + # matches that of the import, since during development, this can and will change def _load_imports(self): - self._visit_imports(self._find_and_loadsuite) + self.visit_imports(self._find_and_loadsuite) def _load_env(self): e = join(self.mxDir, 'env') @@ -1320,7 +1321,7 @@ else: break - envPath = join(_mainSuite.mxDir, 'env') + envPath = join(_primary_suite.mxDir, 'env') if ask_yes_no('Persist this setting by adding "JAVA_HOME=' + javaHome + '" to ' + envPath, 'y'): with open(envPath, 'a') as fp: print >> fp, 'JAVA_HOME=' + javaHome @@ -1341,6 +1342,7 @@ self.add_argument('-v', action='store_true', dest='verbose', help='enable verbose output') self.add_argument('-V', action='store_true', dest='very_verbose', help='enable very verbose output') self.add_argument('-w', action='store_true', dest='warn', help='enable warning messages') + self.add_argument('-p', '--primary-suite-path', help='set the primary suite directory', metavar='') self.add_argument('--dbg', type=int, dest='java_dbg_port', help='make Java processes wait on for a debugger', metavar='') self.add_argument('-d', action='store_const', const=8000, dest='java_dbg_port', help='alias for "-dbg 8000"') self.add_argument('--cp-pfx', dest='cp_prefix', help='class path prefix', metavar='') @@ -1909,7 +1911,7 @@ javaCompliance = java().javaCompliance - defaultEcjPath = join(_mainSuite.mxDir, 'ecj.jar') + defaultEcjPath = join(_primary_suite.mxDir, 'ecj.jar') parser = parser if parser is not None else ArgumentParser(prog='mx build') parser.add_argument('-f', action='store_true', dest='force', help='force build (disables timestamp checking)') @@ -2228,7 +2230,7 @@ if len(modified) != 0: if args.backup: backup = os.path.abspath('eclipseformat.backup.zip') - arcbase = _mainSuite.dir + arcbase = _primary_suite.dir zf = zipfile.ZipFile(backup, 'w', zipfile.ZIP_DEFLATED) for fi in modified: arcname = os.path.relpath(fi.path, arcbase).replace(os.sep, '/') @@ -3125,7 +3127,7 @@ if os.environ.has_key('WORKSPACE'): expected_wsroot = os.environ['WORKSPACE'] else: - expected_wsroot = _mainSuite.dir + expected_wsroot = _primary_suite.dir wsroot = _find_eclipse_wsroot(expected_wsroot) if wsroot is None: @@ -3529,8 +3531,8 @@ if '.hg' in dirnames: dirnames.remove('.hg') # if there are nested suites must not scan those now, as they are not in projectDirs - if _src_suitemodel._nestedsuites_dirname() in dirnames: - dirnames.remove(_src_suitemodel._nestedsuites_dirname()) + if _src_suitemodel.nestedsuites_dirname() in dirnames: + dirnames.remove(_src_suitemodel.nestedsuites_dirname()) elif dirpath in projectDirs: # don't traverse subdirs of an existing project in this suite dirnames[:] = [] @@ -3682,7 +3684,7 @@ names.append(p.name) links = ['-link', 'http://docs.oracle.com/javase/' + str(_java.javaCompliance.value) + '/docs/api/'] - out = join(_mainSuite.dir, docDir) + out = join(_primary_suite.dir, docDir) if args.base is not None: out = join(args.base, docDir) cp = classpath() @@ -3951,7 +3953,7 @@ def sclone(args): """clone a suite repository, and its imported suites""" - _hg.check(True) + _hg.check() parser = ArgumentParser(prog='mx sclone') parser.add_argument('--source', help='url/path of repo containing suite', metavar='') parser.add_argument('--dest', help='destination directory (default basename of source)', metavar='') @@ -3968,11 +3970,11 @@ if args.source is None: # must be primary suite and dest is required - if _mainSuite is None: + if _primary_suite is None: abort('--source missing and no primary suite found') if args.dest is None: abort('--dest required when --source is not given') - source = _mainSuite.dir + source = _primary_suite.dir else: source = args.source @@ -3983,8 +3985,8 @@ dest = os.path.abspath(dest) # We can now set the primary dir for the src/dst suitemodel - _dst_suitemodel._set_primary_dir(dest) - _src_suitemodel._set_primary_dir(source) + _dst_suitemodel.set_primary_dir(dest) + _src_suitemodel.set_primary_dir(source) _sclone(source, dest, None, args.no_imports) @@ -4006,12 +4008,12 @@ # create a Suite (without loading) to enable imports visitor s = Suite(mxDir, False, load=False) if not no_imports: - s._visit_imports(_scloneimports_visitor, source=source) + s.visit_imports(_scloneimports_visitor, source=source) return s def _scloneimports_visitor(s, suite_import, source, **extra_args): """ - cloneimports visitor for Suite._visit_imports. + cloneimports visitor for Suite.visit_imports. The destination information is encapsulated by 's' """ _scloneimports(s, suite_import, source) @@ -4026,21 +4028,21 @@ def _scloneimports(s, suite_import, source): # clone first, then visit imports once we can locate them - importee_source = _src_suitemodel._importee_dir(source, suite_import.name) - importee_dest = _dst_suitemodel._importee_dir(s.dir, suite_import.name) + importee_source = _src_suitemodel.importee_dir(source, suite_import.name) + importee_dest = _dst_suitemodel.importee_dir(s.dir, suite_import.name) if exists(importee_dest): # already exists in the suite model, but may be wrong version importee_suite = _scloneimports_suitehelper(importee_dest) - if suite_import.version is not None and importee_suite.version != suite_import.version: - abort("import version of " + suite_import.name + " in " + s.name + " does not match the version in already existing suite: " + importee_suite.dir) - importee_suite._visit_imports(_scloneimports_visitor, source=importee_source) + if suite_import.version is not None and importee_suite.version() != suite_import.version: + abort("imported version of " + suite_import.name + " in " + s.name + " does not match the version in already existing suite: " + importee_suite.dir) + importee_suite.visit_imports(_scloneimports_visitor, source=importee_source) else: _sclone(importee_source, importee_dest, suite_import.version, False) # _clone handles the recursive visit of the new imports def scloneimports(args): """clone the imports of an existing suite""" - _hg.check(True) + _hg.check() parser = ArgumentParser(prog='mx scloneimports') parser.add_argument('--source', help='url/path of repo containing suite', metavar='') parser.add_argument('nonKWArgs', nargs=REMAINDER, metavar='source [dest]...') @@ -4061,21 +4063,21 @@ # We can now set the primary dir for the dst suitemodel # N.B. source is effectively the destination and the default_path is the (original) source - _dst_suitemodel._set_primary_dir(args.source) - - s._visit_imports(_scloneimports_visitor, source=default_path) + _dst_suitemodel.set_primary_dir(args.source) + + s.visit_imports(_scloneimports_visitor, source=default_path) def _spush_import_visitor(s, suite_import, dest, checks, clonemissing, **extra_args): - """push visitor for Suite._visit_imports""" + """push visitor for Suite.visit_imports""" if dest is not None: - dest = _dst_suitemodel._importee_dir(dest, suite_import.name) + dest = _dst_suitemodel.importee_dir(dest, suite_import.name) _spush(suite(suite_import.name), suite_import, dest, checks, clonemissing) def _spush_check_import_visitor(s, suite_import, **extra_args): - """push check visitor for Suite._visit_imports""" - currentTip = _hg.tip(suite(suite_import.name)) + """push check visitor for Suite.visit_imports""" + currentTip = suite(suite_import.name).version() if currentTip != suite_import.version: - abort('import version of ' + suite_import.name + ' in suite ' + s.name + ' does not match tip') + abort('imported version of ' + suite_import.name + ' in suite ' + s.name + ' does not match tip') def _spush(s, suite_import, dest, checks, clonemissing): if checks: @@ -4084,10 +4086,10 @@ # check imports first if checks: - s._visit_imports(_spush_check_import_visitor) + s.visit_imports(_spush_check_import_visitor) # ok, push imports - s._visit_imports(_spush_import_visitor, dest=dest, checks=checks, clonemissing=clonemissing) + s.visit_imports(_spush_import_visitor, dest=dest, checks=checks, clonemissing=clonemissing) dest_exists = True @@ -4119,7 +4121,7 @@ def spush(args): """push primary suite and all its imports""" - _hg.check(True) + _hg.check() parser = ArgumentParser(prog='mx spush') parser.add_argument('--dest', help='url/path of repo to push to (default as per hg push)', metavar='') parser.add_argument('--no-checks', action='store_true', help='checks on status, versions are disabled') @@ -4142,7 +4144,7 @@ args.nochecks = True if args.dest is not None: - _dst_suitemodel._set_primary_dir(args.dest) + _dst_suitemodel.set_primary_dir(args.dest) _spush(s, None, args.dest, not args.no_checks, args.clonemissing) @@ -4150,29 +4152,29 @@ _supdate(suite(suite_import.name), suite_import) def _supdate(s, suite_import): - s._visit_imports(_supdate_import_visitor) + s.visit_imports(_supdate_import_visitor) run(['hg', '-R', s.dir, 'update']) def supdate(args): """update primary suite and all its imports""" - _hg.check(True) + _hg.check() s = _check_primary_suite() _supdate(s, None) def _scheck_imports_visitor(s, suite_import, update_versions, updated_imports): - """checkimportversions visitor for Suite._visit_imports""" - _scheck_imports(suite(suite_import.name), suite_import, update_versions, updated_imports) - -def _scheck_imports(s, suite_import, update_versions, updated_imports): + """scheckimports visitor for Suite.visit_imports""" + _scheck_imports(s, suite(suite_import.name), suite_import, update_versions, updated_imports) + +def _scheck_imports(importing_suite, imported_suite, suite_import, update_versions, updated_imports): # check imports recursively - s._visit_imports(_scheck_imports_visitor, update_versions=update_versions) - - currentTip = _hg.tip(s) + imported_suite.visit_imports(_scheck_imports_visitor, update_versions=update_versions) + + currentTip = imported_suite.version() if currentTip != suite_import.version: - print('import version of ' + s.name + ' does not match tip' + (': updating' if update_versions else '')) + print('imported version of ' + imported_suite.name + ' in ' + importing_suite.name + ' does not match tip' + (': updating' if update_versions else '')) if update_versions: suite_import.version = currentTip @@ -4184,25 +4186,25 @@ parser = ArgumentParser(prog='mx scheckimports') parser.add_argument('--update-versions', help='update imported version ids', action='store_true') args = parser.parse_args(args) - _check_primary_suite()._visit_imports(_scheck_imports_visitor, update_versions=args.update_versions) + _check_primary_suite().visit_imports(_scheck_imports_visitor, update_versions=args.update_versions) def _spull_import_visitor(s, suite_import, update_versions, updated_imports): - """pull visitor for Suite._visit_imports""" + """pull visitor for Suite.visit_imports""" _spull(suite(suite_import.name), update_versions, updated_imports) def _spull(s, update_versions, updated_imports): - _hg.check(True) + _hg.check() # pull imports first - s._visit_imports(_spull_import_visitor, update_versions=update_versions) + s.visit_imports(_spull_import_visitor, update_versions=update_versions) run(['hg', '-R', s.dir, 'pull', '-u']) if update_versions and updated_imports is not None: - tip = _hg.tip(s) - updated_imports.write(SuiteImport._tostring(s.name, tip) + '\n') + tip = s.version() + updated_imports.write(SuiteImport.tostring(s.name, tip) + '\n') def spull(args): """pull primary suite and all its imports""" - _hg.check(True) + _hg.check() parser = ArgumentParser(prog='mx spull') parser.add_argument('--update-versions', action='store_true', help='update version ids of imported suites') args = parser.parse_args(args) @@ -4383,48 +4385,74 @@ return mxDir def _check_primary_suite(): - if _mainSuite is None: + if _primary_suite is None: abort('no primary suite found') else: - return _mainSuite + return _primary_suite def _needs_primary_suite(command): return not command.startswith("sclone") -def _findPrimarySuiteMxDir(): - # try current working directory first, the look up the tree - curdir = os.getcwd() - while curdir: - mxDir = _is_suite_dir(curdir) +def _needs_primary_suite_cl(): + return not any("sclone" in s for s in sys.argv[1:]) + +def _findPrimarySuiteMxDirFrom(d): + """ search for a suite directory upwards from 'd' """ + while d: + mxDir = _is_suite_dir(d) if mxDir is not None: return mxDir - parent = dirname(curdir) - if curdir == parent: + parent = dirname(d) + if d == parent: return None - curdir = parent + d = parent return None +def _findPrimarySuiteMxDir(): + # check for explicit setting + if _primary_suite_path is not None: + mxDir = _is_suite_dir(_primary_suite_path) + if mxDir is not None: + return mxDir + else: + abort(_primary_suite_path + ' does not contain an mx suite') + + # try current working directory first + mxDir = _findPrimarySuiteMxDirFrom(os.getcwd()) + if mxDir is not None: + return mxDir + # backwards compatibility: search from path of this file + return _findPrimarySuiteMxDirFrom(dirname(__file__)) + def main(): - SuiteModel._parse_options() + SuiteModel.parse_options() global _hg _hg = HgConfig() + primary_suite_error = 'no primary suite found' primarySuiteMxDir = _findPrimarySuiteMxDir() if primarySuiteMxDir: - _src_suitemodel._set_primary_dir(dirname(primarySuiteMxDir)) - global _mainSuite - _mainSuite = _loadSuite(primarySuiteMxDir, True) + _src_suitemodel.set_primary_dir(dirname(primarySuiteMxDir)) + global _primary_suite + _primary_suite = _loadSuite(primarySuiteMxDir, True) + else: + # in general this is an error, except for the sclone/scloneimports commands, + # and an extensions command will likely not parse in this case, as any extra arguments + # will not have been added to _argParser. + # If the command line does not contain a string matching one of the exceptions, we can safely abort, + # but not otherwise, as we can't be sure the string isn't in a value for some other option. + if _needs_primary_suite_cl(): + abort(primary_suite_error) opts, commandAndArgs = _argParser._parse_cmd_line() if primarySuiteMxDir is None: - msg = 'no primary suite found' if len(commandAndArgs) > 0 and _needs_primary_suite(commandAndArgs[0]): - abort(msg) + abort(primary_suite_error) else: - warn(msg) + warn(primary_suite_error) global _opts, _java _opts = opts