# HG changeset patch # User Mick Jordan # Date 1382065165 25200 # Node ID a5d83166dca6e56d89ba46a56e1283fad7974bb8 # Parent 67566565053a26e85690f9e990b1c9d665e11ad1 mxtool hg support for suites diff -r 67566565053a -r a5d83166dca6 mxtool/mx.py --- a/mxtool/mx.py Thu Oct 17 22:22:19 2013 +0200 +++ b/mxtool/mx.py Thu Oct 17 19:59:25 2013 -0700 @@ -55,17 +55,18 @@ commands.py Suite specific extensions to the commands available to mx. - includes + imports Other suites to be loaded. This is recursive. Each - line in an includes file is a path to a suite directory. + line in an imports file is the name of an imported suite. + The suite is located using a SuiteModel (cf searchpath) env A set of environment variable definitions. These override any existing environment variables. Common properties set here include JAVA_HOME and IGNORED_PROJECTS. -The includes and env files are typically not put under version control -as they usually contain local file-system paths. +The env file is typically not put under version control +as it usually contain local file-system paths. The projects file is like the pom.xml file from Maven except that it is a properties file (not XML). Each non-comment line @@ -156,9 +157,12 @@ _suites = dict() _annotationProcessors = None _mainSuite = None +_src_suitemodel = None +_dst_suitemodel = None _opts = None _java = None _check_global_structures = True # can be set False to allow suites with duplicate definitions to load without aborting +_warn = False """ @@ -493,38 +497,193 @@ deps.append(self) return deps +class SuiteModel: + """ + Defines how to locate a URL/path for a suite, including imported suites. + Conceptually a SuiteModel is defined by a primary suite URL/path and a + map from suite name to URL/path for imported suites. + Subclasses define a specfic implementation. + """ + def __init__(self): + self.primaryDir = None + self.suitenamemap = {} + + def _find_suite_dir(self, suitename): + """locates the URL/path for suitename or None if not found""" + abort('_find_suite_dir not implemented') + + def _set_primary_dir(self, d): + """informs that d is the primary suite directory""" + self._primaryDir = d + + def _importee_dir(self, importer_dir, suitename): + """returns the directory path for an import of suitename, given importer_dir""" + abort('_importee_dir not implemented') + + def _nestedsuites_dirname(self): + """Returns the dirname that contains any nested suites if the model supports that""" + return None + + def _mxDirName(self, name): + # temporary workaround until mx.graal exists + if name == 'graal': + return 'mx' + else: + return 'mx.' + name + + def _search_dir(self, searchDir, mxDirName): + for dd in os.listdir(searchDir): + sd = _is_suite_dir(join(searchDir, dd), mxDirName) + if sd is not None: + return sd + + def _create_suitenamemap(self, optionspec, suitemap): + """Three ways to specify a suite name mapping, in order of precedence: + 1. Explicitly in optionspec. + 2. In suitemap. + 3. in MXSUITEMAP environment variable. + """ + if optionspec != '': + spec = optionspec + elif suitemap is not None: + spec = suitemap + elif get_env('MXSUITEMAP') is not None: + spec = get_env('MXSUITEMAP') + else: + return + pairs = spec.split(',') + for pair in pairs: + mappair = pair.split('=') + self.suitenamemap[mappair[0]] = mappair[1] + +class SiblingSuiteModel(SuiteModel): + """All suites are siblings in the same parent directory, recorded as _suiteRootDir""" + def __init__(self, suiteRootDir, option, suitemap): + SuiteModel.__init__(self) + self._suiteRootDir = suiteRootDir + self._create_suitenamemap(option[len('sibling:'):], suitemap) + + def _find_suite_dir(self, name): + return self._search_dir(self._suiteRootDir, self._mxDirName(name)) + + def _set_primary_dir(self, d): + SuiteModel._set_primary_dir(self, d) + self._suiteRootDir = dirname(d) + + def _importee_dir(self, importer_dir, suitename): + if self.suitenamemap.has_key(suitename): + suitename = self.suitenamemap[suitename] + return join(dirname(importer_dir), suitename) + +class NestedImportsSuiteModel(SuiteModel): + """Imported suites are all siblings in an 'imported_suites' directory of the primary suite""" + def _imported_suites_dirname(self): + return "imported_suites" + + def __init__(self, primaryDir, option, suitemap): + SuiteModel.__init__(self) + self._primaryDir = primaryDir + self._create_suitenamemap(option[len('nested:'):], suitemap) + + def _find_suite_dir(self, name): + return self._search_dir(join(self._primaryDir, self._imported_suites_dirname()), self._mxDirName(name)) + + def _importee_dir(self, importer_dir, suitename): + if self.suitenamemap.has_key(suitename): + suitename = self.suitenamemap[suitename] + if basename(importer_dir) == basename(self._primaryDir): + # primary is importer + this_imported_suites_dirname = join(importer_dir, self._imported_suites_dirname()) + if not exists(this_imported_suites_dirname): + os.mkdir(this_imported_suites_dirname) + return join(this_imported_suites_dirname, suitename) + else: + return join(dirname(importer_dir), suitename) + + def _nestedsuites_dirname(self): + return self._imported_suites_dirname() + +class PathSuiteModel(SuiteModel): + """The most general model. Uses a map from suitename to URL/path provided by the user""" + def __init__(self, path): + SuiteModel.__init__(self) + paths = path.split(',') + self.suit_to_url = {} + for path in paths: + pair = path.split('=') + if len(pair) > 1: + suitename = pair[0] + suiteurl = pair[1] + else: + suitename = basename(pair[0]) + suiteurl = pair[0] + self.suit_to_url[suitename] = suiteurl + + def _find_suite_dir(self, suitename): + if self.suit_to_url.has_key(suitename): + return self.suit_to_url[suitename] + else: + return None + + def _importee_dir(self, importer_dir, suitename): + if suitename in self.suit_to_url: + return self.suit_to_url[suitename] + else: + abort('suite ' + suitename + ' not found') + +class SuiteImport: + def __init__(self, name, version): + self.name = name + self.version = version + + @staticmethod + def _parse_specification(specification): + pair = specification.split(',') + name = pair[0] + if len(pair) > 1: + version = pair[1] + else: + version = None + return SuiteImport(name, version) + + @staticmethod + def _tostring(name, version): + return name + ',' + version + + def _self_tostring(self): + return self.name + ',' + self.version + class Suite: - def __init__(self, d, mxDir, primary): - self.dir = d + def __init__(self, mxDir, primary, load=True): + self.dir = dirname(mxDir) self.mxDir = mxDir self.projects = [] self.libs = [] self.dists = [] - self.includes = [] + self.imports = [] self.commands = None self.primary = primary - self._load_env(mxDir) - self._load_commands(mxDir) - self._load_includes(mxDir) - self.name = d # re-initialized in _load_projects + self.name = _suitename(mxDir) # validated in _load_projects + self.version = None # _hgtip checks current version if not None + self.version = _hgtip(self, False) + if load: + # load suites bottom up to make sure command overriding works properly + self._load_imports() + self._load_env() + self._load_commands() + _suites[self.name] = self def __str__(self): return self.name - def _load_projects(self, mxDir): + def _load_projects(self): libsMap = dict() projsMap = dict() distsMap = dict() - projectsFile = join(mxDir, 'projects') + projectsFile = join(self.mxDir, 'projects') if not exists(projectsFile): return - def _find_suite_key(): - for items in _suites.items(): - if items[1].dir == self.dir: - return items[0] - raise KeyError - with open(projectsFile) as f: prefix = '' for line in f: @@ -544,10 +703,7 @@ if parts[0] != 'suite': abort('Single part property must be "suite": ' + key) if self.name != value: - currentKey = _find_suite_key() - _suites.pop(currentKey) - self.name = value - _suites[value] = self + abort('suite name in project file does not match ' + _suitename(self.mxDir)) continue if len(parts) != 3: abort('Property name does not have 3 parts separated by "@": ' + key) @@ -616,38 +772,89 @@ if self.name is None: abort('Missing "suite=" in ' + projectsFile) - def _load_commands(self, mxDir): - commandsPath = join(mxDir, 'commands.py') + def _commands_name(self): + return 'mx_' + self.name + + def _find_commands(self, name): + commandsPath = join(self.mxDir, name + '.py') if exists(commandsPath): + return name + else: + return None + + def _load_commands(self): + commandsName = self._find_commands(self._commands_name()) + if commandsName is None: + # backwards compatibility + commandsName = self._find_commands('commands') + if commandsName is not None: + if commandsName in sys.modules: + abort(commandsName + '.py in suite ' + self.name + ' duplicates ' + sys.modules[commandsName].__file__) # temporarily extend the Python path - sys.path.insert(0, mxDir) - mod = __import__('commands') - - self.commands = sys.modules.pop('commands') - sys.modules[join(mxDir, 'commands')] = self.commands + sys.path.insert(0, self.mxDir) + mod = __import__(commandsName) + + self.commands = sys.modules.pop(commandsName) + sys.modules[commandsName] = self.commands # revert the Python path del sys.path[0] if not hasattr(mod, 'mx_init'): - abort(commandsPath + ' must define an mx_init(env) function') + abort(commandsName + '.py in suite ' + self.name + ' must define an mx_init(suite) function') if hasattr(mod, 'mx_post_parse_cmd_line'): self.mx_post_parse_cmd_line = mod.mx_post_parse_cmd_line mod.mx_init(self) self.commands = mod - def _load_includes(self, mxDir): - includes = join(mxDir, 'includes') - if exists(includes): - with open(includes) as f: + def _visit_imports(self, visitor, **extra_args): + """ + Visitor support for the imports file. + For each line of the imports file that specifies an import, the visitor function is + called with this suite, a SuiteImport instance created from the line and any extra args + passed to this call. In addition, if extra_args contains a key 'update_versions' that is True, + a StringIO value is added to extra_args with key 'updated_imports', and the visitor is responsible + for writing a (possibly) updated import line to the file, and the file is (possibly) updated after + all imports are processed. + N.B. There is no built-in support for avoiding visiting the same suite multiple times, + as this function only visits the imports of a singkle suite. If a (recursive) visitor function + wishes to visit a suite exactly once, it must manage that through extra_args. + """ + importsFile = join(self.mxDir, 'imports') + if exists(importsFile): + update_versions = extra_args.has_key('update_versions') and extra_args['update_versions'] + out = StringIO.StringIO() if update_versions else None + extra_args['updated_imports'] = out + with open(importsFile) as f: for line in f: - include = expandvars_in_property(line.strip()) - self.includes.append(include) - _loadSuite(os.path.abspath(include), False) - - def _load_env(self, mxDir): - e = join(mxDir, 'env') + sline = line.strip() + if len(sline) == 0 or sline.startswith('#'): + if out is not None: + out.write(sline + '\n') + continue + suite_import = SuiteImport._parse_specification(line.strip()) + visitor(self, suite_import, **extra_args) + + if out is not None: + update_file(importsFile, out.getvalue()) + + @staticmethod + def _find_and_loadsuite(suite, suite_import, **extra_args): + """visitor for the initial suite load""" + importMxDir = _src_suitemodel._find_suite_dir(suite_import.name) + if importMxDir is None: + abort('import ' + suite_import.name + ' not found') + suite.imports.append(suite_import) + imported_suite = _loadSuite(importMxDir, False) + if imported_suite.version != suite.version: + warn('import version of ' + imported_suite.name +' does not match tip of ' + suite.version) + + def _load_imports(self): + self._visit_imports(self._find_and_loadsuite) + + def _load_env(self): + e = join(self.mxDir, 'env') if exists(e): with open(e) as f: lineNum = 0 @@ -661,8 +868,8 @@ os.environ[key.strip()] = expandvars_in_property(value.strip()) def _post_init(self, opts): - self._load_projects(self.mxDir) - # set the global data structures, checking for conflicts unless _global_structures is False + self._load_projects() + # set the global data structures, checking for conflicts unless _check_global_structures is False for p in self.projects: existing = _projects.get(p.name) if existing is not None and _check_global_structures: @@ -680,7 +887,7 @@ if existing is not None and _check_global_structures: # allow redefinition, so use path from existing # abort('cannot redefine distribution ' + d.name) - print('WARNING: distribution ' + d.name + ' redefined') + warn('distribution ' + d.name + ' redefined') d.path = existing.path _dists[d.name] = d if hasattr(self, 'mx_post_parse_cmd_line'): @@ -777,28 +984,16 @@ else: abort('Unknown operating system ' + sys.platform) -def _loadSuite(d, primary=False): +def _loadSuite(mxDir, primary=False): """ - Load a suite from the 'mx' or 'mx.bbb' subdirectory of d, where 'bbb' is basename of d + Load a suite from 'mxDir'. """ - mxDefaultDir = join(d, 'mx') - name = os.path.basename(d) - mxTaggedDir = mxDefaultDir + '.' + name - mxDir = None - if exists(mxTaggedDir) and isdir(mxTaggedDir): - mxDir = mxTaggedDir - else: - if exists(mxDefaultDir) and isdir(mxDefaultDir): - mxDir = mxDefaultDir - - - if mxDir is None: - return None - if len([s for s in _suites.itervalues() if s.dir == d]) == 0: - s = Suite(d, mxDir, primary) - # N.B. this will be updated once the projects file has been read - _suites[name] = s - return s + for s in _suites.itervalues(): + if s.mxDir == mxDir: + return s + # create the new suite + s = Suite(mxDir, primary) + return s def suites(opt_limit_to_suite=False): """ @@ -822,6 +1017,7 @@ abort('suite named ' + name + ' not found') return s + def projects_from_names(projectNames): """ Get the list of projects corresponding to projectNames; all projects if None @@ -1020,7 +1216,7 @@ else: break - envPath = join(_mainSuite.dir, 'mx', 'env') + envPath = join(_mainSuite.mxDir, 'env') if ask_yes_no('Persist this setting by adding "JAVA_HOME=' + javaHome + '" to ' + envPath, 'y'): with open(envPath, 'a') as fp: print >> fp, 'JAVA_HOME=' + javaHome @@ -1028,7 +1224,6 @@ return javaHome class ArgParser(ArgumentParser): - # Override parent to append the list of available commands def format_help(self): return ArgumentParser.format_help(self) + _format_commands() @@ -1041,6 +1236,7 @@ self.add_argument('-v', action='store_true', dest='verbose', help='enable verbose output') self.add_argument('-V', action='store_true', dest='very_verbose', help='enable very verbose output') + self.add_argument('-w', action='store_true', dest='warn', help='enable warning messages') self.add_argument('--dbg', type=int, dest='java_dbg_port', help='make Java processes wait on for a debugger', metavar='') self.add_argument('-d', action='store_const', const=8000, dest='java_dbg_port', help='alias for "-dbg 8000"') self.add_argument('--cp-pfx', dest='cp_prefix', help='class path prefix', metavar='') @@ -1052,6 +1248,9 @@ self.add_argument('--java-home', help='bootstrap JDK installation directory (must be JDK 6 or later)', metavar='') self.add_argument('--ignore-project', action='append', dest='ignored_projects', help='name of project to ignore', metavar='', default=[]) self.add_argument('--suite', action='append', dest='specific_suites', help='limit command to given suite', default=[]) + self.add_argument('--src-suitemodel', help='mechanism for locating imported suites', metavar='', default='sibling') + self.add_argument('--dst-suitemodel', help='mechanism for placing cloned/pushed suites', metavar='', default='sibling') + self.add_argument('--suitemap', help='explicit remapping of suite names', metavar='') if get_os() != 'windows': # Time outs are (currently) implemented with Unix specific functionality self.add_argument('--timeout', help='timeout (in seconds) for command', type=int, default=0, metavar='') @@ -1092,6 +1291,33 @@ def _handle_conflict_resolve(self, action, conflicting_actions): self._handle_conflict_error(action, conflicting_actions) +class SMArgParser(ArgParser): + """Parser that just looks for suitemodel options, which must happen before suites are loaded""" + def __init__(self): + ArgParser.__init__(self) + + def _suitemodel(self, option, suitemap): + if option.startswith('sibling'): + return SiblingSuiteModel(os.getcwd(), option, suitemap) + elif option.startswith('nested'): + return NestedImportsSuiteModel(os.getcwd(), option, suitemap) + elif option.startswith('path'): + return PathSuiteModel(option[len('path:'):]) + else: + abort('unknown suitemodel type: ' + option) + + def _parse_suitemodel_options(self): + # the command line may contains args added by suites, so we only parse the currently known args + opts = self.parse_known_args()[0] + # set this early + global _warn + _warn = opts.warn + + global _src_suitemodel + _src_suitemodel = self._suitemodel(opts.src_suitemodel, opts.suitemap) + global _dst_suitemodel + _dst_suitemodel = self._suitemodel(opts.dst_suitemodel, opts.suitemap) + def _format_commands(): msg = '\navailable commands:\n\n' for cmd in sorted(_commands.iterkeys()): @@ -1606,7 +1832,7 @@ javaCompliance = java().javaCompliance - defaultEcjPath = join(_mainSuite.dir, 'mx', 'ecj.jar') + defaultEcjPath = join(_mainSuite.mxDir, 'ecj.jar') parser = parser if parser is not None else ArgumentParser(prog='mx build') parser.add_argument('-f', action='store_true', dest='force', help='force build (disables timestamp checking)') @@ -1803,7 +2029,7 @@ jdtProperties = join(p.dir, '.settings', 'org.eclipse.jdt.core.prefs') - rootJdtProperties = join(p.suite.dir, 'mx', 'eclipse-settings', 'org.eclipse.jdt.core.prefs') + rootJdtProperties = join(p.suite.mxDir, 'eclipse-settings', 'org.eclipse.jdt.core.prefs') if not exists(jdtProperties) or os.path.getmtime(jdtProperties) < os.path.getmtime(rootJdtProperties): # Try to fix a missing properties file by running eclipseinit eclipseinit([], buildProcessorJars=False) @@ -2122,7 +2348,7 @@ The exit code of this command reflects how many files were updated.""" changedFiles = 0 - for s in suites(): + for s in suites(True): projectsFile = join(s.mxDir, 'projects') if not exists(projectsFile): continue @@ -2763,12 +2989,20 @@ launchOut.element('booleanAttribute', {'key' : 'org.eclipse.debug.ui.ATTR_CONSOLE_OUTPUT_ON', 'value': consoleOn}) launchOut.element('booleanAttribute', {'key' : 'org.eclipse.debug.ui.ATTR_LAUNCH_IN_BACKGROUND', 'value': 'true' if async else 'false'}) - baseDir = dirname(dirname(os.path.abspath(__file__))) + # expect to find the OS command to invoke mx in the same directory + baseDir = dirname(os.path.abspath(__file__)) cmd = 'mx.sh' if get_os() == 'windows': cmd = 'mx.cmd' - launchOut.element('stringAttribute', {'key' : 'org.eclipse.ui.externaltools.ATTR_LOCATION', 'value': join(baseDir, cmd) }) + cmdPath = join(baseDir, cmd) + if not os.path.exists(cmdPath): + # backwards compatibility for when the commands lived in parent of mxtool + cmdPath = join(dirname(baseDir), cmd) + if not os.path.exists(cmdPath): + abort('cannot locate ' + cmd) + + launchOut.element('stringAttribute', {'key' : 'org.eclipse.ui.externaltools.ATTR_LOCATION', 'value': cmdPath}) launchOut.element('stringAttribute', {'key' : 'org.eclipse.ui.externaltools.ATTR_RUN_BUILD_KINDS', 'value': 'auto,full,incremental'}) launchOut.element('stringAttribute', {'key' : 'org.eclipse.ui.externaltools.ATTR_TOOL_ARGUMENTS', 'value': mxCommand}) launchOut.element('booleanAttribute', {'key' : 'org.eclipse.ui.externaltools.ATTR_TRIGGERS_CONFIGURED', 'value': 'true'}) @@ -3210,18 +3444,24 @@ """find directories corresponding to deleted Java projects and delete them""" for suite in suites(True): projectDirs = [p.dir for p in suite.projects] - for root, dirnames, files in os.walk(suite.dir): - currentDir = join(suite.dir, root) - if currentDir in projectDirs: - # don't traverse subdirs of an existing project + for dirpath, dirnames, files in os.walk(suite.dir): + if dirpath == suite.dir: + # no point in traversing .hg + if '.hg' in dirnames: + dirnames.remove('.hg') + # if there are nested suites must not scan those now, as they are not in projectDirs + if _src_suitemodel._nestedsuites_dirname() in dirnames: + dirnames.remove(_src_suitemodel._nestedsuites_dirname()) + elif dirpath in projectDirs: + # don't traverse subdirs of an existing project in this suite dirnames[:] = [] else: projectConfigFiles = frozenset(['.classpath', 'nbproject']) indicators = projectConfigFiles.intersection(files) if len(indicators) != 0: - if not sys.stdout.isatty() or ask_yes_no(currentDir + ' looks like a removed project -- delete it', 'n'): - shutil.rmtree(currentDir) - log('Deleted ' + currentDir) + if not sys.stdout.isatty() or ask_yes_no(dirpath + ' looks like a removed project -- delete it', 'n'): + shutil.rmtree(dirpath) + log('Deleted ' + dirpath) def javadoc(args, parser=None, docDir='javadoc', includeDeps=True, stdDoclet=True): """generate javadoc for some/all Java projects""" @@ -3625,9 +3865,292 @@ if exists(tmpbase): shutil.rmtree(tmpbase) +def _kwArg(kwargs): + if len(kwargs) > 0: + return kwargs.pop(0) + return None + +def sclone(args): + """clone a suite repository, and its imported suites""" + parser = ArgumentParser(prog='mx sclone') + parser.add_argument('--source', help='url/path of repo containing suite', metavar='') + parser.add_argument('--dest', help='destination directory (default basename of source)', metavar='') + parser.add_argument("--no-imports", action='store_true', help='do not clone imported suites') + parser.add_argument('nonKWArgs', nargs=REMAINDER, metavar='source [dest]...') + args = parser.parse_args(args) + # check for non keyword args + if args.source is None: + args.source = _kwArg(args.nonKWArgs) + if args.dest is None: + args.dest = _kwArg(args.nonKWArgs) + if len(args.nonKWArgs) > 0: + abort('unrecognized args: ' + ' '.join(args.nonKWArgs)) + + if args.source is None: + # must be primary suite and dest is required + if _mainSuite is None: + abort('--source missing and no primary suite found') + if args.dest is None: + abort('--dest required when --source is not given') + source = _mainSuite.dir + else: + source = args.source + + if args.dest is not None: + dest = args.dest + else: + dest = basename(source) + + dest = os.path.abspath(dest) + # We can now set the primary dir for the src/dst suitemodel + _dst_suitemodel._set_primary_dir(dest) + _src_suitemodel._set_primary_dir(source) + + _sclone(source, dest, None, args.no_imports) + +def _sclone(source, dest, version, no_imports): + cmd = ['hg', 'clone'] + if version is not None: + cmd.append('-r') + cmd.append(version) + cmd.append(source) + cmd.append(dest) + + run(cmd) + + mxDir = _is_suite_dir(dest) + if mxDir is None: + warn(source + ' is not an mx suite') + return None + + # create a Suite (without loading) to enable imports visitor + s = Suite(mxDir, False, load=False) + if not no_imports: + s._visit_imports(_scloneimports_visitor, source=source) + return s + +def _scloneimports_visitor(s, suite_import, source, **extra_args): + """ + cloneimports visitor for Suite._visit_imports. + The destination information is encapsulated by 's' + """ + _scloneimports(s, suite_import, source) + +def _scloneimports_suitehelper(sdir): + mxDir = _is_suite_dir(sdir) + if mxDir is None: + abort(sdir + ' is not an mx suite') + else: + # create a Suite (without loading) to enable imports visitor + return Suite(mxDir, False, load=False) + +def _scloneimports(s, suite_import, source): + # clone first, then visit imports once we can locate them + importee_source = _src_suitemodel._importee_dir(source, suite_import.name) + importee_dest = _dst_suitemodel._importee_dir(s.dir, suite_import.name) + if exists(importee_dest): + importee_suite = _scloneimports_suitehelper(importee_dest) + importee_suite._visit_imports(_scloneimports_visitor, source=importee_source) + else: + _sclone(importee_source, importee_dest, suite_import.version, False) + # _clone handles the recursive visit of the new imports + +def scloneimports(args): + """clone the imports of an existing suite""" + parser = ArgumentParser(prog='mx scloneimports') + parser.add_argument('--source', help='url/path of repo containing suite', metavar='') + parser.add_argument('nonKWArgs', nargs=REMAINDER, metavar='source [dest]...') + args = parser.parse_args(args) + # check for non keyword args + if args.source is None: + args.source = _kwArg(args.nonKWArgs) + + if not os.path.isdir(args.source): + abort(args.source + ' is not a directory') + + s = _scloneimports_suitehelper(args.source) + + default_path = _hgdefault_push(args.source) + + if default_path is None: + abort('no default path in ' + join(args.source, '.hg', 'hgrc')) + + # We can now set the primary dir for the dst suitemodel + # N.B. source is effectively the destination and the default_path is the (original) source + _dst_suitemodel._set_primary_dir(args.source) + + s._visit_imports(_scloneimports_visitor, source=default_path) + +def _spush_import_visitor(s, suite_import, dest, checks, clonemissing, **extra_args): + """push visitor for Suite._visit_imports""" + if dest is not None: + dest = _dst_suitemodel._importee_dir(dest, suite_import.name) + _spush(suite(suite_import.name), suite_import, dest, checks, clonemissing) + +def _spush_check_import_visitor(s, suite_import, **extra_args): + """push check visitor for Suite._visit_imports""" + currentTip = _hgtip(suite(suite_import.name)) + if currentTip != suite_import.version: + abort('import version of ' + suite_import.name + ' in suite ' + s.name + ' does not match tip') + +def _spush(s, suite_import, dest, checks, clonemissing): + if checks: + if not _hgcanpush(s): + abort('working directory ' + s.dir + ' contains uncommitted changes, push aborted') + + # check imports first + if checks: + s._visit_imports(_spush_check_import_visitor) + + # ok, push imports + s._visit_imports(_spush_import_visitor, dest=dest, checks=checks, clonemissing=clonemissing) + + dest_exists = True + + if clonemissing: + if not os.path.exists(dest): + dest_exists = False + + def add_version(cmd, suite_import): + if suite_import is not None and suite_import.version is not None: + cmd.append('-r') + cmd.append(suite_import.version) + + if dest_exists: + cmd = ['hg', '-R', s.dir, 'push'] + add_version(cmd, suite_import) + if dest is not None: + cmd.append(dest) + rc = run(cmd, nonZeroIsFatal=False) + if rc != 0: + # rc of 1 not an error, means no changes + if rc != 1: + abort("push failed, exit code " + str(rc)) + else: + cmd = ['hg', 'clone'] + add_version(cmd, suite_import) + cmd.append(s.dir) + cmd.append(dest) + run(cmd) + +def spush(args): + """push primary suite and all its imports""" + parser = ArgumentParser(prog='mx spush') + parser.add_argument('--dest', help='url/path of repo to push to (default as per hg push)', metavar='') + parser.add_argument('--no-checks', action='store_true', help='checks on status, versions are disabled') + parser.add_argument('--clonemissing', action='store_true', help='clone missing imported repos at destination (forces --no-checks)') + parser.add_argument('nonKWArgs', nargs=REMAINDER, metavar='source [dest]...') + args = parser.parse_args(args) + if args.dest is None: + args.dest = _kwArg(args.nonKWArgs) + if len(args.nonKWArgs) > 0: + abort('unrecognized args: ' + ' '.join(args.nonKWArgs)) + + if args.dest is not None and not os.path.isdir(args.dest): + abort('destination must be a directory') + + s = _check_primary_suite() + + if args.clonemissing: + if args.dest is None: + abort('--dest required with --clonemissing') + args.nochecks = True + + if args.dest is not None: + _dst_suitemodel._set_primary_dir(args.dest) + + _spush(s, None, args.dest, not args.nochecks, args.clonemissing) + +def _supdate_import_visitor(s, suite_import, **extra_args): + _supdate(suite(suite_import.name), suite_import) + +def _supdate(s, suite_import): + s._visit_imports(_supdate_import_visitor) + + run(['hg', '-R', s.dir, 'update']) + +def supdate(args): + """update primary suite and all its imports""" + + s = _check_primary_suite() + + _supdate(s, None) + +def _scheck_imports_visitor(s, suite_import, update_versions, updated_imports): + """checkimportversions visitor for Suite._visit_imports""" + _scheck_imports(suite(suite_import.name), suite_import, update_versions, updated_imports) + +def _scheck_imports(s, suite_import, update_versions, updated_imports): + # check imports recursively + s._visit_imports(_scheck_imports_visitor, update_versions=update_versions) + + currentTip = _hgtip(s) + if currentTip != suite_import.version: + print('import version of ' + s.name + ' does not match tip' + (': updating' if update_versions else '')) + + if update_versions: + suite_import.version = currentTip + line = suite_import._self_tostring() + updated_imports.write(line + '\n') + +def scheckimports(args): + """check that suite import versions are up to date""" + parser = ArgumentParser(prog='mx scheckimports') + parser.add_argument('--update-versions', help='update imported version ids', action='store_true') + args = parser.parse_args(args) + _check_primary_suite()._visit_imports(_scheck_imports_visitor, update_versions=args.update_versions) + +def _hgtip(s, abortOnError=True): + try: + version = subprocess.check_output(['hg', 'tip', '-R', s.dir, '--template', '{node}']) + if s.version is not None and s.version != version: + abort('version of suite ' + s.name +' has changed during run') + return version + except subprocess.CalledProcessError: + if abortOnError: + abort('failed to get tip revision id') + else: + return None + +def _hgcanpush(s): + try: + output = subprocess.check_output(['hg', '-R', s.dir, 'status']) + # super strict + return output == '' + except subprocess.CalledProcessError: + return False + +def _hgdefault_push(sdir): + with open(join(sdir, '.hg', 'hgrc')) as f: + for line in f: + line = line.rstrip() + if line.startswith('default = '): + return line[len('default = '):] + return None + +def _spull_import_visitor(s, suite_import, update_versions, updated_imports): + """pull visitor for Suite._visit_imports""" + _spull(suite(suite_import.name), update_versions, updated_imports) + +def _spull(s, update_versions, updated_imports): + # pull imports first + s._visit_imports(_spull_import_visitor, update_versions=update_versions) + + run(['hg', '-R', s.dir, 'pull', '-u']) + if update_versions and updated_imports is not None: + tip = _hgtip(s) + updated_imports.write(SuiteImport._tostring(s.name, tip) + '\n') + +def spull(args): + """pull primary suite and all its imports""" + parser = ArgumentParser(prog='mx spull') + parser.add_argument('--update-versions', action='store_true', help='update version ids of imported suites') + args = parser.parse_args(args) + + _spull(_check_primary_suite(), args.update_versions, None) + def findclass(args, logToConsole=True): """find all classes matching a given substring""" - matches = [] for entry, filename in classpath_walk(includeBootClasspath=True): if filename.endswith('.class'): @@ -3734,17 +4257,19 @@ def update_commands(suite, new_commands): for key, value in new_commands.iteritems(): - if _commands.has_key(key) and not suite.primary: - pass - # print("WARNING: attempt to redefine command '" + key + "' in suite " + suite.dir) - else: - _commands[key] = value + if _commands.has_key(key): + warn("redefining command '" + key + "' in suite " + suite.name) + _commands[key] = value + +def warn(msg): + if _warn: + print('WARNING: ' + msg) # Table of commands in alphabetical order. # Keys are command names, value are lists: [, , ...] # If any of the format args are instances of Callable, then they are called with an 'env' are before being # used in the call to str.format(). -# Extensions should update this table directly +# Suite extensions should not update this table directly, but use update_commands _commands = { 'about': [about, ''], 'build': [build, '[options]'], @@ -3760,6 +4285,12 @@ 'ideinit': [ideinit, ''], 'archive': [archive, '[options]'], 'projectgraph': [projectgraph, ''], + 'sclone': [sclone, '[options]'], + 'scheckimports': [scheckimports, ''], + 'scloneimports': [scloneimports, '[options]'], + 'spull': [spull, '[options'], + 'spush': [spush, '[options'], + 'supdate': [supdate, ''], 'pylint': [pylint, ''], 'javap': [javap, ''], 'javadoc': [javadoc, '[options]'], @@ -3770,36 +4301,68 @@ _argParser = ArgParser() -def _findPrimarySuite(): - def is_suite_dir(d): +def _suitename(mxDir): + base = os.path.basename(mxDir) + parts = base.split('.') + # temporary workaround until mx.graal exists + if len(parts) == 1: + return 'graal' + else: + return parts[1] + +def _is_suite_dir(d, mxDirName=None): + """ + Checks if d contains a suite. + If mxDirName is None, matches any suite name, otherwise checks for exactly that suite. + """ + if os.path.isdir(d): for f in os.listdir(d): - if f == 'mx' or fnmatch.fnmatch(f, 'mx.*'): + if (mxDirName == None and (f == 'mx' or fnmatch.fnmatch(f, 'mx.*'))) or f == mxDirName: mxDir = join(d, f) if exists(mxDir) and isdir(mxDir) and exists(join(mxDir, 'projects')): - return dirname(mxDir) - - - # try current working directory first - if is_suite_dir(os.getcwd()): - return os.getcwd() - - # now search path of my executable - me = sys.argv[0] - parent = dirname(me) - while parent: - if is_suite_dir(parent): - return parent - parent = dirname(parent) + return mxDir + +def _check_primary_suite(): + if _mainSuite is None: + abort('no primary suite found') + else: + return _mainSuite + +def _needs_primary_suite(command): + return not command.startswith("sclone") + +def _findPrimarySuiteMxDir(): + # try current working directory first, the look up the tree + curdir = os.getcwd() + while curdir: + mxDir = _is_suite_dir(curdir) + if mxDir is not None: + return mxDir + parent = dirname(curdir) + if curdir == parent: + return None + curdir = parent + return None def main(): - primarySuiteDir = _findPrimarySuite() - if primarySuiteDir: + SMArgParser()._parse_suitemodel_options() + + primarySuiteMxDir = _findPrimarySuiteMxDir() + if primarySuiteMxDir: + _src_suitemodel._set_primary_dir(dirname(primarySuiteMxDir)) global _mainSuite - _mainSuite = _loadSuite(primarySuiteDir, True) + _mainSuite = _loadSuite(primarySuiteMxDir, True) opts, commandAndArgs = _argParser._parse_cmd_line() + if primarySuiteMxDir is None: + msg = 'no primary suite found' + if len(commandAndArgs) > 0 and _needs_primary_suite(commandAndArgs[0]): + abort(msg) + else: + warn(msg) + global _opts, _java _opts = opts _java = JavaConfig(opts)