# HG changeset patch # User Doug Simon # Date 1411127333 -7200 # Node ID ef5212ce8091de814be925386b208e61e5da88a2 # Parent ae749a34de4994cfb6af18ffccd0f03383141c3c support for new projects file format including support for automatically updating from old format diff -r ae749a34de49 -r ef5212ce8091 mxtool/mx.py --- a/mxtool/mx.py Fri Sep 19 11:51:17 2014 +0200 +++ b/mxtool/mx.py Fri Sep 19 13:48:53 2014 +0200 @@ -43,7 +43,7 @@ import shutil, re, xml.dom.minidom import pipes import difflib -from collections import Callable +from collections import Callable, OrderedDict from threading import Thread from argparse import ArgumentParser, REMAINDER from os.path import join, basename, dirname, exists, getmtime, isabs, expandvars, isdir, isfile @@ -787,6 +787,227 @@ else: return None +# TODO: remove this function once all repos have transitioned +# to the new project format +def _read_projects_file(projectsFile): + suite = OrderedDict() + + suite['projects'] = OrderedDict() + suite['libraries'] = OrderedDict() + suite['jrelibraries'] = OrderedDict() + suite['distributions'] = OrderedDict() + + with open(projectsFile) as f: + prefix = '' + lineNum = 0 + + def error(message): + abort(projectsFile + ':' + str(lineNum) + ': ' + message) + + for line in f: + lineNum = lineNum + 1 + line = line.strip() + if line.endswith('\\'): + prefix = prefix + line[:-1] + continue + if len(prefix) != 0: + line = prefix + line + prefix = '' + if len(line) != 0 and line[0] != '#': + if '=' not in line: + error('non-comment line does not contain an "=" character') + key, value = line.split('=', 1) + + parts = key.split('@') + + if len(parts) == 1: + if parts[0] == 'suite': + suite['name'] = value + elif parts[0] == 'mxversion': + suite['mxversion'] = value + else: + error('Single part property must be "suite": ' + key) + + continue + if len(parts) != 3: + error('Property name does not have 3 parts separated by "@": ' + key) + kind, name, attr = parts + if kind == 'project': + m = suite['projects'] + elif kind == 'library': + m = suite['libraries'] + elif kind == 'jrelibrary': + m = suite['jrelibraries'] + elif kind == 'distribution': + m = suite['distributions'] + else: + error('Property name does not start with "project@", "library@" or "distribution@": ' + key) + + attrs = m.get(name) + if attrs is None: + attrs = OrderedDict() + m[name] = attrs + value = expandvars_in_property(value) + attrs[attr] = value + return suite + +# TODO: remove this command once all repos have transitioned +# to the new project format +def convertprojects(args, verbose=True): + """convert old style projects file to projects*.py file(s)""" + + class Printer: + def __init__(self, fp, indent): + self.fp = fp + self.indent = indent + self.prefix = '' + def println(self, s): + if len(s) == 0: + print >> self.fp, s + else: + print >> self.fp, self.prefix + s + def inc(self): + self.prefix = ''.rjust(len(self.prefix) + self.indent) + def dec(self): + self.prefix = ''.rjust(len(self.prefix) - self.indent) + + for projectsFile in args: + suite = _read_projects_file(projectsFile) + def print_attrs(p, name, attrs, listKeys, is_last=False): + p.println('"' + name + '" : {') + p.inc() + for n, v in attrs.iteritems(): + if n in listKeys: + if len(v) == 0: + p.println('"{}" : [],'.format(n)) + else: + p.println('"{}" : ['.format(n)) + p.inc() + for e in v.split(','): + p.println('"' + e.strip() + '",') + p.dec() + p.println('],') + else: + p.println('"{}" : "{}",'.format(n, v)) + p.dec() + if is_last: + p.println('}') + else: + p.println('},') + p.println('') + + def print_section(p, sname, suite, is_last=False): + section = suite.get(sname) + if section: + p.println('"' + sname + '" : {') + p.inc() + i = 0 + for name, attrs in section.iteritems(): + i = i + 1 + print_attrs(p, name, attrs, ['urls', 'dependencies', 'sourceUrls'], i == len(section)) + + p.dec() + if is_last: + p.println('}') + else: + p.println('},') + p.println('') + + existing, projectsPyFile = _load_suite_dict(dirname(projectsFile)) + if existing: + assert existing['name'] == suite.pop('name') + assert existing['mxversion'] == suite.pop('mxversion') + for s in ['projects', 'libraries', 'jrelibraries', 'distributions']: + for k in existing[s].iterkeys(): + suite[s].pop(k) + if len(suite[s]) == 0: + suite.pop(s) + + if len(suite): + out = StringIO.StringIO() + p = Printer(out, 2) + p.println(('extra' if existing else 'suite') + ' = {') + p.inc() + if not existing: + p.println('"mxversion" : "' + suite['mxversion'] + '",') + p.println('"name" : "' + suite['name'] + '",') + print_section(p, 'libraries', suite) + print_section(p, 'jrelibraries', suite) + print_section(p, 'projects', suite) + print_section(p, 'distributions', suite, is_last=True) + p.dec() + p.println('}') + + with open(projectsPyFile, 'w') as fp: + fp.write(out.getvalue()) + if verbose: + print 'created: ' + projectsPyFile + +def _load_suite_dict(mxDir): + + suffix = 1 + suite = None + dictName = 'suite' + + moduleName = 'projects' + modulePath = join(mxDir, moduleName + '.py') + while exists(modulePath): + + savedModule = sys.modules.get(moduleName) + if savedModule: + warn(modulePath + ' conflicts with ' + savedModule.__file__) + # temporarily extend the Python path + sys.path.insert(0, mxDir) + + snapshot = frozenset(sys.modules.viewkeys()) + module = __import__(moduleName) + + if savedModule: + # restore the old module into the module name space + sys.modules[moduleName] = savedModule + else: + # remove moduleName from the module name space + sys.modules.pop(moduleName) + + # For now fail fast if extra modules were loaded. + # This can later be relaxed to simply remove the extra modules + # from the sys.modules name space if necessary. + extraModules = snapshot - sys.modules.viewkeys() + assert len(extraModules) == 0, 'loading ' + modulePath + ' caused extra modules to be loaded: ' + ', '.join([m.__file__ for m in extraModules]) + + # revert the Python path + del sys.path[0] + + if not hasattr(module, dictName): + abort(modulePath + ' must define a variable named "' + dictName + '"') + d = getattr(module, dictName) + sections = ['projects', 'libraries', 'jrelibraries', 'distributions'] + ([] if suite else ['name', 'mxversion']) + unknown = d.viewkeys() - sections + if unknown: + abort(modulePath + ' defines unsupported suite sections: ' + ', '.join(unknown)) + + if suite is None: + suite = d + else: + for s in sections: + existing = suite.get(s) + additional = d.get(s) + if additional: + if not existing: + suite[s] = additional + else: + conflicting = additional.viewkeys() & existing.viewkeys() + if conflicting: + abort(modulePath + ' redefines: ' + ', '.join(conflicting)) + existing.update(additional) + + dictName = 'extra' + moduleName = 'projects' + str(suffix) + modulePath = join(mxDir, moduleName + '.py') + suffix = suffix + 1 + + return suite, modulePath + class Suite: def __init__(self, mxDir, primary, load=True): self.dir = dirname(mxDir) @@ -810,73 +1031,35 @@ return self.name def _load_projects(self): - libsMap = dict() - jreLibsMap = dict() - projsMap = dict() - distsMap = dict() + # TODO: remove once mx/projects has been deprecated projectsFile = join(self.mxDir, 'projects') - if not exists(projectsFile): + if exists(projectsFile): + convertprojects([projectsFile], verbose=False) + + projectsPyFile = join(self.mxDir, 'projects.py') + if not exists(projectsPyFile): return - with open(projectsFile) as f: - prefix = '' - lineNum = 0 - - def error(message): - abort(projectsFile + ':' + str(lineNum) + ': ' + message) - - for line in f: - lineNum = lineNum + 1 - line = line.strip() - if line.endswith('\\'): - prefix = prefix + line[:-1] - continue - if len(prefix) != 0: - line = prefix + line - prefix = '' - if len(line) != 0 and line[0] != '#': - if '=' not in line: - error('non-comment line does not contain an "=" character') - key, value = line.split('=', 1) - - parts = key.split('@') - - if len(parts) == 1: - if parts[0] == 'suite': - if self.name != value: - error('suite name in project file does not match ' + _suitename(self.mxDir)) - elif parts[0] == 'mxversion': - try: - self.requiredMxVersion = VersionSpec(value) - except AssertionError as ae: - error('Exception while parsing "mxversion" in project file: ' + str(ae)) - else: - error('Single part property must be "suite": ' + key) - - continue - if len(parts) != 3: - error('Property name does not have 3 parts separated by "@": ' + key) - kind, name, attr = parts - if kind == 'project': - m = projsMap - elif kind == 'library': - m = libsMap - elif kind == 'jrelibrary': - m = jreLibsMap - elif kind == 'distribution': - m = distsMap - else: - error('Property name does not start with "project@", "library@" or "distribution@": ' + key) - - attrs = m.get(name) - if attrs is None: - attrs = dict() - m[name] = attrs - value = expandvars_in_property(value) - attrs[attr] = value + suiteDict, _ = _load_suite_dict(self.mxDir) + + if suiteDict.get('name') is not None and suiteDict.get('name') != self.name: + abort('suite name in project file does not match ' + _suitename(self.mxDir)) + + if suiteDict.has_key('mxversion'): + try: + self.requiredMxVersion = VersionSpec(suiteDict['mxversion']) + except AssertionError as ae: + abort('Exception while parsing "mxversion" in project file: ' + str(ae)) + + libsMap = suiteDict['libraries'] + jreLibsMap = suiteDict['jrelibraries'] + projsMap = suiteDict['projects'] + distsMap = suiteDict['distributions'] def pop_list(attrs, name): v = attrs.pop(name, None) + if isinstance(v, list): + return v if v is None or len(v.strip()) == 0: return [] return [n.strip() for n in v.split(',')] @@ -896,7 +1079,7 @@ p.checkstyleProj = attrs.pop('checkstyle', name) p.native = attrs.pop('native', '') == 'true' if not p.native and p.javaCompliance is None: - error('javaCompliance property required for non-native project ' + name) + abort('javaCompliance property required for non-native project ' + name) if len(ap) > 0: p._declaredAnnotationProcessors = ap p.__dict__.update(attrs) @@ -911,7 +1094,8 @@ for name, attrs in libsMap.iteritems(): if "|" in name: - assert name.count("|") == 2, "syntax: libname|os-platform|architecture" + if name.count('|') != 2: + abort("Format error in library name: " + name + "\nsyntax: libname|os-platform|architecture") name, platform, architecture = name.split("|") if platform != get_os() or architecture != get_arch(): continue @@ -2866,61 +3050,60 @@ return archives def canonicalizeprojects(args): - """process all project files to canonicalize the dependencies - - The exit code of this command reflects how many files were updated.""" - - changedFiles = 0 + """check all project specifications for canonical dependencies + + The exit code of this command reflects how many projects have non-canonical dependencies.""" + + nonCanonical = [] for s in suites(True): - projectsFile = join(s.mxDir, 'projects') - if not exists(projectsFile): + projectsPyFile = join(s.mxDir, 'projects') + if not exists(projectsPyFile): continue - with open(projectsFile) as f: - out = StringIO.StringIO() - pattern = re.compile('project@([^@]+)@dependencies=.*') - lineNo = 1 - for line in f: - line = line.strip() - m = pattern.match(line) - p = project(m.group(1), fatalIfMissing=False) if m else None - if m is None or p is None: - out.write(line + '\n') - else: - for pkg in p.defined_java_packages(): - if not pkg.startswith(p.name): - abort('package in {0} does not have prefix matching project name: {1}'.format(p, pkg)) - - ignoredDeps = set([name for name in p.deps if project(name, False) is not None]) - for pkg in p.imported_java_packages(): - for name in p.deps: - dep = project(name, False) - if dep is None: - ignoredDeps.discard(name) - else: - if pkg in dep.defined_java_packages(): - ignoredDeps.discard(name) - if pkg in dep.extended_java_packages(): - ignoredDeps.discard(name) - if len(ignoredDeps) != 0: - candidates = set() - # Compute dependencies based on projects required by p - for d in sorted_deps(): - if not d.defined_java_packages().isdisjoint(p.imported_java_packages()): - candidates.add(d) - # Remove non-canonical candidates - for c in list(candidates): - candidates.difference_update(c.all_deps([], False, False)) - candidates = [d.name for d in candidates] - - abort('{0}:{1}: {2} does not use any packages defined in these projects: {3}\nComputed project dependencies: {4}'.format( - projectsFile, lineNo, p, ', '.join(ignoredDeps), ','.join(candidates))) - - out.write('project@' + m.group(1) + '@dependencies=' + ','.join(p.canonical_deps()) + '\n') - lineNo = lineNo + 1 - content = out.getvalue() - if update_file(projectsFile, content): - changedFiles += 1 - return changedFiles + + for p in s.projects: + for pkg in p.defined_java_packages(): + if not pkg.startswith(p.name): + abort('package in {0} does not have prefix matching project name: {1}'.format(p, pkg)) + + ignoredDeps = set([name for name in p.deps if project(name, False) is not None]) + for pkg in p.imported_java_packages(): + for name in p.deps: + dep = project(name, False) + if dep is None: + ignoredDeps.discard(name) + else: + if pkg in dep.defined_java_packages(): + ignoredDeps.discard(name) + if pkg in dep.extended_java_packages(): + ignoredDeps.discard(name) + if len(ignoredDeps) != 0: + candidates = set() + # Compute dependencies based on projects required by p + for d in sorted_deps(): + if not d.defined_java_packages().isdisjoint(p.imported_java_packages()): + candidates.add(d) + # Remove non-canonical candidates + for c in list(candidates): + candidates.difference_update(c.all_deps([], False, False)) + candidates = [d.name for d in candidates] + + abort('{} does not use any packages defined in these projects: {}\nComputed project dependencies: {}'.format( + p, ', '.join(ignoredDeps), ','.join(candidates))) + + excess = frozenset(p.deps) - set(p.canonical_deps()) + if len(excess) != 0: + nonCanonical.append(p) + if len(nonCanonical) != 0: + for p in nonCanonical: + canonicalDeps = p.canonical_deps() + if len(canonicalDeps) != 0: + log('Canonical dependencies for project ' + p.name + ' are: [') + for d in canonicalDeps: + log(' "' + d + '",') + log(' ],') + else: + log('Canonical dependencies for project ' + p.name + ' are: []') + return len(nonCanonical) class TimeStampFile: def __init__(self, path): @@ -4952,6 +5135,7 @@ 'about': [about, ''], 'build': [build, '[options]'], 'checkstyle': [checkstyle, ''], + 'convertprojects' : [convertprojects, ''], 'canonicalizeprojects': [canonicalizeprojects, ''], 'clean': [clean, ''], 'eclipseinit': [eclipseinit, ''],