# HG changeset patch # User Doug Simon # Date 1400160682 -7200 # Node ID 50fbda571d99cf275d2b1c0a7c55e3e82ac7f79e # Parent 304e1c30adaf830a3c093b44eb05adf8c378642b mx: added jrelibrary dependency type mx: once mx/projects is loaded, projects and libraries that depend on non-existent JRE library or have a javaCompliance higher than any available JDK are ignored diff -r 304e1c30adaf -r 50fbda571d99 CHANGELOG.md --- a/CHANGELOG.md Thu May 15 11:13:44 2014 +0200 +++ b/CHANGELOG.md Thu May 15 15:31:22 2014 +0200 @@ -3,6 +3,8 @@ ## `tip` ### Graal * Made initialization of Graal runtime lazy in hosted mode. +* Added supported for new 'jrelibrary' dependency type in mx/projects. +* Java projects with compliance level higher than the JDKs specified by JAVA_HOME and EXTRA_JAVA_HOMES are ignored once mx/projects has been processed. ### Truffle * `truffle.jar`: strip out build-time only dependency into a seperated JAR file (`truffle-dsl-processor.jar`) diff -r 304e1c30adaf -r 50fbda571d99 mxtool/mx.py --- a/mxtool/mx.py Thu May 15 11:13:44 2014 +0200 +++ b/mxtool/mx.py Thu May 15 15:31:22 2014 +0200 @@ -49,6 +49,7 @@ _projects = dict() _libs = dict() +_jreLibs = dict() _dists = dict() _suites = dict() _annotationProcessors = None @@ -123,7 +124,7 @@ for arcname in lp.namelist(): overwriteCheck(srcArc.zf, arcname, lpath + '!' + arcname) srcArc.zf.writestr(arcname, lp.read(arcname)) - else: + elif dep.isProject(): p = dep isCoveredByDependecy = False @@ -138,9 +139,7 @@ # skip a Java project if its Java compliance level is "higher" than the configured JDK jdk = java(p.javaCompliance) - if not jdk: - log('Excluding {0} from {2} (Java compliance level {1} required)'.format(p.name, p.javaCompliance, self.path)) - continue + assert jdk logv('[' + self.path + ': adding project ' + p.name + ']') outputDir = p.output_dir() @@ -207,6 +206,9 @@ def isLibrary(self): return isinstance(self, Library) + def isJreLibrary(self): + return isinstance(self, JreLibrary) + def isProject(self): return isinstance(self, Project) @@ -235,7 +237,7 @@ if not exists(s): os.mkdir(s) - def all_deps(self, deps, includeLibs, includeSelf=True, includeAnnotationProcessors=False): + def all_deps(self, deps, includeLibs, includeSelf=True, includeJreLibs=False, includeAnnotationProcessors=False): """ Add the transitive set of dependencies for this project, including libraries if 'includeLibs' is true, to the 'deps' list. @@ -248,8 +250,8 @@ for name in childDeps: assert name != self.name dep = dependency(name) - if not dep in deps and (includeLibs or not dep.isLibrary()): - dep.all_deps(deps, includeLibs=includeLibs, includeAnnotationProcessors=includeAnnotationProcessors) + if not dep in deps and (dep.isProject or (dep.isLibrary() and includeLibs) or (dep.isJreLibrary() and includeJreLibs)): + dep.all_deps(deps, includeLibs=includeLibs, includeJreLibs=includeJreLibs, includeAnnotationProcessors=includeAnnotationProcessors) if not self in deps and includeSelf: deps.append(self) return deps @@ -512,17 +514,74 @@ return path -class Library(Dependency): - def __init__(self, suite, name, path, mustExist, urls, sha1, sourcePath, sourceUrls, sourceSha1, deps): +class BaseLibrary(Dependency): + def __init__(self, suite, name, optional): Dependency.__init__(self, suite, name) + self.optional = optional + + def __ne__(self, other): + result = self.__eq__(other) + if result is NotImplemented: + return result + return not result + +""" +A library that will be provided by the JDK but may be absent. +Any project or normal library that depends on a missing library +will be removed from the global project and library dictionaries +(i.e., _projects and _libs). + +This mechanism exists primarily to be able to support code +that may use functionality in one JDK (e.g., Oracle JDK) +that is not present in another JDK (e.g., OpenJDK). A +motivating example is the Java Flight Recorder library +found in the Oracle JDK. +""" +class JreLibrary(BaseLibrary): + def __init__(self, suite, name, jar, optional): + BaseLibrary.__init__(self, suite, name, optional) + self.jar = jar + + def __eq__(self, other): + if isinstance(other, JreLibrary): + return self.jar == other.jar + else: + return NotImplemented + + def is_present_in_jdk(self, jdk): + for e in jdk.bootclasspath().split(os.pathsep): + if basename(e) == self.jar: + return True + for d in jdk.extdirs().split(os.pathsep): + if len(d) and self.jar in os.listdir(d): + return True + for d in jdk.endorseddirs().split(os.pathsep): + if len(d) and self.jar in os.listdir(d): + return True + return False + + def all_deps(self, deps, includeLibs, includeSelf=True, includeJreLibs=False, includeAnnotationProcessors=False): + """ + Add the transitive set of dependencies for this JRE library to the 'deps' list. + """ + if includeJreLibs and includeSelf and not self in deps: + deps.append(self) + return deps + +class Library(BaseLibrary): + def __init__(self, suite, name, path, optional, urls, sha1, sourcePath, sourceUrls, sourceSha1, deps): + BaseLibrary.__init__(self, suite, name, optional) self.path = path.replace('/', os.sep) self.urls = urls self.sha1 = sha1 - self.mustExist = mustExist self.sourcePath = sourcePath self.sourceUrls = sourceUrls self.sourceSha1 = sourceSha1 self.deps = deps + abspath = _make_absolute(self.path, self.suite.dir) + if not optional and not exists(abspath): + if not len(urls): + abort('Non-optional library {} must either exist at {} or specify one or more URLs from which it can be retrieved'.format(name, abspath)) for url in urls: if url.endswith('/') != self.path.endswith(os.sep): abort('Path for dependency directory must have a URL ending with "/": path=' + self.path + ' url=' + url) @@ -536,14 +595,6 @@ else: return NotImplemented - - def __ne__(self, other): - result = self.__eq__(other) - if result is NotImplemented: - return result - return not result - - def get_path(self, resolve): path = _make_absolute(self.path, self.suite.dir) sha1path = path + '.sha1' @@ -552,8 +603,7 @@ if includedInJDK and java().javaCompliance >= JavaCompliance(includedInJDK): return None - return _download_file_with_sha1(self.name, path, self.urls, self.sha1, sha1path, resolve, self.mustExist) - + return _download_file_with_sha1(self.name, path, self.urls, self.sha1, sha1path, resolve, self.optional) def get_source_path(self, resolve): if self.sourcePath is None: @@ -568,7 +618,7 @@ if path and (exists(path) or not resolve): cp.append(path) - def all_deps(self, deps, includeLibs, includeSelf=True, includeAnnotationProcessors=False): + def all_deps(self, deps, includeLibs, includeSelf=True, includeJreLibs=False, includeAnnotationProcessors=False): """ Add the transitive set of dependencies for this library to the 'deps' list. """ @@ -581,7 +631,7 @@ assert name != self.name dep = library(name) if not dep in deps: - dep.all_deps(deps, includeLibs=includeLibs, includeAnnotationProcessors=includeAnnotationProcessors) + dep.all_deps(deps, includeLibs=includeLibs, includeJreLibs=includeJreLibs, includeAnnotationProcessors=includeAnnotationProcessors) if not self in deps and includeSelf: deps.append(self) return deps @@ -637,6 +687,7 @@ self.mxDir = mxDir self.projects = [] self.libs = [] + self.jreLibs = [] self.dists = [] self.commands = None self.primary = primary @@ -654,6 +705,7 @@ def _load_projects(self): libsMap = dict() + jreLibsMap = dict() projsMap = dict() distsMap = dict() projectsFile = join(self.mxDir, 'projects') @@ -703,6 +755,8 @@ m = projsMap elif kind == 'library': m = libsMap + elif kind == 'jrelibrary': + m = jreLibsMap elif kind == 'distribution': m = distsMap else: @@ -743,16 +797,24 @@ p.__dict__.update(attrs) self.projects.append(p) + for name, attrs in jreLibsMap.iteritems(): + jar = attrs.pop('jar') + # JRE libraries are optional by default + optional = attrs.pop('optional', 'true') != 'false' + l = JreLibrary(self, name, jar, optional) + self.jreLibs.append(l) + for name, attrs in libsMap.iteritems(): path = attrs.pop('path') - mustExist = attrs.pop('optional', 'false') != 'true' urls = pop_list(attrs, 'urls') sha1 = attrs.pop('sha1', None) sourcePath = attrs.pop('sourcePath', None) sourceUrls = pop_list(attrs, 'sourceUrls') sourceSha1 = attrs.pop('sourceSha1', None) deps = pop_list(attrs, 'dependencies') - l = Library(self, name, path, mustExist, urls, sha1, sourcePath, sourceUrls, sourceSha1, deps) + # Add support optional libraries once we have a good use case + optional = False + l = Library(self, name, path, optional, urls, sha1, sourcePath, sourceUrls, sourceSha1, deps) l.__dict__.update(attrs) self.libs.append(l) @@ -842,6 +904,12 @@ if existing is not None and existing != l: abort('inconsistent library redefinition of ' + l.name + ' in ' + existing.suite.dir + ' and ' + l.suite.dir) _libs[l.name] = l + for l in self.jreLibs: + existing = _jreLibs.get(l.name) + # Check that suites that define same library are consistent + if existing is not None and existing != l: + abort('inconsistent JRE library redefinition of ' + l.name + ' in ' + existing.suite.dir + ' and ' + l.suite.dir) + _jreLibs[l.name] = l for d in self.dists: existing = _dists.get(d.name) if existing is not None: @@ -850,6 +918,54 @@ warn('distribution ' + d.name + ' redefined') d.path = existing.path _dists[d.name] = d + + # Remove projects and libraries that (recursively) depend on an optional library + # whose artifact does not exist or on a JRE library that is not present in the + # JDK for a project. Also remove projects whose Java compliance requirement + # cannot be satisfied by the configured JDKs. + # + # Removed projects and libraries are also removed from + # distributions in they are listed as dependencies. + for d in sorted_deps(includeLibs=True): + if d.isLibrary(): + if d.optional: + try: + d.optional = False + path = d.get_path(resolve=True) + except SystemExit: + path = None + finally: + d.optional = True + if not path: + logv('[omitting optional library {} as {} does not exist]'.format(d, d.path)) + del _libs[d.name] + self.libs.remove(d) + elif d.isProject(): + if java(d.javaCompliance) is None: + logv('[omitting project {} as Java compliance {} cannot be satisfied by configured JDKs]'.format(d, d.javaCompliance)) + del _projects[d.name] + self.projects.remove(d) + else: + for name in list(d.deps): + jreLib = _jreLibs.get(name) + if jreLib: + if not jreLib.is_present_in_jdk(java(d.javaCompliance)): + if jreLib.optional: + logv('[omitting project {} as dependency {} is missing]'.format(d, name)) + del _projects[d.name] + self.projects.remove(d) + else: + abort('JRE library {} required by {} not found'.format(jreLib, d)) + elif not dependency(name, fatalIfMissing=False): + logv('[omitting project {} as dependency {} is missing]'.format(d, name)) + del _projects[d.name] + self.projects.remove(d) + for dist in _dists.values(): + for name in list(dist.deps): + if not dependency(name, fatalIfMissing=False): + logv('[omitting {} from distribution {}]'.format(name, dist)) + dist.deps.remove(name) + if hasattr(self, 'mx_post_parse_cmd_line'): self.mx_post_parse_cmd_line(opts) @@ -1031,6 +1147,8 @@ d = _projects.get(name) if d is None: d = _libs.get(name) + if d is None: + d = _jreLibs.get(name) if d is None and fatalIfMissing: if name in _opts.ignored_projects: abort('project named ' + name + ' is ignored') @@ -1546,6 +1664,8 @@ self.javadoc = exe_suffix(join(self.jdk, 'bin', 'javadoc')) self.toolsjar = join(self.jdk, 'lib', 'tools.jar') self._bootclasspath = None + self._extdirs = None + self._endorseddirs = None if not exists(self.java): abort('Java launcher does not exist: ' + self.java) @@ -1714,8 +1834,6 @@ if _opts.killwithsigquit: _send_sigquit() - # import traceback - # traceback.print_stack() for p, args in _currentSubprocesses: try: if get_os() == 'windows': @@ -1725,6 +1843,9 @@ except BaseException as e: log('error while killing subprocess {} "{}": {}'.format(p.pid, ' '.join(args), e)) + if _opts and _opts.verbose: + import traceback + traceback.print_stack() raise SystemExit(codeOrMessage) def download(path, urls, verbose=False): @@ -2030,9 +2151,7 @@ # skip building this Java project if its Java compliance level is "higher" than the configured JDK requiredCompliance = p.javaCompliance if p.javaCompliance else JavaCompliance(args.compliance) if args.compliance else None jdk = java(requiredCompliance) - if not jdk: - log('Excluding {0} from build (Java compliance level {1} required)'.format(p.name, requiredCompliance)) - continue + assert jdk outputDir = prepareOutputDirs(p, args.clean) @@ -2628,9 +2747,7 @@ # skip checking this Java project if its Java compliance level is "higher" than the configured JDK jdk = java(p.javaCompliance) - if not jdk: - log('Excluding {0} from checking (Java compliance level {1} required)'.format(p.name, p.javaCompliance)) - continue + assert jdk for sourceDir in sourceDirs: javafilelist = [] @@ -2874,7 +2991,7 @@ elif dep.get_source_path(resolve=True): memento = XMLDoc().element('archive', {'detectRoot' : 'true', 'path' : dep.get_source_path(resolve=True)}).xml(standalone='no') slm.element('container', {'memento' : memento, 'typeId':'org.eclipse.debug.core.containerType.externalArchive'}) - else: + elif dep.isProject(): memento = XMLDoc().element('javaProject', {'name' : dep.name}).xml(standalone='no') slm.element('container', {'memento' : memento, 'typeId':'org.eclipse.jdt.launching.sourceContainer.javaProject'}) if javaCompliance is None or dep.javaCompliance < javaCompliance: @@ -3041,9 +3158,7 @@ if p.native: continue - if not java(p.javaCompliance): - log('Excluding {0} (JDK with compliance level {1} not available)'.format(p.name, p.javaCompliance)) - continue + assert java(p.javaCompliance) if not exists(p.dir): os.makedirs(p.dir) @@ -3084,7 +3199,7 @@ libraryDeps -= set(dep.all_deps([], True)) else: libraryDeps.add(dep) - else: + elif dep.isProject(): projectDeps.add(dep) for dep in containerDeps: @@ -3093,8 +3208,6 @@ for dep in libraryDeps: path = dep.path dep.get_path(resolve=True) - if not path or (not exists(path) and not dep.mustExist): - continue # Relative paths for "lib" class path entries have various semantics depending on the Eclipse # version being used (e.g. see https://bugs.eclipse.org/bugs/show_bug.cgi?id=274737) so it's @@ -3252,16 +3365,13 @@ for dep in dependency(ap).all_deps([], True): if dep.isLibrary(): if not hasattr(dep, 'eclipse.container') and not hasattr(dep, 'eclipse.project'): - if dep.mustExist: - path = dep.get_path(resolve=True) - if path: - # Relative paths for "lib" class path entries have various semantics depending on the Eclipse - # version being used (e.g. see https://bugs.eclipse.org/bugs/show_bug.cgi?id=274737) so it's - # safest to simply use absolute paths. - path = _make_absolute(path, p.suite.dir) - out.element('factorypathentry', {'kind' : 'EXTJAR', 'id' : path, 'enabled' : 'true', 'runInBatchMode' : 'false'}) - files.append(path) - else: + # Relative paths for "lib" class path entries have various semantics depending on the Eclipse + # version being used (e.g. see https://bugs.eclipse.org/bugs/show_bug.cgi?id=274737) so it's + # safest to simply use absolute paths. + path = _make_absolute(dep.get_path(resolve=True), p.suite.dir) + out.element('factorypathentry', {'kind' : 'EXTJAR', 'id' : path, 'enabled' : 'true', 'runInBatchMode' : 'false'}) + files.append(path) + elif dep.isProject(): out.element('factorypathentry', {'kind' : 'WKSPJAR', 'id' : '/' + dep.name + '/' + dep.name + '.jar', 'enabled' : 'true', 'runInBatchMode' : 'false'}) out.close('factorypath') update_file(join(p.dir, '.factorypath'), out.xml(indent='\t', newl='\n')) @@ -3564,10 +3674,7 @@ os.makedirs(join(p.dir, 'nbproject')) jdk = java(p.javaCompliance) - - if not jdk: - log('Excluding {0} (JDK with compliance level {1} not available)'.format(p.name, p.javaCompliance)) - continue + assert jdk jdks.add(jdk) @@ -3608,7 +3715,7 @@ if dep == p: continue - if not dep.isLibrary(): + if dep.isProject(): n = dep.name.replace('.', '_') if firstDep: out.open('references', {'xmlns' : 'http://www.netbeans.org/ns/ant-project-references/1'}) @@ -3743,8 +3850,6 @@ continue if dep.isLibrary(): - if not dep.mustExist: - continue path = dep.get_path(resolve=True) if path: if os.sep == '\\': @@ -3753,7 +3858,7 @@ print >> out, ref + '=' + path libFiles.append(path) - else: + elif dep.isProject(): n = dep.name.replace('.', '_') relDepPath = os.path.relpath(dep.dir, p.dir).replace(os.sep, '/') ref = 'reference.' + n + '.jar' @@ -3820,9 +3925,7 @@ if p.native: continue - if not java(p.javaCompliance): - log('Excluding {0} (JDK with compliance level {1} not available)'.format(p.name, p.javaCompliance)) - continue + assert java(p.javaCompliance) if not exists(p.dir): os.makedirs(p.dir) @@ -3869,10 +3972,9 @@ continue if dep.isLibrary(): - if dep.mustExist: - libraries.add(dep) - moduleXml.element('orderEntry', attributes={'type': 'library', 'name': dep.name, 'level': 'project'}) - else: + libraries.add(dep) + moduleXml.element('orderEntry', attributes={'type': 'library', 'name': dep.name, 'level': 'project'}) + elif dep.isProject(): moduleXml.element('orderEntry', attributes={'type': 'module', 'module-name': dep.name}) moduleXml.close('component') @@ -3944,7 +4046,7 @@ for entry in pDep.all_deps([], True): if entry.isLibrary(): compilerXml.element('entry', attributes={'name': '$PROJECT_DIR$/' + os.path.relpath(entry.path, suite.dir)}) - else: + elif entry.isProject(): assert entry.isProject() compilerXml.element('entry', attributes={'name': '$PROJECT_DIR$/' + os.path.relpath(entry.output_dir(), suite.dir)}) compilerXml.close('processorPath')