Mercurial > hg > truffle

comparison mxtool/mx.py @ 18602:7d8270532cd9

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
mx: changes towards supporting python 2.6 for call to mx from make/Makefile
author Doug Simon <doug.simon@oracle.com>
date Wed, 03 Dec 2014 16:07:22 +0100
parents 676f1800077c
children 23de014b38ab
comparison
equal deleted inserted replaced
18601:676f1800077c 18602:7d8270532cd9
46 from collections import Callable 46 from collections import Callable
47 from threading import Thread 47 from threading import Thread
48 from argparse import ArgumentParser, REMAINDER 48 from argparse import ArgumentParser, REMAINDER
49 from os.path import join, basename, dirname, exists, getmtime, isabs, expandvars, isdir, isfile 49 from os.path import join, basename, dirname, exists, getmtime, isabs, expandvars, isdir, isfile
50 50
51 # Support for Python 2.6
52 def check_output(*popenargs, **kwargs):
53 process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs)
54 output, _ = process.communicate()
55 retcode = process.poll()
56 if retcode:
57 cmd = kwargs.get("args")
58 if cmd is None:
59 cmd = popenargs[0]
60 error = subprocess.CalledProcessError(retcode, cmd)
61 error.output = output
62 raise error
63 return output
64
65 try: subprocess.check_output
66 except: subprocess.check_output = check_output
67
68 try: zipfile.ZipFile.__enter__
69 except:
70 zipfile.ZipFile.__enter__ = lambda self: self
71 zipfile.ZipFile.__exit__ = lambda self, t, value, traceback: self.close()
72
51 _projects = dict() 73 _projects = dict()
52 _libs = dict() 74 _libs = dict()
53 _jreLibs = dict() 75 _jreLibs = dict()
54 _dists = dict() 76 _dists = dict()
55 _suites = dict() 77 _suites = dict()
113 135
114 def make_archive(self): 136 def make_archive(self):
115 # are sources combined into main archive? 137 # are sources combined into main archive?
116 unified = self.path == self.sourcesPath 138 unified = self.path == self.sourcesPath
117 139
118 with Archiver(self.path) as arc, Archiver(None if unified else self.sourcesPath) as srcArcRaw: 140 with Archiver(self.path) as arc:
119 srcArc = arc if unified else srcArcRaw 141 with Archiver(None if unified else self.sourcesPath) as srcArcRaw:
120 services = {} 142 srcArc = arc if unified else srcArcRaw
121 def overwriteCheck(zf, arcname, source): 143 services = {}
122 if not hasattr(zf, '_provenance'): 144 def overwriteCheck(zf, arcname, source):
123 zf._provenance = {} 145 if not hasattr(zf, '_provenance'):
124 existingSource = zf._provenance.get(arcname, None) 146 zf._provenance = {}
125 isOverwrite = False 147 existingSource = zf._provenance.get(arcname, None)
126 if existingSource and existingSource != source: 148 isOverwrite = False
127 if arcname[-1] != os.path.sep: 149 if existingSource and existingSource != source:
128 logv('warning: ' + self.path + ': avoid overwrite of ' + arcname + '\n new: ' + source + '\n old: ' + existingSource) 150 if arcname[-1] != os.path.sep:
129 isOverwrite = True 151 logv('warning: ' + self.path + ': avoid overwrite of ' + arcname + '\n new: ' + source + '\n old: ' + existingSource)
130 zf._provenance[arcname] = source 152 isOverwrite = True
131 return isOverwrite 153 zf._provenance[arcname] = source
132 154 return isOverwrite
133 if self.mainClass: 155
134 manifest = "Manifest-Version: 1.0\nMain-Class: %s\n\n" % (self.mainClass) 156 if self.mainClass:
135 if not overwriteCheck(arc.zf, "META-INF/MANIFEST.MF", "project files"): 157 manifest = "Manifest-Version: 1.0\nMain-Class: %s\n\n" % (self.mainClass)
136 arc.zf.writestr("META-INF/MANIFEST.MF", manifest) 158 if not overwriteCheck(arc.zf, "META-INF/MANIFEST.MF", "project files"):
137 159 arc.zf.writestr("META-INF/MANIFEST.MF", manifest)
138 for dep in self.sorted_deps(includeLibs=True): 160
139 isCoveredByDependecy = False 161 for dep in self.sorted_deps(includeLibs=True):
140 for d in self.distDependencies: 162 isCoveredByDependecy = False
141 if dep in _dists[d].sorted_deps(includeLibs=True, transitive=True): 163 for d in self.distDependencies:
142 logv("Excluding {0} from {1} because it's provided by the dependency {2}".format(dep.name, self.path, d)) 164 if dep in _dists[d].sorted_deps(includeLibs=True, transitive=True):
143 isCoveredByDependecy = True 165 logv("Excluding {0} from {1} because it's provided by the dependency {2}".format(dep.name, self.path, d))
144 break 166 isCoveredByDependecy = True
145 167 break
146 if isCoveredByDependecy: 168
147 continue 169 if isCoveredByDependecy:
148 170 continue
149 if dep.isLibrary(): 171
150 l = dep 172 if dep.isLibrary():
151 # merge library jar into distribution jar 173 l = dep
152 logv('[' + self.path + ': adding library ' + l.name + ']') 174 # merge library jar into distribution jar
153 lpath = l.get_path(resolve=True) 175 logv('[' + self.path + ': adding library ' + l.name + ']')
154 libSourcePath = l.get_source_path(resolve=True) 176 lpath = l.get_path(resolve=True)
155 if lpath: 177 libSourcePath = l.get_source_path(resolve=True)
156 with zipfile.ZipFile(lpath, 'r') as lp: 178 if lpath:
157 for arcname in lp.namelist(): 179 with zipfile.ZipFile(lpath, 'r') as lp:
158 if arcname.startswith('META-INF/services/') and not arcname == 'META-INF/services/': 180 for arcname in lp.namelist():
159 service = arcname[len('META-INF/services/'):] 181 if arcname.startswith('META-INF/services/') and not arcname == 'META-INF/services/':
160 assert '/' not in service 182 service = arcname[len('META-INF/services/'):]
161 services.setdefault(service, []).extend(lp.read(arcname).splitlines()) 183 assert '/' not in service
162 else: 184 services.setdefault(service, []).extend(lp.read(arcname).splitlines())
163 if not overwriteCheck(arc.zf, arcname, lpath + '!' + arcname): 185 else:
164 arc.zf.writestr(arcname, lp.read(arcname)) 186 if not overwriteCheck(arc.zf, arcname, lpath + '!' + arcname):
165 if srcArc.zf and libSourcePath: 187 arc.zf.writestr(arcname, lp.read(arcname))
166 with zipfile.ZipFile(libSourcePath, 'r') as lp: 188 if srcArc.zf and libSourcePath:
167 for arcname in lp.namelist(): 189 with zipfile.ZipFile(libSourcePath, 'r') as lp:
168 if not overwriteCheck(srcArc.zf, arcname, lpath + '!' + arcname): 190 for arcname in lp.namelist():
169 srcArc.zf.writestr(arcname, lp.read(arcname)) 191 if not overwriteCheck(srcArc.zf, arcname, lpath + '!' + arcname):
170 elif dep.isProject(): 192 srcArc.zf.writestr(arcname, lp.read(arcname))
171 p = dep 193 elif dep.isProject():
172 194 p = dep
173 if self.javaCompliance: 195
174 if p.javaCompliance > self.javaCompliance: 196 if self.javaCompliance:
175 abort("Compliance level doesn't match: Distribution {0} requires {1}, but {2} is {3}.".format(self.name, self.javaCompliance, p.name, p.javaCompliance)) 197 if p.javaCompliance > self.javaCompliance:
176 198 abort("Compliance level doesn't match: Distribution {0} requires {1}, but {2} is {3}.".format(self.name, self.javaCompliance, p.name, p.javaCompliance))
177 # skip a Java project if its Java compliance level is "higher" than the configured JDK 199
178 jdk = java(p.javaCompliance) 200 # skip a Java project if its Java compliance level is "higher" than the configured JDK
179 assert jdk 201 jdk = java(p.javaCompliance)
180 202 assert jdk
181 logv('[' + self.path + ': adding project ' + p.name + ']') 203
182 outputDir = p.output_dir() 204 logv('[' + self.path + ': adding project ' + p.name + ']')
183 for root, _, files in os.walk(outputDir): 205 outputDir = p.output_dir()
184 relpath = root[len(outputDir) + 1:] 206 for root, _, files in os.walk(outputDir):
185 if relpath == join('META-INF', 'services'): 207 relpath = root[len(outputDir) + 1:]
186 for service in files: 208 if relpath == join('META-INF', 'services'):
187 with open(join(root, service), 'r') as fp: 209 for service in files:
188 services.setdefault(service, []).extend([provider.strip() for provider in fp.readlines()]) 210 with open(join(root, service), 'r') as fp:
189 elif relpath == join('META-INF', 'providers'): 211 services.setdefault(service, []).extend([provider.strip() for provider in fp.readlines()])
190 for provider in files: 212 elif relpath == join('META-INF', 'providers'):
191 with open(join(root, provider), 'r') as fp: 213 for provider in files:
192 for service in fp: 214 with open(join(root, provider), 'r') as fp:
193 services.setdefault(service.strip(), []).append(provider) 215 for service in fp:
194 else: 216 services.setdefault(service.strip(), []).append(provider)
195 for f in files: 217 else:
196 arcname = join(relpath, f).replace(os.sep, '/')
197 if not overwriteCheck(arc.zf, arcname, join(root, f)):
198 arc.zf.write(join(root, f), arcname)
199 if srcArc.zf:
200 sourceDirs = p.source_dirs()
201 if p.source_gen_dir():
202 sourceDirs.append(p.source_gen_dir())
203 for srcDir in sourceDirs:
204 for root, _, files in os.walk(srcDir):
205 relpath = root[len(srcDir) + 1:]
206 for f in files: 218 for f in files:
207 if f.endswith('.java'): 219 arcname = join(relpath, f).replace(os.sep, '/')
208 arcname = join(relpath, f).replace(os.sep, '/') 220 if not overwriteCheck(arc.zf, arcname, join(root, f)):
209 if not overwriteCheck(srcArc.zf, arcname, join(root, f)): 221 arc.zf.write(join(root, f), arcname)
210 srcArc.zf.write(join(root, f), arcname) 222 if srcArc.zf:
211 223 sourceDirs = p.source_dirs()
212 for service, providers in services.iteritems(): 224 if p.source_gen_dir():
213 arcname = 'META-INF/services/' + service 225 sourceDirs.append(p.source_gen_dir())
214 arc.zf.writestr(arcname, '\n'.join(providers)) 226 for srcDir in sourceDirs:
227 for root, _, files in os.walk(srcDir):
228 relpath = root[len(srcDir) + 1:]
229 for f in files:
230 if f.endswith('.java'):
231 arcname = join(relpath, f).replace(os.sep, '/')
232 if not overwriteCheck(srcArc.zf, arcname, join(root, f)):
233 srcArc.zf.write(join(root, f), arcname)
234
235 for service, providers in services.iteritems():
236 arcname = 'META-INF/services/' + service
237 arc.zf.writestr(arcname, '\n'.join(providers))
215 238
216 self.notify_updated() 239 self.notify_updated()
217 240
218 241
219 def notify_updated(self): 242 def notify_updated(self):
818 if savedModule: 841 if savedModule:
819 warn(modulePath + ' conflicts with ' + savedModule.__file__) 842 warn(modulePath + ' conflicts with ' + savedModule.__file__)
820 # temporarily extend the Python path 843 # temporarily extend the Python path
821 sys.path.insert(0, mxDir) 844 sys.path.insert(0, mxDir)
822 845
823 snapshot = frozenset(sys.modules.viewkeys()) 846 snapshot = frozenset(sys.modules.keys())
824 module = __import__(moduleName) 847 module = __import__(moduleName)
825 848
826 if savedModule: 849 if savedModule:
827 # restore the old module into the module name space 850 # restore the old module into the module name space
828 sys.modules[moduleName] = savedModule 851 sys.modules[moduleName] = savedModule
831 sys.modules.pop(moduleName) 854 sys.modules.pop(moduleName)
832 855
833 # For now fail fast if extra modules were loaded. 856 # For now fail fast if extra modules were loaded.
834 # This can later be relaxed to simply remove the extra modules 857 # This can later be relaxed to simply remove the extra modules
835 # from the sys.modules name space if necessary. 858 # from the sys.modules name space if necessary.
836 extraModules = sys.modules.viewkeys() - snapshot 859 extraModules = frozenset(sys.modules.keys()) - snapshot
837 assert len(extraModules) == 0, 'loading ' + modulePath + ' caused extra modules to be loaded: ' + ', '.join([m for m in extraModules]) 860 assert len(extraModules) == 0, 'loading ' + modulePath + ' caused extra modules to be loaded: ' + ', '.join([m for m in extraModules])
838 861
839 # revert the Python path 862 # revert the Python path
840 del sys.path[0] 863 del sys.path[0]
841 864
842 if not hasattr(module, dictName): 865 if not hasattr(module, dictName):
843 abort(modulePath + ' must define a variable named "' + dictName + '"') 866 abort(modulePath + ' must define a variable named "' + dictName + '"')
844 d = expand(getattr(module, dictName), [dictName]) 867 d = expand(getattr(module, dictName), [dictName])
845 sections = ['projects', 'libraries', 'jrelibraries', 'distributions'] + (['distribution_extensions'] if suite else ['name', 'mxversion']) 868 sections = ['projects', 'libraries', 'jrelibraries', 'distributions'] + (['distribution_extensions'] if suite else ['name', 'mxversion'])
846 unknown = d.viewkeys() - sections 869 unknown = frozenset(d.keys()) - frozenset(sections)
847 if unknown: 870 if unknown:
848 abort(modulePath + ' defines unsupported suite sections: ' + ', '.join(unknown)) 871 abort(modulePath + ' defines unsupported suite sections: ' + ', '.join(unknown))
849 872
850 if suite is None: 873 if suite is None:
851 suite = d 874 suite = d
855 additional = d.get(s) 878 additional = d.get(s)
856 if additional: 879 if additional:
857 if not existing: 880 if not existing:
858 suite[s] = additional 881 suite[s] = additional
859 else: 882 else:
860 conflicting = additional.viewkeys() & existing.viewkeys() 883 conflicting = additional.keys() & existing.keys()
861 if conflicting: 884 if conflicting:
862 abort(modulePath + ' redefines: ' + ', '.join(conflicting)) 885 abort(modulePath + ' redefines: ' + ', '.join(conflicting))
863 existing.update(additional) 886 existing.update(additional)
864 distExtensions = d.get('distribution_extensions') 887 distExtensions = d.get('distribution_extensions')
865 if distExtensions: 888 if distExtensions:
1037 config = 'META-INF/services/javax.annotation.processing.Processor' 1060 config = 'META-INF/services/javax.annotation.processing.Processor'
1038 with zipfile.ZipFile(apsJar, 'r') as zf: 1061 with zipfile.ZipFile(apsJar, 'r') as zf:
1039 currentAps = zf.read(config).split() 1062 currentAps = zf.read(config).split()
1040 if currentAps != aps: 1063 if currentAps != aps:
1041 logv('[updating ' + config + ' in ' + apsJar + ']') 1064 logv('[updating ' + config + ' in ' + apsJar + ']')
1042 with Archiver(apsJar) as arc, zipfile.ZipFile(apsJar, 'r') as lp: 1065 with Archiver(apsJar) as arc:
1043 for arcname in lp.namelist(): 1066 with zipfile.ZipFile(apsJar, 'r') as lp:
1044 if arcname == config: 1067 for arcname in lp.namelist():
1045 arc.zf.writestr(arcname, '\n'.join(aps)) 1068 if arcname == config:
1046 else: 1069 arc.zf.writestr(arcname, '\n'.join(aps))
1047 arc.zf.writestr(arcname, lp.read(arcname)) 1070 else:
1071 arc.zf.writestr(arcname, lp.read(arcname))
1048 d.add_update_listener(_refineAnnotationProcessorServiceConfig) 1072 d.add_update_listener(_refineAnnotationProcessorServiceConfig)
1049 self.dists.append(d) 1073 self.dists.append(d)
1050 1074
1051 if self.name is None: 1075 if self.name is None:
1052 abort('Missing "suite=<name>" in ' + suitePyFile) 1076 abort('Missing "suite=<name>" in ' + suitePyFile)