Mercurial > hg > truffle

comparison mxtool/mx.py @ 3723:6c5f528c7aac

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Added a copy of the mxtool to repo.
author Doug Simon <doug.simon@oracle.com>
date Fri, 16 Dec 2011 16:46:33 +0100
parents
children 3e2e8b8abdaf
comparison
equal deleted inserted replaced
3722:7c5524a4e86e 3723:6c5f528c7aac
1 #!/usr/bin/python
2 #
3 # ----------------------------------------------------------------------------------------------------
4 #
5 # Copyright (c) 2007, 2011, Oracle and/or its affiliates. All rights reserved.
6 # DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
7 #
8 # This code is free software; you can redistribute it and/or modify it
9 # under the terms of the GNU General Public License version 2 only, as
10 # published by the Free Software Foundation.
11 #
12 # This code is distributed in the hope that it will be useful, but WITHOUT
13 # ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
14 # FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License
15 # version 2 for more details (a copy is included in the LICENSE file that
16 # accompanied this code).
17 #
18 # You should have received a copy of the GNU General Public License version
19 # 2 along with this work; if not, write to the Free Software Foundation,
20 # Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
21 #
22 # Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
23 # or visit www.oracle.com if you need additional information or have any
24 # questions.
25 #
26 # ----------------------------------------------------------------------------------------------------
27 #
28 # mx is a command line tool inspired by mvn (http://maven.apache.org/)
29 # and hg (http://mercurial.selenic.com/). It includes a mechanism for
30 # managing the dependencies between a set of projects (like Maven)
31 # as well as making it simple to run commands
32 # (like hg is the interface to the Mercurial commands).
33 #
34 # When launched, mx looks for a mx configuration (i.e. a directory named 'mx')
35 # in the current working directory. This is the primary mx configuration. Any
36 # other mx configurations are included mx configurations.
37 #
38 # If an mx configuration exists, then the following files in the configuration
39 # are processed (if they exist):
40 #
41 # projects - Lists projects, libraries and dependencies between them
42 # commands.py - Extensions to the commands launchable by mx. This is only processed
43 # for the primary mx configuration.
44 # includes - Other directories containing mx configurations to be loaded.
45 # This is a recursive action.
46 # env - A set of environment variable definitions.
47 #
48 # The MX_INCLUDES environment variable can also be used to specify
49 # other directories containing mx configurations.
50 # This value of this variable has the same format as a Java class path.
51 #
52 # The includes and env files are typically not put under version control
53 # as they usually contain local filesystem paths.
54 #
55 # The projects file is like the pom.xml file from Maven except that
56 # it is in a properties file format instead of XML. Each non-comment line
57 # in the file specifies an attribute of a project or library. The main
58 # difference between a project and a library is that the former contains
59 # source code built by the mx tool where as the latter is an external
60 # dependency. The format of the projects file is
61 #
62 # Library specification format:
63 #
64 # library@<name>@<prop>=<value>
65 #
66 # Built-in library properties (* = required):
67 #
68 # *path: the file system path for the library to appear on a class path
69 # urls: a comma seperated list of URLs from which the library can be downloaded
70 # optional: if "true" then this library will be omitted from a class path if it doesn't exist on the file system and no URLs are specified
71 #
72 # Project specification format:
73 #
74 # project@<name>@<prop>=<value>
75 #
76 # The name of a project also denotes the directory it is in.
77 #
78 # Built-in project properties:
79 #
80 # *sourceDirs: a comma separated list of source directoriy names (relative to the project directory)
81 # dependencies: a comma separated list of the libraries and project the project depends upon (transitive dependencies may be omitted)
82 # checkstyle: the project whose Checkstyle configuration (i.e. <project>/.checkstyle_checks.xml) is used
83 #
84 # Other properties can be specified for projects and libraries for use by extension commands.
85 #
86 # Values can use environment variables with Bash syntax (e.g. ${HOME}).
87
88 import sys
89 import os
90 import subprocess
91 from collections import Callable
92 from threading import Thread
93 from argparse import ArgumentParser, REMAINDER
94 from os.path import join, dirname, exists, getmtime, isabs, expandvars, isdir
95 import shlex
96 import types
97 import urllib2
98 import contextlib
99 import StringIO
100 import zipfile
101 import shutil, fnmatch, re, xml.dom.minidom
102
103 DEFAULT_JAVA_ARGS = '-ea -Xss2m -Xmx1g'
104
105 class Dependency:
106 def __init__(self, name, baseDir):
107 self.name = name
108 self.baseDir = baseDir
109 self.env = None
110
111 def __str__(self):
112 return self.name
113
114 def __eq__(self, other):
115 return self.name == other.name
116
117 def __ne__(self, other):
118 return self.name != other.name
119
120 def __hash__(self):
121 return hash(self.name)
122
123 def isLibrary(self):
124 return isinstance(self, Library)
125
126 class Project(Dependency):
127 def __init__(self, baseDir, name, srcDirs, deps):
128 Dependency.__init__(self, name, baseDir)
129 self.srcDirs = srcDirs
130 self.deps = deps
131 self.checkstyleProj = name
132 self.dir = join(baseDir, name)
133 self.native = False
134
135 def all_deps(self, deps, pdb, includeLibs):
136 if self in deps:
137 return deps
138 for name in self.deps:
139 assert name != self.name
140 dep = pdb.libs.get(name, None)
141 if dep is not None:
142 if includeLibs and not dep in deps:
143 deps.append(dep)
144 else:
145 dep = pdb.project(name)
146 if not dep in deps:
147 dep.all_deps(deps, pdb, includeLibs)
148 if not self in deps:
149 deps.append(self)
150 return deps
151
152 def _compute_max_dep_distances(self, name, distances, dist, pdb):
153 currentDist = distances.get(name);
154 if currentDist is None or currentDist < dist:
155 distances[name] = dist
156 if pdb.projects.has_key(name):
157 p = pdb.project(name)
158 for dep in p.deps:
159 self._compute_max_dep_distances(dep, distances, dist + 1, pdb)
160
161
162 def canonical_deps(self, env, pdb):
163 distances = dict()
164 result = set()
165 self._compute_max_dep_distances(self.name, distances, 0, pdb)
166 for n,d in distances.iteritems():
167 assert d > 0 or n == self.name
168 if d == 1:
169 result.add(n)
170
171
172 if len(result) == len(self.deps) and frozenset(self.deps) == result:
173 return self.deps
174 return result;
175
176
177 def source_dirs(self):
178 return [join(self.baseDir, self.name, s) for s in self.srcDirs]
179
180 def output_dir(self):
181 return join(self.baseDir, self.name, 'bin')
182
183 def classpath(self, resolve, env):
184 classesDir = join(self.baseDir, 'classes')
185 if exists(classesDir):
186 return [self.output_dir(), classesDir]
187 return [self.output_dir()]
188
189
190
191 class Library(Dependency):
192 def __init__(self, baseDir, name, path, mustExist, urls):
193 Dependency.__init__(self, name, baseDir)
194 self.path = path
195 self.urls = urls
196 self.mustExist = mustExist
197
198 def classpath(self, resolve, env):
199 path = self.path
200 if not isabs(path):
201 path = join(self.baseDir, path)
202 if resolve and self.mustExist and not exists(path):
203 assert not len(self.urls) == 0, 'cannot find required library ' + self.name + " " + path;
204 env.download(path, self.urls)
205
206 if exists(path) or not resolve:
207 return [path]
208 return []
209
210 class ProjectsDB():
211
212 def __init__(self, env):
213 self.env = env
214 self.projects = dict()
215 self.libs = dict()
216 self.commandModules = dict()
217 self.baseDirs = []
218 self.primary = ''
219
220 def _load_projects(self, mxDir, baseDir):
221 env = self.env
222 libsMap = dict()
223 projsMap = dict()
224 projectsFile = join(mxDir, 'projects')
225 if not exists(projectsFile):
226 return
227 with open(projectsFile) as f:
228 for line in f:
229 line = line.strip()
230 if len(line) != 0 and line[0] != '#':
231 key, value = line.split('=', 1)
232
233 parts = key.split('@')
234 if len(parts) != 3:
235 env.abort('Property name does not have 3 parts separated by "@": ' + key)
236 kind, name, attr = parts
237 if kind == 'project':
238 m = projsMap
239 elif kind == 'library':
240 m = libsMap
241 else:
242 env.abort('Property name does not start with "project@" or "library@": ' + key)
243
244 attrs = m.get(name)
245 if attrs is None:
246 attrs = dict()
247 m[name] = attrs
248 value = env.expandvars_in_property(value)
249 attrs[attr] = value
250
251 def pop_list(attrs, name):
252 v = attrs.pop(name, None)
253 if v is None or len(v.strip()) == 0:
254 return []
255 return [n.strip() for n in v.split(',')]
256
257 for name, attrs in projsMap.iteritems():
258 if self.projects.has_key(name):
259 env.abort('cannot override project ' + name + ' in ' + self.project(name).baseDir + " with project of the same name in " + mxDir)
260 srcDirs = pop_list(attrs, 'sourceDirs')
261 deps = pop_list(attrs, 'dependencies')
262 p = Project(baseDir, name, srcDirs, deps)
263 p.checkstyleProj = attrs.pop('checkstyle', name)
264 p.native = attrs.pop('native', '') == 'true'
265 p.__dict__.update(attrs)
266 self.projects[name] = p
267
268 for name, attrs in libsMap.iteritems():
269 if self.libs.has_key(name):
270 env.abort('cannot redefine library ' + name)
271
272 path = attrs['path']
273 mustExist = attrs.pop('optional', 'false') != 'true'
274 urls = pop_list(attrs, 'urls')
275 l = Library(baseDir, name, path, mustExist, urls)
276 l.__dict__.update(attrs)
277 self.libs[name] = l
278
279 def _load_commands(self, mxDir, baseDir):
280 env = self.env
281 commands = join(mxDir, 'commands.py')
282 if exists(commands):
283 # temporarily extend the Python path
284 sys.path.insert(0, mxDir)
285
286 mod = __import__('commands')
287
288 # revert the Python path
289 del sys.path[0]
290
291 if not hasattr(mod, 'mx_init'):
292 env.abort(commands + ' must define an mx_init(env) function')
293
294 mod.mx_init(env)
295
296 name = baseDir + '.commands'
297 sfx = 1
298 while sys.modules.has_key(name):
299 name = baseDir + str(sfx) + '.commands'
300 sfx += 1
301
302 sys.modules[name] = sys.modules.pop('commands')
303 self.commandModules[name] = mod
304
305 def _load_includes(self, mxDir, baseDir):
306 includes = join(mxDir, 'includes')
307 if exists(includes):
308 with open(includes) as f:
309 for line in f:
310 includeMxDir = join(self.env.expandvars_in_property(line.strip()), 'mx')
311 self.load(includeMxDir)
312
313 def _load_env(self, mxDir, baseDir):
314 env = join(mxDir, 'env')
315 if exists(env):
316 with open(env) as f:
317 for line in f:
318 line = line.strip()
319 if len(line) != 0 and line[0] != '#':
320 key, value = line.split('=', 1)
321 os.environ[key.strip()] = self.env.expandvars_in_property(value.strip())
322
323 def load(self, mxDir, primary=False):
324 """ loads the mx data from a given directory """
325 if not exists(mxDir) or not isdir(mxDir):
326 self.env.abort('Directory does not exist: ' + mxDir)
327 baseDir = dirname(mxDir)
328 if primary:
329 self.primary = baseDir
330 if not baseDir in self.baseDirs:
331 self.baseDirs.append(baseDir)
332 self._load_includes(mxDir, baseDir)
333 self._load_projects(mxDir, baseDir)
334 self._load_env(mxDir, baseDir)
335 if primary:
336 self._load_commands(mxDir, baseDir)
337
338 def project_names(self):
339 return ' '.join(self.projects.keys())
340
341 def project(self, name, fatalIfMissing=True):
342 p = self.projects.get(name)
343 if p is None:
344 self.env.abort('project named ' + name + ' not found')
345 return p
346
347 def library(self, name):
348 l = self.libs.get(name)
349 if l is None:
350 self.env.abort('library named ' + name + ' not found')
351 return l
352
353 def _as_classpath(self, deps, resolve):
354 cp = []
355 if self.env.cp_prefix is not None:
356 cp = [self.env.cp_prefix]
357 for d in deps:
358 cp += d.classpath(resolve, self.env)
359 if self.env.cp_suffix is not None:
360 cp += [self.env.cp_suffix]
361 return os.pathsep.join(cp)
362
363 def classpath(self, names=None, resolve=True):
364 if names is None:
365 return self._as_classpath(self.sorted_deps(True), resolve)
366 deps = []
367 if isinstance(names, types.StringTypes):
368 self.project(names).all_deps(deps, self, True)
369 else:
370 for n in names:
371 self.project(n).all_deps(deps, self, True)
372 return self._as_classpath(deps, resolve)
373
374 def sorted_deps(self, includeLibs=False):
375 deps = []
376 for p in self.projects.itervalues():
377 p.all_deps(deps, self, includeLibs)
378 return deps
379
380 class Env(ArgumentParser):
381
382 def format_commands(self):
383 msg = '\navailable commands:\n\n'
384 for cmd in sorted(self.commands.iterkeys()):
385 c, _ = self.commands[cmd][:2]
386 doc = c.__doc__
387 if doc is None:
388 doc = ''
389 msg += ' {0:<20} {1}\n'.format(cmd, doc.split('\n', 1)[0])
390 return msg + '\n'
391
392 # Override parent to append the list of available commands
393 def format_help(self):
394 return ArgumentParser.format_help(self) + self.format_commands()
395
396
397 def __init__(self):
398 self.java_initialized = False
399 self.pdb = ProjectsDB(self)
400 self.commands = dict()
401 ArgumentParser.__init__(self, prog='mx')
402
403 self.add_argument('-v', action='store_true', dest='verbose', help='enable verbose output')
404 self.add_argument('-d', action='store_true', dest='java_dbg', help='make Java processes wait on port 8000 for a debugger')
405 self.add_argument('--cp-pfx', dest='cp_prefix', help='class path prefix', metavar='<arg>')
406 self.add_argument('--cp-sfx', dest='cp_suffix', help='class path suffix', metavar='<arg>')
407 self.add_argument('--J', dest='java_args', help='Java VM arguments (e.g. --J @-dsa)', metavar='@<args>', default=DEFAULT_JAVA_ARGS)
408 self.add_argument('--Jp', action='append', dest='java_args_pfx', help='prefix Java VM arguments (e.g. --Jp @-dsa)', metavar='@<args>', default=[])
409 self.add_argument('--Ja', action='append', dest='java_args_sfx', help='suffix Java VM arguments (e.g. --Ja @-dsa)', metavar='@<args>', default=[])
410 self.add_argument('--user-home', help='users home directory', metavar='<path>', default=os.path.expanduser('~'))
411 self.add_argument('--java-home', help='JDK installation directory (must be JDK 6 or later)', metavar='<path>', default=self.default_java_home())
412 self.add_argument('--java', help='Java VM executable (default: bin/java under $JAVA_HOME)', metavar='<path>')
413 self.add_argument('--os', dest='os', help='operating system override')
414
415 def _parse_cmd_line(self, args=None):
416 if args is None:
417 args = sys.argv[1:]
418
419 self.add_argument('commandAndArgs', nargs=REMAINDER, metavar='command args...')
420
421 self.parse_args(namespace=self)
422
423 if self.java_home is None or self.java_home == '':
424 self.abort('Could not find Java home. Use --java-home option or ensure JAVA_HOME environment variable is set.')
425
426 if self.user_home is None or self.user_home == '':
427 self.abort('Could not find user home. Use --user-home option or ensure HOME environment variable is set.')
428
429 if self.os is None:
430 self.remote = False
431 if sys.platform.startswith('darwin'):
432 self.os = 'darwin'
433 elif sys.platform.startswith('linux'):
434 self.os = 'linux'
435 elif sys.platform.startswith('sunos'):
436 self.os = 'solaris'
437 elif sys.platform.startswith('win32') or sys.platform.startswith('cygwin'):
438 self.os = 'windows'
439 else:
440 print 'Supported operating system could not be derived from', sys.platform, '- use --os option explicitly.'
441 sys.exit(1)
442 else:
443 self.java_args += ' -Dmax.os=' + self.os
444 self.remote = True
445
446 if self.java is None:
447 self.java = join(self.java_home, 'bin', 'java')
448
449 os.environ['JAVA_HOME'] = self.java_home
450 os.environ['HOME'] = self.user_home
451
452 self.javac = join(self.java_home, 'bin', 'javac')
453
454 for mod in self.pdb.commandModules.itervalues():
455 if hasattr(mod, 'mx_post_parse_cmd_line'):
456 mod.mx_post_parse_cmd_line(self)
457
458 def expandvars_in_property(self, value):
459 result = expandvars(value)
460 if '$' in result or '%' in result:
461 self.abort('Property contains an undefined environment variable: ' + value)
462 return result
463
464
465 def load_config_file(self, configFile, override=False):
466 """ adds attributes to this object from a file containing key=value lines """
467 if exists(configFile):
468 with open(configFile) as f:
469 for line in f:
470 k, v = line.split('=', 1)
471 k = k.strip().lower()
472 if (override or not hasattr(self, k)):
473 setattr(self, k, self.expandvars_in_property(v.strip()))
474
475 def format_java_cmd(self, args):
476 self.init_java()
477 return [self.java] + self.java_args_pfx + self.java_args + self.java_args_sfx + args
478
479 def run_java(self, args, nonZeroIsFatal=True, out=None, err=None, cwd=None):
480 return self.run(self.format_java_cmd(args), nonZeroIsFatal=nonZeroIsFatal, out=out, err=err, cwd=cwd)
481
482 def run(self, args, nonZeroIsFatal=True, out=None, err=None, cwd=None):
483 """
484 Run a command in a subprocess, wait for it to complete and return the exit status of the process.
485 If the exit status is non-zero and `nonZeroIsFatal` is true, then the program is exited with
486 the same exit status.
487 Each line of the standard output and error streams of the subprocess are redirected to the
488 provided out and err functions if they are not None.
489 """
490
491 assert isinstance(args, types.ListType), "'args' must be a list: " + str(args)
492 for arg in args:
493 assert isinstance(arg, types.StringTypes), 'argument is not a string: ' + str(arg)
494
495 if self.verbose:
496 self.log(' '.join(args))
497
498 try:
499 if out is None and err is None:
500 retcode = subprocess.call(args, cwd=cwd)
501 else:
502 def redirect(stream, f):
503 for line in iter(stream.readline, ''):
504 f(line)
505 stream.close()
506 p = subprocess.Popen(args, cwd=cwd, stdout=None if out is None else subprocess.PIPE, stderr=None if err is None else subprocess.PIPE)
507 if out is not None:
508 t = Thread(target=redirect, args=(p.stdout, out))
509 t.daemon = True # thread dies with the program
510 t.start()
511 if err is not None:
512 t = Thread(target=redirect, args=(p.stderr, err))
513 t.daemon = True # thread dies with the program
514 t.start()
515 retcode = p.wait()
516 except OSError as e:
517 self.log('Error executing \'' + ' '.join(args) + '\': ' + str(e))
518 if self.verbose:
519 raise e
520 self.abort(e.errno)
521
522
523 if retcode and nonZeroIsFatal:
524 if self.verbose:
525 raise subprocess.CalledProcessError(retcode, ' '.join(args))
526 self.abort(retcode)
527
528 return retcode
529
530 def check_get_env(self, key):
531 """
532 Gets an environment variable, aborting with a useful message if it is not set.
533 """
534 value = os.environ.get(key)
535 if value is None:
536 self.abort('Required environment variable ' + key + ' must be set (e.g. in ' + join(self.pdb.primary, 'env') + ')')
537 return value
538
539 def exe_suffix(self, name):
540 """
541 Gets the platform specific suffix for an executable
542 """
543 if self.os == 'windows':
544 return name + '.exe'
545 return name
546
547 def log(self, msg=None):
548 """
549 Write a message to the console.
550 All script output goes through this method thus allowing a subclass
551 to redirect it.
552 """
553 if msg is None:
554 print
555 else:
556 print msg
557
558 def expand_project_in_class_path_arg(self, cpArg):
559 cp = []
560 for part in cpArg.split(os.pathsep):
561 if part.startswith('@'):
562 cp += self.pdb.classpath(part[1:]).split(os.pathsep)
563 else:
564 cp.append(part)
565 return os.pathsep.join(cp)
566
567 def expand_project_in_args(self, args):
568 for i in range(len(args)):
569 if args[i] == '-cp' or args[i] == '-classpath':
570 if i + 1 < len(args):
571 args[i + 1] = self.expand_project_in_class_path_arg(args[i + 1])
572 return
573
574
575 def init_java(self):
576 """
577 Lazy initialization and preprocessing of this object's fields before running a Java command.
578 """
579 if self.java_initialized:
580 return
581
582 def delAtAndSplit(s):
583 return shlex.split(s.lstrip('@'))
584
585 self.java_args = delAtAndSplit(self.java_args)
586 self.java_args_pfx = sum(map(delAtAndSplit, self.java_args_pfx), [])
587 self.java_args_sfx = sum(map(delAtAndSplit, self.java_args_sfx), [])
588
589 # Prepend the -d64 VM option only if the java command supports it
590 output = ''
591 try:
592 output = subprocess.check_output([self.java, '-d64', '-version'], stderr=subprocess.STDOUT)
593 self.java_args = ['-d64'] + self.java_args
594 except subprocess.CalledProcessError as e:
595 try:
596 output = subprocess.check_output([self.java, '-version'], stderr=subprocess.STDOUT)
597 except subprocess.CalledProcessError as e:
598 print e.output
599 self.abort(e.returncode)
600
601 output = output.split()
602 assert output[0] == 'java' or output[0] == 'openjdk'
603 assert output[1] == 'version'
604 version = output[2]
605 if not version.startswith('"1.6') and not version.startswith('"1.7'):
606 self.abort('Requires Java version 1.6 or 1.7, got version ' + version)
607
608 if self.java_dbg:
609 self.java_args += ['-Xdebug', '-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=8000']
610
611 self.java_initialized = True
612
613 def default_java_home(self):
614 javaHome = os.getenv('JAVA_HOME')
615 if javaHome is None:
616 if exists('/usr/lib/java/java-6-sun'):
617 javaHome = '/usr/lib/java/java-6-sun'
618 elif exists('/System/Library/Frameworks/JavaVM.framework/Versions/1.6/Home'):
619 javaHome = '/System/Library/Frameworks/JavaVM.framework/Versions/1.6/Home'
620 elif exists('/usr/jdk/latest'):
621 javaHome = '/usr/jdk/latest'
622 return javaHome
623
624 def gmake_cmd(self):
625 for a in ['make', 'gmake', 'gnumake']:
626 try:
627 output = subprocess.check_output([a, '--version'])
628 if 'GNU' in output:
629 return a;
630 except:
631 pass
632 self.abort('Could not find a GNU make executable on the current path.')
633
634
635 def abort(self, codeOrMessage):
636 """
637 Aborts the program with a SystemExit exception.
638 If 'codeOrMessage' is a plain integer, it specifies the system exit status;
639 if it is None, the exit status is zero; if it has another type (such as a string),
640 the object's value is printed and the exit status is one.
641 """
642 raise SystemExit(codeOrMessage)
643
644 def download(self, path, urls):
645 """
646 Attempts to downloads content for each URL in a list, stopping after the first successful download.
647 If the content cannot be retrieved from any URL, the program is aborted. The downloaded content
648 is written to the file indicated by 'path'.
649 """
650 d = dirname(path)
651 if d != '' and not exists(d):
652 os.makedirs(d)
653
654 def url_open(url):
655 userAgent = 'Mozilla/5.0 (compatible)'
656 headers = { 'User-Agent' : userAgent }
657 req = urllib2.Request(url, headers=headers)
658 return urllib2.urlopen(req);
659
660 for url in urls:
661 try:
662 self.log('Downloading ' + url + ' to ' + path)
663 if url.startswith('zip:') or url.startswith('jar:'):
664 i = url.find('!/')
665 if i == -1:
666 self.abort('Zip or jar URL does not contain "!/": ' + url)
667 url, _, entry = url[len('zip:'):].partition('!/')
668 with contextlib.closing(url_open(url)) as f:
669 data = f.read()
670 zipdata = StringIO.StringIO(f.read())
671
672 zf = zipfile.ZipFile(zipdata, 'r')
673 data = zf.read(entry)
674 with open(path, 'w') as f:
675 f.write(data)
676 else:
677 with contextlib.closing(url_open(url)) as f:
678 data = f.read()
679 with open(path, 'w') as f:
680 f.write(data)
681 return
682 except IOError as e:
683 self.log('Error reading from ' + url + ': ' + str(e))
684 except zipfile.BadZipfile as e:
685 self.log('Error in zip file downloaded from ' + url + ': ' + str(e))
686
687 # now try it with Java - urllib2 does not handle meta refreshes which are used by Sourceforge
688 myDir = dirname(__file__)
689
690 javaSource = join(myDir, 'URLConnectionDownload.java')
691 javaClass = join(myDir, 'URLConnectionDownload.class')
692 if not exists(javaClass) or getmtime(javaClass) < getmtime(javaSource):
693 subprocess.check_call([self.javac, '-d', myDir, javaSource])
694 if self.run([self.java, '-cp', myDir, 'URLConnectionDownload', path] + urls) != 0:
695 self.abort('Could not download to ' + path + ' from any of the following URLs:\n\n ' +
696 '\n '.join(urls) + '\n\nPlease use a web browser to do the download manually')
697
698 def update_file(self, path, content):
699 """
700 Updates a file with some given content if the content differs from what's in
701 the file already. The return value indicates if the file was updated.
702 """
703 existed = exists(path)
704 try:
705 old = None
706 if existed:
707 with open(path) as f:
708 old = f.read()
709
710 if old == content:
711 return False
712
713 with open(path, 'w') as f:
714 f.write(content)
715
716 self.log(('modified ' if existed else 'created ') + path)
717 return True;
718 except IOError as e:
719 self.abort('Error while writing to ' + path + ': ' + str(e));
720
721 # Builtin commands
722
723 def build(env, args):
724 """compile the Java and C sources, linking the latter
725
726 Compile all the Java source code using the appropriate compilers
727 and linkers for the various source code types."""
728
729 parser = ArgumentParser(prog='mx build');
730 parser.add_argument('-f', action='store_true', dest='force', help='force compilation even if class files are up to date')
731 parser.add_argument('-c', action='store_true', dest='clean', help='removes existing build output')
732 parser.add_argument('--no-native', action='store_false', dest='native', help='do not build com.oracle.max.vm.native')
733 parser.add_argument('--jdt', help='Eclipse installation or path to ecj.jar for using the Eclipse batch compiler instead of javac', metavar='<path>')
734
735 args = parser.parse_args(args)
736
737 jdtJar = None
738 if args.jdt is not None:
739 if args.jdt.endswith('.jar'):
740 jdtJar=args.jdt
741 elif isdir(args.jdt):
742 plugins = join(args.jdt, 'plugins')
743 choices = [f for f in os.listdir(plugins) if fnmatch.fnmatch(f, 'org.eclipse.jdt.core_*.jar')]
744 if len(choices) != 0:
745 jdtJar = join(plugins, sorted(choices, reverse=True)[0])
746
747 projects = [p.name for p in env.pdb.sorted_deps()]
748 built = set()
749 for project in projects:
750 p = env.pdb.project(project)
751 projectDir = join(p.baseDir, project)
752
753 if p.native:
754 if env.os == 'windows':
755 env.log('Skipping C compilation on Windows until it is supported')
756 pass
757
758 env.log('Compiling C sources in {0}...'.format(projectDir))
759
760 if args.clean:
761 env.run([env.gmake_cmd(), 'clean'], cwd=projectDir)
762
763 env.run([env.gmake_cmd()], cwd=projectDir)
764 built.add(project)
765 continue
766
767 outputDir = p.output_dir()
768 if exists(outputDir):
769 if args.clean:
770 env.log('Cleaning {0}...'.format(outputDir))
771 shutil.rmtree(outputDir)
772 os.mkdir(outputDir)
773 else:
774 os.mkdir(outputDir)
775
776 classpath = env.pdb.classpath(project)
777 sourceDirs = env.pdb.project(project).source_dirs()
778 mustBuild = args.force
779 if not mustBuild:
780 for dep in p.all_deps([], env.pdb, False):
781 if dep.name in built:
782 mustBuild = True
783
784 for sourceDir in sourceDirs:
785 javafilelist = []
786 nonjavafilelist = []
787 for root, _, files in os.walk(sourceDir):
788 javafiles = [join(root, name) for name in files if name.endswith('.java') and name != 'package-info.java']
789 javafilelist += javafiles
790 nonjavafilelist += [join(root, name) for name in files if not name.endswith('.java')]
791 if not mustBuild:
792 for javafile in javafiles:
793 classfile = outputDir + javafile[len(sourceDir):-len('java')] + 'class'
794 if not exists(classfile) or os.path.getmtime(javafile) > os.path.getmtime(classfile):
795 mustBuild = True
796 break
797
798 if not mustBuild:
799 env.log('[all class files in {0} are up to date - skipping]'.format(sourceDir))
800 continue
801
802 if len(javafilelist) == 0:
803 env.log('[no Java sources in {0} - skipping]'.format(sourceDir))
804 continue
805
806 built.add(project)
807
808 argfileName = join(projectDir, 'javafilelist.txt')
809 argfile = open(argfileName, 'w')
810 argfile.write('\n'.join(javafilelist))
811 argfile.close()
812
813 try:
814 if jdtJar is None:
815 env.log('Compiling Java sources in {0} with javac...'.format(sourceDir))
816
817 class Filter:
818 """
819 Class to filter the 'is Sun proprietary API and may be removed in a future release'
820 warning when compiling the VM classes.
821
822 """
823 def __init__(self):
824 self.c = 0
825
826 def eat(self, line):
827 if 'Sun proprietary API' in line:
828 self.c = 2
829 elif self.c != 0:
830 self.c -= 1
831 else:
832 print line.rstrip()
833
834 env.run([env.javac, '-g', '-J-Xmx1g', '-classpath', classpath, '-d', outputDir, '@' + argfile.name], err=Filter().eat)
835 else:
836 env.log('Compiling Java sources in {0} with JDT...'.format(sourceDir))
837 jdtProperties = join(projectDir, '.settings', 'org.eclipse.jdt.core.prefs')
838 if not exists(jdtProperties):
839 raise SystemError('JDT properties file {0} not found'.format(jdtProperties))
840 env.run([env.java, '-Xmx1g', '-jar', jdtJar, '-1.6', '-cp', classpath, '-g',
841 '-properties', jdtProperties,
842 '-warn:-unusedImport,-unchecked',
843 '-d', outputDir, '@' + argfile.name])
844 finally:
845 os.remove(argfileName)
846
847
848 for name in nonjavafilelist:
849 dst = join(outputDir, name[len(sourceDir) + 1:])
850 if exists(dirname(dst)):
851 shutil.copyfile(name, dst)
852
853 def canonicalizeprojects(env, args):
854 """process all project files to canonicalize the dependencies
855
856 The exit code of this command reflects how many files were updated."""
857
858 changedFiles = 0
859 pdb = env.pdb
860 for d in pdb.baseDirs:
861 projectsFile = join(d, 'mx', 'projects')
862 if not exists(projectsFile):
863 continue
864 with open(projectsFile) as f:
865 out = StringIO.StringIO()
866 pattern = re.compile('project@([^@]+)@dependencies=.*')
867 for line in f:
868 line = line.strip()
869 m = pattern.match(line)
870 if m is None:
871 out.write(line + '\n')
872 else:
873 p = pdb.project(m.group(1))
874 out.write('project@' + m.group(1) + '@dependencies=' + ','.join(p.canonical_deps(env, pdb)) + '\n')
875 content = out.getvalue()
876 if env.update_file(projectsFile, content):
877 changedFiles += 1
878 return changedFiles;
879
880 def checkstyle(env, args):
881 """run Checkstyle on the Java sources
882
883 Run Checkstyle over the Java sources. Any errors or warnings
884 produced by Checkstyle result in a non-zero exit code.
885
886 If no projects are given, then all Java projects are checked."""
887
888 allProjects = [p.name for p in env.pdb.sorted_deps()]
889 if len(args) == 0:
890 projects = allProjects
891 else:
892 projects = args
893 unknown = set(projects).difference(allProjects)
894 if len(unknown) != 0:
895 env.error('unknown projects: ' + ', '.join(unknown))
896
897 for project in projects:
898 p = env.pdb.project(project)
899 projectDir = join(p.baseDir, project)
900 sourceDirs = env.pdb.project(project).source_dirs()
901 dotCheckstyle = join(projectDir, '.checkstyle')
902
903 if not exists(dotCheckstyle):
904 continue
905
906 for sourceDir in sourceDirs:
907 javafilelist = []
908 for root, _, files in os.walk(sourceDir):
909 javafilelist += [join(root, name) for name in files if name.endswith('.java') and name != 'package-info.java']
910 if len(javafilelist) == 0:
911 env.log('[no Java sources in {0} - skipping]'.format(sourceDir))
912 continue
913
914 timestampFile = join(p.baseDir, 'mx', '.checkstyle' + sourceDir[len(p.baseDir):].replace(os.sep, '_') + '.timestamp')
915 mustCheck = False
916 if exists(timestampFile):
917 timestamp = os.path.getmtime(timestampFile)
918 for f in javafilelist:
919 if os.path.getmtime(f) > timestamp:
920 mustCheck = True
921 break
922 else:
923 mustCheck = True
924
925 if not mustCheck:
926 env.log('[all Java sources in {0} already checked - skipping]'.format(sourceDir))
927 continue
928
929 if exists(timestampFile):
930 os.utime(timestampFile, None)
931 else:
932 file(timestampFile, 'a')
933
934 dotCheckstyleXML = xml.dom.minidom.parse(dotCheckstyle)
935 localCheckConfig = dotCheckstyleXML.getElementsByTagName('local-check-config')[0]
936 configLocation = localCheckConfig.getAttribute('location')
937 if configLocation.startswith('/'):
938 config = join(p.baseDir, configLocation.lstrip('/'))
939 else:
940 config = join(projectDir, configLocation)
941
942 exclude = join(projectDir, '.checkstyle.exclude')
943
944
945
946
947 if exists(exclude):
948 with open(exclude) as f:
949 # Convert patterns to OS separators
950 patterns = [name.rstrip().replace('/', os.sep) for name in f.readlines()]
951 def match(name):
952 for p in patterns:
953 if p in name:
954 env.log('excluding: ' + name)
955 return True
956 return False
957
958 javafilelist = [name for name in javafilelist if not match(name)]
959
960 auditfileName = join(projectDir, 'checkstyleOutput.txt')
961 env.log('Running Checkstyle on {0} using {1}...'.format(sourceDir, config))
962
963 try:
964
965 # Checkstyle is unable to read the filenames to process from a file, and the
966 # CreateProcess function on Windows limits the length of a command line to
967 # 32,768 characters (http://msdn.microsoft.com/en-us/library/ms682425%28VS.85%29.aspx)
968 # so calling Checkstyle must be done in batches.
969 while len(javafilelist) != 0:
970 i = 0
971 size = 0
972 while i < len(javafilelist):
973 s = len(javafilelist[i]) + 1
974 if (size + s < 30000):
975 size += s
976 i += 1
977 else:
978 break
979
980 batch = javafilelist[:i]
981 javafilelist = javafilelist[i:]
982 try:
983 env.run_java(['-Xmx1g', '-jar', env.pdb.library('CHECKSTYLE').classpath(True, env)[0], '-c', config, '-o', auditfileName] + batch)
984 finally:
985 if exists(auditfileName):
986 with open(auditfileName) as f:
987 warnings = [line.strip() for line in f if 'warning:' in line]
988 if len(warnings) != 0:
989 map(env.log, warnings)
990 return 1
991 finally:
992 if exists(auditfileName):
993 os.unlink(auditfileName)
994 return 0
995
996 def clean(env, args):
997 """remove all class files, images, and executables
998
999 Removes all files created by a build, including Java class files, executables, and
1000 generated images.
1001 """
1002
1003 projects = env.pdb.projects.keys()
1004 for project in projects:
1005 p = env.pdb.project(project)
1006 if p.native:
1007 env.run([env.gmake_cmd(), '-C', p.dir, 'clean'])
1008 else:
1009 outputDir = p.output_dir()
1010 if outputDir != '' and exists(outputDir):
1011 env.log('Removing {0}...'.format(outputDir))
1012 shutil.rmtree(outputDir)
1013
1014 def help_(env, args):
1015 """show help for a given command
1016
1017 With no arguments, print a list of commands and short help for each command.
1018
1019 Given a command name, print help for that command."""
1020 if len(args) == 0:
1021 env.print_help()
1022 return
1023
1024 name = args[0]
1025 if not env.commands.has_key(name):
1026 env.error('unknown command: ' + name)
1027
1028 value = env.commands[name]
1029 (func, usage) = value[:2]
1030 doc = func.__doc__
1031 if len(value) > 2:
1032 docArgs = value[2:]
1033 fmtArgs = []
1034 for d in docArgs:
1035 if isinstance(d, Callable):
1036 fmtArgs += [d(env)]
1037 else:
1038 fmtArgs += [str(d)]
1039 doc = doc.format(*fmtArgs)
1040 print 'mx {0} {1}\n\n{2}\n'.format(name, usage, doc)
1041
1042
1043 # Commands are in alphabetical order in this file.
1044
1045 def javap(env, args):
1046 """launch javap with a -classpath option denoting all available classes
1047
1048 Run the JDK javap class file disassembler with the following prepended options:
1049
1050 -private -verbose -classpath <path to project classes>"""
1051
1052 javap = join(env.java_home, 'bin', 'javap')
1053 if not exists(javap):
1054 env.abort('The javap executable does not exists: ' + javap)
1055 else:
1056 env.run([javap, '-private', '-verbose', '-classpath', env.pdb.classpath()] + args)
1057
1058 def projects(env, args):
1059 """show all loaded projects"""
1060 pdb = env.pdb
1061 for d in pdb.baseDirs:
1062 projectsFile = join(d, 'mx', 'projects')
1063 if exists(projectsFile):
1064 env.log('# file: ' + projectsFile)
1065 for p in pdb.projects.values():
1066 if p.baseDir == d:
1067 env.log(p.name)
1068
1069
1070 def main(env):
1071
1072 # Table of commands in alphabetical order.
1073 # Keys are command names, value are lists: [<function>, <usage msg>, <format args to doc string of function>...]
1074 # If any of the format args are instances of Callable, then they are called with an 'env' are before being
1075 # used in the call to str.format().
1076 # Extensions should update this table directly
1077 env.commands = {
1078 'build': [build, '[options] projects...'],
1079 'checkstyle': [checkstyle, 'projects...'],
1080 'canonicalizeprojects': [canonicalizeprojects, ''],
1081 'clean': [clean, ''],
1082 'help': [help_, '[command]'],
1083 'javap': [javap, ''],
1084 'projects': [projects, ''],
1085 }
1086
1087 MX_INCLUDES = os.environ.get('MX_INCLUDES', None)
1088 if MX_INCLUDES is not None:
1089 for path in MX_INCLUDES.split(os.pathsep):
1090 d = join(path, 'mx')
1091 if exists(d) and isdir(d):
1092 env.pdb.load(d)
1093
1094 cwdMxDir = join(os.getcwd(), 'mx')
1095 if exists(cwdMxDir) and isdir(cwdMxDir):
1096 env.pdb.load(cwdMxDir, primary=True)
1097
1098 env._parse_cmd_line()
1099
1100 if len(env.commandAndArgs) == 0:
1101 env.print_help()
1102 return
1103
1104 env.command = env.commandAndArgs[0]
1105 env.command_args = env.commandAndArgs[1:]
1106
1107 if not env.commands.has_key(env.command):
1108 env.abort('mx: unknown command \'{0}\'\n{1}use "mx help" for more options'.format(env.command, env.format_commands()))
1109
1110 c, _ = env.commands[env.command][:2]
1111 try:
1112 retcode = c(env, env.command_args)
1113 if retcode is not None and retcode != 0:
1114 env.abort(retcode)
1115 except KeyboardInterrupt:
1116 # no need to show the stack trace when the user presses CTRL-C
1117 env.abort(1)
1118
1119 if __name__ == '__main__':
1120 main(Env())