Mercurial > hg > truffle

comparison mxtool/mx.py @ 15528:a3f897fb3289

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
Merge.
author Thomas Wuerthinger <thomas.wuerthinger@oracle.com>
date Mon, 05 May 2014 22:18:46 +0200
parents 1f28c463e452
children 406a94c03ffa
comparison
equal deleted inserted replaced
15527:ff5cacf47b68 15528:a3f897fb3289
1298 abort('Process timed out after {0} seconds: {1}'.format(timeout, ' '.join(args))) 1298 abort('Process timed out after {0} seconds: {1}'.format(timeout, ' '.join(args)))
1299 delay = min(delay * 2, remaining, .05) 1299 delay = min(delay * 2, remaining, .05)
1300 time.sleep(delay) 1300 time.sleep(delay)
1301 1301
1302 # Makes the current subprocess accessible to the abort() function 1302 # Makes the current subprocess accessible to the abort() function
1303 # This is a tuple of the Popen object and args. 1303 # This is a list of tuples of the subprocess.Popen or
1304 _currentSubprocess = (None, None) 1304 # multiprocessing.Process object and args.
1305 _currentSubprocesses = []
1306
1307 def _addSubprocess(p, args):
1308 entry = (p, args)
1309 _currentSubprocesses.append(entry)
1310 return entry
1311
1312 def _removeSubprocess(entry):
1313 if entry and entry in _currentSubprocesses:
1314 try:
1315 _currentSubprocesses.remove(entry)
1316 except:
1317 pass
1305 1318
1306 def waitOn(p): 1319 def waitOn(p):
1307 if get_os() == 'windows': 1320 if get_os() == 'windows':
1308 # on windows use a poll loop, otherwise signal does not get handled 1321 # on windows use a poll loop, otherwise signal does not get handled
1309 retcode = None 1322 retcode = None
1338 log(' '.join(map(pipes.quote, args))) 1351 log(' '.join(map(pipes.quote, args)))
1339 1352
1340 if timeout is None and _opts.ptimeout != 0: 1353 if timeout is None and _opts.ptimeout != 0:
1341 timeout = _opts.ptimeout 1354 timeout = _opts.ptimeout
1342 1355
1343 global _currentSubprocess 1356 sub = None
1344
1345 try: 1357 try:
1346 # On Unix, the new subprocess should be in a separate group so that a timeout alarm 1358 # On Unix, the new subprocess should be in a separate group so that a timeout alarm
1347 # can use os.killpg() to kill the whole subprocess group 1359 # can use os.killpg() to kill the whole subprocess group
1348 preexec_fn = None 1360 preexec_fn = None
1349 creationflags = 0 1361 creationflags = 0
1357 f(line) 1369 f(line)
1358 stream.close() 1370 stream.close()
1359 stdout = out if not callable(out) else subprocess.PIPE 1371 stdout = out if not callable(out) else subprocess.PIPE
1360 stderr = err if not callable(err) else subprocess.PIPE 1372 stderr = err if not callable(err) else subprocess.PIPE
1361 p = subprocess.Popen(args, cwd=cwd, stdout=stdout, stderr=stderr, preexec_fn=preexec_fn, creationflags=creationflags, env=env) 1373 p = subprocess.Popen(args, cwd=cwd, stdout=stdout, stderr=stderr, preexec_fn=preexec_fn, creationflags=creationflags, env=env)
1362 _currentSubprocess = (p, args) 1374 sub = _addSubprocess(p, args)
1363 if callable(out): 1375 if callable(out):
1364 t = Thread(target=redirect, args=(p.stdout, out)) 1376 t = Thread(target=redirect, args=(p.stdout, out))
1365 t.daemon = True # thread dies with the program 1377 t.daemon = True # thread dies with the program
1366 t.start() 1378 t.start()
1367 if callable(err): 1379 if callable(err):
1380 raise e 1392 raise e
1381 abort(e.errno) 1393 abort(e.errno)
1382 except KeyboardInterrupt: 1394 except KeyboardInterrupt:
1383 abort(1) 1395 abort(1)
1384 finally: 1396 finally:
1385 _currentSubprocess = (None, None) 1397 _removeSubprocess(sub)
1386 1398
1387 if retcode and nonZeroIsFatal: 1399 if retcode and nonZeroIsFatal:
1388 if _opts.verbose: 1400 if _opts.verbose:
1389 if _opts.very_verbose: 1401 if _opts.very_verbose:
1390 raise subprocess.CalledProcessError(retcode, ' '.join(args)) 1402 raise subprocess.CalledProcessError(retcode, ' '.join(args))
1661 if '$' in result or '%' in result: 1673 if '$' in result or '%' in result:
1662 abort('Property contains an undefined environment variable: ' + value) 1674 abort('Property contains an undefined environment variable: ' + value)
1663 return result 1675 return result
1664 1676
1665 def _send_sigquit(): 1677 def _send_sigquit():
1666 p, args = _currentSubprocess 1678 for p, args in _currentSubprocesses:
1667 1679
1668 def _isJava(): 1680 def _isJava():
1669 if args: 1681 if args:
1670 name = args[0].split(os.sep)[-1] 1682 name = args[0].split(os.sep)[-1]
1671 return name == "java" 1683 return name == "java"
1672 return False 1684 return False
1673 1685
1674 if p is not None and _isJava(): 1686 if p is not None and _isJava():
1675 if get_os() == 'windows': 1687 if get_os() == 'windows':
1676 log("mx: implement me! want to send SIGQUIT to my child process") 1688 log("mx: implement me! want to send SIGQUIT to my child process")
1677 else: 1689 else:
1678 _kill_process_group(p.pid, sig=signal.SIGQUIT) 1690 _kill_process_group(p.pid, sig=signal.SIGQUIT)
1679 time.sleep(0.1) 1691 time.sleep(0.1)
1680
1681 1692
1682 def abort(codeOrMessage): 1693 def abort(codeOrMessage):
1683 """ 1694 """
1684 Aborts the program with a SystemExit exception. 1695 Aborts the program with a SystemExit exception.
1685 If 'codeOrMessage' is a plain integer, it specifies the system exit status; 1696 If 'codeOrMessage' is a plain integer, it specifies the system exit status;
1690 if _opts.killwithsigquit: 1701 if _opts.killwithsigquit:
1691 _send_sigquit() 1702 _send_sigquit()
1692 1703
1693 # import traceback 1704 # import traceback
1694 # traceback.print_stack() 1705 # traceback.print_stack()
1695 p, _ = _currentSubprocess 1706 for p, args in _currentSubprocesses:
1696 if p is not None: 1707 try:
1697 if get_os() == 'windows': 1708 if get_os() == 'windows':
1698 p.kill() 1709 p.terminate()
1699 else: 1710 else:
1700 _kill_process_group(p.pid, signal.SIGKILL) 1711 _kill_process_group(p.pid, signal.SIGKILL)
1712 except BaseException as e:
1713 log('error while killing subprocess {} "{}": {}'.format(p.pid, ' '.join(args), e))
1701 1714
1702 raise SystemExit(codeOrMessage) 1715 raise SystemExit(codeOrMessage)
1703 1716
1704 def download(path, urls, verbose=False): 1717 def download(path, urls, verbose=False):
1705 """ 1718 """
1796 1809
1797 def _defaultEcjPath(): 1810 def _defaultEcjPath():
1798 return get_env('JDT', join(_primary_suite.mxDir, 'ecj.jar')) 1811 return get_env('JDT', join(_primary_suite.mxDir, 'ecj.jar'))
1799 1812
1800 class JavaCompileTask: 1813 class JavaCompileTask:
1801 def __init__(self, args, proj, reason, javafilelist, jdk, outputDir, deps): 1814 def __init__(self, args, proj, reason, javafilelist, jdk, outputDir, jdtJar, deps):
1802 self.proj = proj 1815 self.proj = proj
1803 self.reason = reason 1816 self.reason = reason
1804 self.javafilelist = javafilelist 1817 self.javafilelist = javafilelist
1805 self.deps = deps 1818 self.deps = deps
1806 self.jdk = jdk 1819 self.jdk = jdk
1807 self.outputDir = outputDir 1820 self.outputDir = outputDir
1808 self.done = False 1821 self.done = False
1822 self.jdtJar = jdtJar
1809 self.args = args 1823 self.args = args
1810 1824
1811 def __str__(self): 1825 def __str__(self):
1812 return self.proj.name 1826 return self.proj.name
1813 1827
1838 outputDir = self.outputDir 1852 outputDir = self.outputDir
1839 compliance = str(jdk.javaCompliance) 1853 compliance = str(jdk.javaCompliance)
1840 cp = classpath(self.proj.name, includeSelf=True) 1854 cp = classpath(self.proj.name, includeSelf=True)
1841 toBeDeleted = [argfileName] 1855 toBeDeleted = [argfileName]
1842 1856
1843 jdtJar = None
1844 if not args.javac and args.jdt is not None:
1845 if not args.jdt.endswith('.jar'):
1846 abort('Path for Eclipse batch compiler does not look like a jar file: ' + args.jdt)
1847 jdtJar = args.jdt
1848 if not exists(jdtJar):
1849 if os.path.abspath(jdtJar) == os.path.abspath(_defaultEcjPath()) and get_env('JDT', None) is None:
1850 # Silently ignore JDT if default location is used but does not exist
1851 jdtJar = None
1852 else:
1853 abort('Eclipse batch compiler jar does not exist: ' + args.jdt)
1854
1855 try: 1857 try:
1856 if not jdtJar: 1858 if not self.jdtJar:
1857 mainJava = java() 1859 mainJava = java()
1858 if not args.error_prone: 1860 if not args.error_prone:
1859 self.logCompilation('javac') 1861 self.logCompilation('javac')
1860 javacCmd = [mainJava.javac, '-g', '-J-Xmx1g', '-source', compliance, '-target', compliance, '-classpath', cp, '-d', outputDir, '-bootclasspath', jdk.bootclasspath(), '-endorseddirs', jdk.endorseddirs(), '-extdirs', jdk.extdirs()] 1862 javacCmd = [mainJava.javac, '-g', '-J-Xmx1g', '-source', compliance, '-target', compliance, '-classpath', cp, '-d', outputDir, '-bootclasspath', jdk.bootclasspath(), '-endorseddirs', jdk.endorseddirs(), '-extdirs', jdk.extdirs()]
1861 if jdk.debug_port is not None: 1863 if jdk.debug_port is not None:
1876 javacArgs.append('-XDignore.symbol.file') 1878 javacArgs.append('-XDignore.symbol.file')
1877 run_java(javaArgs + ['-cp', os.pathsep.join([mainJava.toolsjar, args.error_prone]), 'com.google.errorprone.ErrorProneCompiler'] + javacArgs) 1879 run_java(javaArgs + ['-cp', os.pathsep.join([mainJava.toolsjar, args.error_prone]), 'com.google.errorprone.ErrorProneCompiler'] + javacArgs)
1878 else: 1880 else:
1879 self.logCompilation('JDT') 1881 self.logCompilation('JDT')
1880 1882
1881 jdtVmArgs = ['-Xmx1g', '-jar', jdtJar] 1883 jdtVmArgs = ['-Xmx1g', '-jar', self.jdtJar]
1882 1884
1883 jdtArgs = ['-' + compliance, 1885 jdtArgs = ['-' + compliance,
1884 '-cp', cp, '-g', '-enableJavadoc', 1886 '-cp', cp, '-g', '-enableJavadoc',
1885 '-d', outputDir, 1887 '-d', outputDir,
1886 '-bootclasspath', jdk.bootclasspath(), 1888 '-bootclasspath', jdk.bootclasspath(),
1944 compilerSelect = parser.add_mutually_exclusive_group() 1946 compilerSelect = parser.add_mutually_exclusive_group()
1945 compilerSelect.add_argument('--error-prone', dest='error_prone', help='path to error-prone.jar', metavar='<path>') 1947 compilerSelect.add_argument('--error-prone', dest='error_prone', help='path to error-prone.jar', metavar='<path>')
1946 compilerSelect.add_argument('--jdt', help='path to ecj.jar, the Eclipse batch compiler', default=_defaultEcjPath(), metavar='<path>') 1948 compilerSelect.add_argument('--jdt', help='path to ecj.jar, the Eclipse batch compiler', default=_defaultEcjPath(), metavar='<path>')
1947 compilerSelect.add_argument('--force-javac', action='store_true', dest='javac', help='use javac despite ecj.jar is found or not') 1949 compilerSelect.add_argument('--force-javac', action='store_true', dest='javac', help='use javac despite ecj.jar is found or not')
1948 1950
1949
1950 if suppliedParser: 1951 if suppliedParser:
1951 parser.add_argument('remainder', nargs=REMAINDER, metavar='...') 1952 parser.add_argument('remainder', nargs=REMAINDER, metavar='...')
1952 1953
1953 args = parser.parse_args(args) 1954 args = parser.parse_args(args)
1955
1956 jdtJar = None
1957 if not args.javac and args.jdt is not None:
1958 if not args.jdt.endswith('.jar'):
1959 abort('Path for Eclipse batch compiler does not look like a jar file: ' + args.jdt)
1960 jdtJar = args.jdt
1961 if not exists(jdtJar):
1962 if os.path.abspath(jdtJar) == os.path.abspath(_defaultEcjPath()) and get_env('JDT', None) is None:
1963 # Silently ignore JDT if default location is used but does not exist
1964 jdtJar = None
1965 else:
1966 abort('Eclipse batch compiler jar does not exist: ' + args.jdt)
1954 1967
1955 if args.only is not None: 1968 if args.only is not None:
1956 # N.B. This build will not include dependencies including annotation processor dependencies 1969 # N.B. This build will not include dependencies including annotation processor dependencies
1957 sortedProjects = [project(name) for name in args.only.split(',')] 1970 sortedProjects = [project(name) for name in args.only.split(',')]
1958 else: 1971 else:
2085 2098
2086 if len(javafilelist) == 0: 2099 if len(javafilelist) == 0:
2087 logv('[no Java sources for {0} - skipping]'.format(p.name)) 2100 logv('[no Java sources for {0} - skipping]'.format(p.name))
2088 continue 2101 continue
2089 2102
2090 task = JavaCompileTask(args, p, buildReason, javafilelist, jdk, outputDir, taskDeps) 2103 task = JavaCompileTask(args, p, buildReason, javafilelist, jdk, outputDir, jdtJar, taskDeps)
2091 2104
2092 if args.parallelize: 2105 if args.parallelize:
2093 # Best to initialize class paths on main process 2106 # Best to initialize class paths on main process
2094 jdk.bootclasspath() 2107 jdk.bootclasspath()
2095 task.proc = None 2108 task.proc = None
2111 active = [] 2124 active = []
2112 for t in tasks: 2125 for t in tasks:
2113 if t.proc.is_alive(): 2126 if t.proc.is_alive():
2114 active.append(t) 2127 active.append(t)
2115 else: 2128 else:
2129 _removeSubprocess(t.sub)
2116 if t.proc.exitcode != 0: 2130 if t.proc.exitcode != 0:
2117 return ([], joinTasks(tasks)) 2131 return ([], joinTasks(tasks))
2118 return (active, []) 2132 return (active, [])
2119 2133
2120 def remainingDepsDepth(task): 2134 def remainingDepsDepth(task):
2124 task._d = 0 2138 task._d = 0
2125 else: 2139 else:
2126 task._d = max([remainingDepsDepth(t) for t in incompleteDeps]) + 1 2140 task._d = max([remainingDepsDepth(t) for t in incompleteDeps]) + 1
2127 return task._d 2141 return task._d
2128 2142
2143 def compareTasks(t1, t2):
2144 d = remainingDepsDepth(t1) - remainingDepsDepth(t2)
2145 if d == 0:
2146 t1Work = (1 + len(t1.proj.annotation_processors())) * len(t1.javafilelist)
2147 t2Work = (1 + len(t2.proj.annotation_processors())) * len(t2.javafilelist)
2148 d = t1Work - t2Work
2149 return d
2150
2129 def sortWorklist(tasks): 2151 def sortWorklist(tasks):
2130 for t in tasks: 2152 for t in tasks:
2131 t._d = None 2153 t._d = None
2132 return sorted(tasks, lambda x, y: remainingDepsDepth(x) - remainingDepsDepth(y)) 2154 return sorted(tasks, compareTasks)
2133 2155
2134 import multiprocessing 2156 import multiprocessing
2135 cpus = multiprocessing.cpu_count() 2157 cpus = multiprocessing.cpu_count()
2136 worklist = sortWorklist(tasks.values()) 2158 worklist = sortWorklist(tasks.values())
2137 active = [] 2159 active = []
2160 if depsDone(task): 2182 if depsDone(task):
2161 worklist.remove(task) 2183 worklist.remove(task)
2162 task.proc = multiprocessing.Process(target=executeTask, args=(task,)) 2184 task.proc = multiprocessing.Process(target=executeTask, args=(task,))
2163 task.proc.start() 2185 task.proc.start()
2164 active.append(task) 2186 active.append(task)
2187 task.sub = _addSubprocess(task.proc, ['JavaCompileTask', str(task)])
2165 if len(active) == cpus: 2188 if len(active) == cpus:
2166 break 2189 break
2167 2190
2168 worklist = sortWorklist(worklist) 2191 worklist = sortWorklist(worklist)
2169 joinTasks(active) 2192 joinTasks(active)