Mercurial > hg > truffle

comparison mxtool/mx.py @ 15489:d0e3f6963ed7

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
mx: made parallel Java builds interact correctly with management of subprocesses upon abort/quit
author Doug Simon <doug.simon@oracle.com>
date Sun, 04 May 2014 01:26:50 +0200
parents 05d3f069cff2
children 5d0dd6a6f6b3
comparison
equal deleted inserted replaced
15488:d370d87e528f 15489:d0e3f6963ed7
1298 abort('Process timed out after {0} seconds: {1}'.format(timeout, ' '.join(args))) 1298 abort('Process timed out after {0} seconds: {1}'.format(timeout, ' '.join(args)))
1299 delay = min(delay * 2, remaining, .05) 1299 delay = min(delay * 2, remaining, .05)
1300 time.sleep(delay) 1300 time.sleep(delay)
1301 1301
1302 # Makes the current subprocess accessible to the abort() function 1302 # Makes the current subprocess accessible to the abort() function
1303 # This is a tuple of the Popen object and args. 1303 # This is a list of tuples of the subprocess.Popen or
1304 _currentSubprocess = (None, None) 1304 # multiprocessing.Process object and args.
1305 _currentSubprocesses = []
1306
1307 def _addSubprocess(p, args):
1308 entry = (p, args)
1309 _currentSubprocesses.append(entry)
1310 return entry
1311
1312 def _removeSubprocess(entry):
1313 if entry and entry in _currentSubprocesses:
1314 try:
1315 _currentSubprocesses.remove(entry)
1316 except:
1317 pass
1305 1318
1306 def waitOn(p): 1319 def waitOn(p):
1307 if get_os() == 'windows': 1320 if get_os() == 'windows':
1308 # on windows use a poll loop, otherwise signal does not get handled 1321 # on windows use a poll loop, otherwise signal does not get handled
1309 retcode = None 1322 retcode = None
1338 log(' '.join(map(pipes.quote, args))) 1351 log(' '.join(map(pipes.quote, args)))
1339 1352
1340 if timeout is None and _opts.ptimeout != 0: 1353 if timeout is None and _opts.ptimeout != 0:
1341 timeout = _opts.ptimeout 1354 timeout = _opts.ptimeout
1342 1355
1343 global _currentSubprocess 1356 sub = None
1344
1345 try: 1357 try:
1346 # On Unix, the new subprocess should be in a separate group so that a timeout alarm 1358 # On Unix, the new subprocess should be in a separate group so that a timeout alarm
1347 # can use os.killpg() to kill the whole subprocess group 1359 # can use os.killpg() to kill the whole subprocess group
1348 preexec_fn = None 1360 preexec_fn = None
1349 creationflags = 0 1361 creationflags = 0
1357 f(line) 1369 f(line)
1358 stream.close() 1370 stream.close()
1359 stdout = out if not callable(out) else subprocess.PIPE 1371 stdout = out if not callable(out) else subprocess.PIPE
1360 stderr = err if not callable(err) else subprocess.PIPE 1372 stderr = err if not callable(err) else subprocess.PIPE
1361 p = subprocess.Popen(args, cwd=cwd, stdout=stdout, stderr=stderr, preexec_fn=preexec_fn, creationflags=creationflags, env=env) 1373 p = subprocess.Popen(args, cwd=cwd, stdout=stdout, stderr=stderr, preexec_fn=preexec_fn, creationflags=creationflags, env=env)
1362 _currentSubprocess = (p, args) 1374 sub = _addSubprocess(p, args)
1363 if callable(out): 1375 if callable(out):
1364 t = Thread(target=redirect, args=(p.stdout, out)) 1376 t = Thread(target=redirect, args=(p.stdout, out))
1365 t.daemon = True # thread dies with the program 1377 t.daemon = True # thread dies with the program
1366 t.start() 1378 t.start()
1367 if callable(err): 1379 if callable(err):
1380 raise e 1392 raise e
1381 abort(e.errno) 1393 abort(e.errno)
1382 except KeyboardInterrupt: 1394 except KeyboardInterrupt:
1383 abort(1) 1395 abort(1)
1384 finally: 1396 finally:
1385 _currentSubprocess = (None, None) 1397 _removeSubprocess(sub)
1386 1398
1387 if retcode and nonZeroIsFatal: 1399 if retcode and nonZeroIsFatal:
1388 if _opts.verbose: 1400 if _opts.verbose:
1389 if _opts.very_verbose: 1401 if _opts.very_verbose:
1390 raise subprocess.CalledProcessError(retcode, ' '.join(args)) 1402 raise subprocess.CalledProcessError(retcode, ' '.join(args))
1661 if '$' in result or '%' in result: 1673 if '$' in result or '%' in result:
1662 abort('Property contains an undefined environment variable: ' + value) 1674 abort('Property contains an undefined environment variable: ' + value)
1663 return result 1675 return result
1664 1676
1665 def _send_sigquit(): 1677 def _send_sigquit():
1666 p, args = _currentSubprocess 1678 for p, args in _currentSubprocesses:
1667 1679
1668 def _isJava(): 1680 def _isJava():
1669 if args: 1681 if args:
1670 name = args[0].split(os.sep)[-1] 1682 name = args[0].split(os.sep)[-1]
1671 return name == "java" 1683 return name == "java"
1672 return False 1684 return False
1673 1685
1674 if p is not None and _isJava(): 1686 if p is not None and _isJava():
1675 if get_os() == 'windows': 1687 if get_os() == 'windows':
1676 log("mx: implement me! want to send SIGQUIT to my child process") 1688 log("mx: implement me! want to send SIGQUIT to my child process")
1677 else: 1689 else:
1678 _kill_process_group(p.pid, sig=signal.SIGQUIT) 1690 _kill_process_group(p.pid, sig=signal.SIGQUIT)
1679 time.sleep(0.1) 1691 time.sleep(0.1)
1680
1681 1692
1682 def abort(codeOrMessage): 1693 def abort(codeOrMessage):
1683 """ 1694 """
1684 Aborts the program with a SystemExit exception. 1695 Aborts the program with a SystemExit exception.
1685 If 'codeOrMessage' is a plain integer, it specifies the system exit status; 1696 If 'codeOrMessage' is a plain integer, it specifies the system exit status;
1690 if _opts.killwithsigquit: 1701 if _opts.killwithsigquit:
1691 _send_sigquit() 1702 _send_sigquit()
1692 1703
1693 # import traceback 1704 # import traceback
1694 # traceback.print_stack() 1705 # traceback.print_stack()
1695 p, _ = _currentSubprocess 1706 for p, args in _currentSubprocesses:
1696 if p is not None: 1707 try:
1697 if get_os() == 'windows': 1708 if get_os() == 'windows':
1698 p.kill() 1709 p.terminate()
1699 else: 1710 else:
1700 _kill_process_group(p.pid, signal.SIGKILL) 1711 _kill_process_group(p.pid, signal.SIGKILL)
1712 except BaseException as e:
1713 log('error while killing subprocess {} "{}": {}'.format(p.pid, ' '.join(args), e))
1701 1714
1702 raise SystemExit(codeOrMessage) 1715 raise SystemExit(codeOrMessage)
1703 1716
1704 def download(path, urls, verbose=False): 1717 def download(path, urls, verbose=False):
1705 """ 1718 """
2111 active = [] 2124 active = []
2112 for t in tasks: 2125 for t in tasks:
2113 if t.proc.is_alive(): 2126 if t.proc.is_alive():
2114 active.append(t) 2127 active.append(t)
2115 else: 2128 else:
2129 _removeSubprocess(t.sub)
2116 if t.proc.exitcode != 0: 2130 if t.proc.exitcode != 0:
2117 return ([], joinTasks(tasks)) 2131 return ([], joinTasks(tasks))
2118 return (active, []) 2132 return (active, [])
2119 2133
2120 def remainingDepsDepth(task): 2134 def remainingDepsDepth(task):
2160 if depsDone(task): 2174 if depsDone(task):
2161 worklist.remove(task) 2175 worklist.remove(task)
2162 task.proc = multiprocessing.Process(target=executeTask, args=(task,)) 2176 task.proc = multiprocessing.Process(target=executeTask, args=(task,))
2163 task.proc.start() 2177 task.proc.start()
2164 active.append(task) 2178 active.append(task)
2179 task.sub = _addSubprocess(task.proc, ['JavaCompileTask', str(task)])
2165 if len(active) == cpus: 2180 if len(active) == cpus:
2166 break 2181 break
2167 2182
2168 worklist = sortWorklist(worklist) 2183 worklist = sortWorklist(worklist)
2169 joinTasks(active) 2184 joinTasks(active)