# HG changeset patch # User Doug Simon # Date 1398981293 -7200 # Node ID 7d24ff89dc7dd9105a5690207e862aa79b6f7ab5 # Parent 0dae565d92895cdaceb4f66595e18bf3e3270b67 mx: parallelized Java builds (GRAAL-350) diff -r 0dae565d9289 -r 7d24ff89dc7d mxtool/mx.py --- a/mxtool/mx.py Thu May 01 18:26:25 2014 +0200 +++ b/mxtool/mx.py Thu May 01 23:54:53 2014 +0200 @@ -1553,9 +1553,13 @@ def _init_classpaths(self): myDir = dirname(__file__) + outDir = join(dirname(__file__), '.jdk' + str(self.version)) + if not exists(outDir): + os.makedirs(outDir) javaSource = join(myDir, 'ClasspathDump.java') - subprocess.check_call([self.javac, '-d', myDir, javaSource]) - self._bootclasspath, self._extdirs, self._endorseddirs = [x if x != 'null' else None for x in subprocess.check_output([self.java, '-cp', myDir, 'ClasspathDump']).split('|')] + if not exists(join(outDir, 'ClasspathDump.class')): + subprocess.check_call([self.javac, '-d', outDir, javaSource]) + self._bootclasspath, self._extdirs, self._endorseddirs = [x if x != 'null' else None for x in subprocess.check_output([self.java, '-cp', outDir, 'ClasspathDump']).split('|')] if not self._bootclasspath or not self._extdirs or not self._endorseddirs: warn("Could not find all classpaths: boot='" + str(self._bootclasspath) + "' extdirs='" + str(self._extdirs) + "' endorseddirs='" + str(self._endorseddirs) + "'") self._bootclasspath = _filter_non_existant_paths(self._bootclasspath) @@ -1790,6 +1794,131 @@ # Builtin commands +def _defaultEcjPath(): + return get_env('JDT', join(_primary_suite.mxDir, 'ecj.jar')) + +class JavaCompileTask: + def __init__(self, args, proj, reason, javafilelist, jdk, outputDir, deps): + self.proj = proj + self.reason = reason + self.javafilelist = javafilelist + self.deps = deps + self.jdk = jdk + self.outputDir = outputDir + self.done = False + self.args = args + + def __str__(self): + return self.proj.name + + def logCompilation(self, compiler): + log('Compiling Java sources for {} with {}... [{}]'.format(self.proj.name, compiler, self.reason)) + + def execute(self): + argfileName = join(self.proj.dir, 'javafilelist.txt') + argfile = open(argfileName, 'wb') + argfile.write('\n'.join(self.javafilelist)) + argfile.close() + + processorArgs = [] + + aps = self.proj.annotation_processors() + if len(aps) > 0: + processorPath = classpath(aps, resolve=True) + genDir = self.proj.source_gen_dir() + if exists(genDir): + shutil.rmtree(genDir) + os.mkdir(genDir) + processorArgs += ['-processorpath', join(processorPath), '-s', genDir] + else: + processorArgs += ['-proc:none'] + + args = self.args + jdk = self.jdk + outputDir = self.outputDir + compliance = str(jdk.javaCompliance) + cp = classpath(self.proj.name, includeSelf=True) + toBeDeleted = [argfileName] + + jdtJar = None + if not args.javac and args.jdt is not None: + if not args.jdt.endswith('.jar'): + abort('Path for Eclipse batch compiler does not look like a jar file: ' + args.jdt) + jdtJar = args.jdt + if not exists(jdtJar): + if os.path.abspath(jdtJar) == os.path.abspath(_defaultEcjPath()) and get_env('JDT', None) is None: + # Silently ignore JDT if default location is used but does not exist + jdtJar = None + else: + abort('Eclipse batch compiler jar does not exist: ' + args.jdt) + + try: + if not jdtJar: + mainJava = java() + if not args.error_prone: + self.logCompilation('javac') + javacCmd = [mainJava.javac, '-g', '-J-Xmx1g', '-source', compliance, '-target', compliance, '-classpath', cp, '-d', outputDir, '-bootclasspath', jdk.bootclasspath(), '-endorseddirs', jdk.endorseddirs(), '-extdirs', jdk.extdirs()] + if jdk.debug_port is not None: + javacCmd += ['-J-Xdebug', '-J-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=' + str(jdk.debug_port)] + javacCmd += processorArgs + javacCmd += ['@' + argfile.name] + + if not args.warnAPI: + javacCmd.append('-XDignore.symbol.file') + run(javacCmd) + else: + self.logCompilation('javac (with error-prone)') + javaArgs = ['-Xmx1g'] + javacArgs = ['-g', '-source', compliance, '-target', compliance, '-classpath', cp, '-d', outputDir, '-bootclasspath', jdk.bootclasspath(), '-endorseddirs', jdk.endorseddirs(), '-extdirs', jdk.extdirs()] + javacArgs += processorArgs + javacArgs += ['@' + argfile.name] + if not args.warnAPI: + javacArgs.append('-XDignore.symbol.file') + run_java(javaArgs + ['-cp', os.pathsep.join([mainJava.toolsjar, args.error_prone]), 'com.google.errorprone.ErrorProneCompiler'] + javacArgs) + else: + self.logCompilation('JDT') + + jdtVmArgs = ['-Xmx1g', '-jar', jdtJar] + + jdtArgs = ['-' + compliance, + '-cp', cp, '-g', '-enableJavadoc', + '-d', outputDir, + '-bootclasspath', jdk.bootclasspath(), + '-endorseddirs', jdk.endorseddirs(), + '-extdirs', jdk.extdirs()] + jdtArgs += processorArgs + + jdtProperties = join(self.proj.dir, '.settings', 'org.eclipse.jdt.core.prefs') + rootJdtProperties = join(self.proj.suite.mxDir, 'eclipse-settings', 'org.eclipse.jdt.core.prefs') + if not exists(jdtProperties) or os.path.getmtime(jdtProperties) < os.path.getmtime(rootJdtProperties): + # Try to fix a missing properties file by running eclipseinit + eclipseinit([], buildProcessorJars=False) + if not exists(jdtProperties): + log('JDT properties file {0} not found'.format(jdtProperties)) + else: + with open(jdtProperties) as fp: + origContent = fp.read() + content = origContent + if args.jdt_warning_as_error: + content = content.replace('=warning', '=error') + if not args.jdt_show_task_tags: + content = content + '\norg.eclipse.jdt.core.compiler.problem.tasks=ignore' + if origContent != content: + jdtPropertiesTmp = jdtProperties + '.tmp' + with open(jdtPropertiesTmp, 'w') as fp: + fp.write(content) + toBeDeleted.append(jdtPropertiesTmp) + jdtArgs += ['-properties', jdtPropertiesTmp] + else: + jdtArgs += ['-properties', jdtProperties] + jdtArgs.append('@' + argfile.name) + + run_java(jdtVmArgs + jdtArgs) + finally: + for n in toBeDeleted: + os.remove(n) + self.done = True + def build(args, parser=None): """compile the Java and C sources, linking the latter @@ -1800,11 +1929,10 @@ if not suppliedParser: parser = ArgumentParser(prog='mx build') - defaultEcjPath = get_env('JDT', join(_primary_suite.mxDir, 'ecj.jar')) - parser = parser if parser is not None else ArgumentParser(prog='mx build') parser.add_argument('-f', action='store_true', dest='force', help='force build (disables timestamp checking)') parser.add_argument('-c', action='store_true', dest='clean', help='removes existing build output') + parser.add_argument('-p', action='store_true', dest='parallelize', help='parallelizes Java compilation') parser.add_argument('--source', dest='compliance', help='Java compliance level for projects without an explicit one') parser.add_argument('--Wapi', action='store_true', dest='warnAPI', help='show warnings about using internal APIs') parser.add_argument('--projects', action='store', help='comma separated projects to build (omit to build all projects)') @@ -1815,7 +1943,7 @@ parser.add_argument('--jdt-show-task-tags', action='store_true', help='show task tags as Eclipse batch compiler warnings') compilerSelect = parser.add_mutually_exclusive_group() compilerSelect.add_argument('--error-prone', dest='error_prone', help='path to error-prone.jar', metavar='') - compilerSelect.add_argument('--jdt', help='path to ecj.jar, the Eclipse batch compiler (default: ' + defaultEcjPath + ')', default=defaultEcjPath, metavar='') + compilerSelect.add_argument('--jdt', help='path to ecj.jar, the Eclipse batch compiler', default=_defaultEcjPath(), metavar='') compilerSelect.add_argument('--force-javac', action='store_true', dest='javac', help='use javac despite ecj.jar is found or not') @@ -1824,20 +1952,6 @@ args = parser.parse_args(args) - jdtJar = None - if not args.javac and args.jdt is not None: - if not args.jdt.endswith('.jar'): - abort('Path for Eclipse batch compiler does not look like a jar file: ' + args.jdt) - jdtJar = args.jdt - if not exists(jdtJar): - if os.path.abspath(jdtJar) == os.path.abspath(defaultEcjPath) and get_env('JDT', None) is None: - # Silently ignore JDT if default location is used but does not exist - jdtJar = None - else: - abort('Eclipse batch compiler jar does not exist: ' + args.jdt) - - built = set() - if args.only is not None: # N.B. This build will not include dependencies including annotation processor dependencies sortedProjects = [project(name) for name in args.only.split(',')] @@ -1870,6 +1984,7 @@ shutil.rmtree(join(genDir, f)) return outputDir + tasks = {} for p in sortedProjects: if p.native: if args.native: @@ -1879,7 +1994,6 @@ run([gmake_cmd(), 'clean'], cwd=p.dir) run([gmake_cmd()], cwd=p.dir) - built.add(p.name) continue else: if not args.java: @@ -1893,17 +2007,19 @@ if not jdk: log('Excluding {0} from build (Java compliance level {1} required)'.format(p.name, requiredCompliance)) continue - compliance = str(jdk.javaCompliance) outputDir = prepareOutputDirs(p, args.clean) - cp = classpath(p.name, includeSelf=True) sourceDirs = p.source_dirs() buildReason = 'forced build' if args.force else None + taskDeps = [] if not buildReason: - for dep in p.all_deps([], False): - if dep.name in built: - buildReason = dep.name + ' rebuilt' + for dep in p.all_deps([], includeLibs=False, includeAnnotationProcessors=True): + taskDep = tasks.get(dep.name) + if taskDep: + if not buildReason: + buildReason = dep.name + ' rebuilt' + taskDeps.append(taskDep) jasminAvailable = None javafilelist = [] @@ -1959,7 +2075,6 @@ buildReason = 'class file(s) out of date' break - aps = p.annotation_processors() apsOutOfDate = p.update_current_annotation_processors_file() if apsOutOfDate: buildReason = 'annotation processor(s) changed' @@ -1972,99 +2087,86 @@ logv('[no Java sources for {0} - skipping]'.format(p.name)) continue - # Ensure that the output directories are clean - # prepareOutputDirs(p, True) - - built.add(p.name) - - argfileName = join(p.dir, 'javafilelist.txt') - argfile = open(argfileName, 'wb') - argfile.write('\n'.join(javafilelist)) - argfile.close() - - processorArgs = [] - - if len(aps) > 0: - processorPath = classpath(aps, resolve=True) - genDir = p.source_gen_dir() - if exists(genDir): - shutil.rmtree(genDir) - os.mkdir(genDir) - processorArgs += ['-processorpath', join(processorPath), '-s', genDir] + task = JavaCompileTask(args, p, buildReason, javafilelist, jdk, outputDir, taskDeps) + + if args.parallelize: + # Best to initialize class paths on main process + jdk.bootclasspath() + task.proc = None + tasks[p.name] = task else: - processorArgs += ['-proc:none'] - - toBeDeleted = [argfileName] - try: - - def logCompilation(p, compiler, reason): - log('Compiling Java sources for {} with {}... [{}]'.format(p.name, compiler, reason)) - - if not jdtJar: - mainJava = java() - if not args.error_prone: - logCompilation(p, 'javac', buildReason) - javacCmd = [mainJava.javac, '-g', '-J-Xmx1g', '-source', compliance, '-target', compliance, '-classpath', cp, '-d', outputDir, '-bootclasspath', jdk.bootclasspath(), '-endorseddirs', jdk.endorseddirs(), '-extdirs', jdk.extdirs()] - if jdk.debug_port is not None: - javacCmd += ['-J-Xdebug', '-J-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=' + str(jdk.debug_port)] - javacCmd += processorArgs - javacCmd += ['@' + argfile.name] - - if not args.warnAPI: - javacCmd.append('-XDignore.symbol.file') - run(javacCmd) + task.execute() + + if args.parallelize: + + def joinTasks(tasks): + failed = [] + for t in tasks: + t.proc.join() + if t.proc.exitcode != 0: + failed.append(t) + return failed + + def checkTasks(tasks): + active = [] + for t in tasks: + if t.proc.is_alive(): + active.append(t) + else: + if t.proc.exitcode != 0: + return ([], joinTasks(tasks)) + return (active, []) + + def remainingDepsDepth(task): + if task._d is None: + incompleteDeps = [d for d in task.deps if d.proc is None or d.proc.is_alive()] + if len(incompleteDeps) == 0: + task._d = 0 else: - logCompilation(p, 'javac (with error-prone)', buildReason) - javaArgs = ['-Xmx1g'] - javacArgs = ['-g', '-source', compliance, '-target', compliance, '-classpath', cp, '-d', outputDir, '-bootclasspath', jdk.bootclasspath(), '-endorseddirs', jdk.endorseddirs(), '-extdirs', jdk.extdirs()] - javacArgs += processorArgs - javacArgs += ['@' + argfile.name] - if not args.warnAPI: - javacArgs.append('-XDignore.symbol.file') - run_java(javaArgs + ['-cp', os.pathsep.join([mainJava.toolsjar, args.error_prone]), 'com.google.errorprone.ErrorProneCompiler'] + javacArgs) - else: - logCompilation(p, 'JDT', buildReason) - - jdtVmArgs = ['-Xmx1g', '-jar', jdtJar] - - jdtArgs = ['-' + compliance, - '-cp', cp, '-g', '-enableJavadoc', - '-d', outputDir, - '-bootclasspath', jdk.bootclasspath(), - '-endorseddirs', jdk.endorseddirs(), - '-extdirs', jdk.extdirs()] - jdtArgs += processorArgs - - - jdtProperties = join(p.dir, '.settings', 'org.eclipse.jdt.core.prefs') - rootJdtProperties = join(p.suite.mxDir, 'eclipse-settings', 'org.eclipse.jdt.core.prefs') - if not exists(jdtProperties) or os.path.getmtime(jdtProperties) < os.path.getmtime(rootJdtProperties): - # Try to fix a missing properties file by running eclipseinit - eclipseinit([], buildProcessorJars=False) - if not exists(jdtProperties): - log('JDT properties file {0} not found'.format(jdtProperties)) + task._d = max([remainingDepsDepth(t) for t in incompleteDeps]) + 1 + return task._d + + def sortWorklist(tasks): + for t in tasks: + t._d = None + return sorted(tasks, lambda x, y : remainingDepsDepth(x) - remainingDepsDepth(y)) + + import multiprocessing + cpus = multiprocessing.cpu_count() + worklist = sortWorklist(tasks.values()) + active = [] + while len(worklist) != 0: + while True: + active, failed = checkTasks(active) + if len(failed) != 0: + assert not active, active + break + if len(active) == cpus: + # Sleep for 1 second + time.sleep(1) else: - with open(jdtProperties) as fp: - origContent = fp.read() - content = origContent - if args.jdt_warning_as_error: - content = content.replace('=warning', '=error') - if not args.jdt_show_task_tags: - content = content + '\norg.eclipse.jdt.core.compiler.problem.tasks=ignore' - if origContent != content: - jdtPropertiesTmp = jdtProperties + '.tmp' - with open(jdtPropertiesTmp, 'w') as fp: - fp.write(content) - toBeDeleted.append(jdtPropertiesTmp) - jdtArgs += ['-properties', jdtPropertiesTmp] - else: - jdtArgs += ['-properties', jdtProperties] - jdtArgs.append('@' + argfile.name) - - run_java(jdtVmArgs + jdtArgs) - finally: - for n in toBeDeleted: - os.remove(n) + break + + def executeTask(task): + task.execute() + + def depsDone(task): + for d in task.deps: + if d.proc is None or d.proc.exitcode is None: + return False + return True + + for task in worklist: + if depsDone(task): + worklist.remove(task) + task.proc = multiprocessing.Process(target=executeTask, args=(task,)) + task.proc.start() + active.append(task) + if len(active) == cpus: + break + + worklist = sortWorklist(worklist) + joinTasks(active) for dist in _dists.values(): archive(['@' + dist.name])