Mercurial > hg > graal-jvmci-8
comparison mxtool/mx.py @ 15460:7d24ff89dc7d
mx: parallelized Java builds (GRAAL-350)
author | Doug Simon <doug.simon@oracle.com> |
---|---|
date | Thu, 01 May 2014 23:54:53 +0200 |
parents | c0e1a8693e0e |
children | 05d3f069cff2 |
comparison
equal
deleted
inserted
replaced
15459:0dae565d9289 | 15460:7d24ff89dc7d |
---|---|
1551 if self.debug_port is not None: | 1551 if self.debug_port is not None: |
1552 self.java_args += ['-Xdebug', '-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=' + str(self.debug_port)] | 1552 self.java_args += ['-Xdebug', '-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=' + str(self.debug_port)] |
1553 | 1553 |
1554 def _init_classpaths(self): | 1554 def _init_classpaths(self): |
1555 myDir = dirname(__file__) | 1555 myDir = dirname(__file__) |
1556 outDir = join(dirname(__file__), '.jdk' + str(self.version)) | |
1557 if not exists(outDir): | |
1558 os.makedirs(outDir) | |
1556 javaSource = join(myDir, 'ClasspathDump.java') | 1559 javaSource = join(myDir, 'ClasspathDump.java') |
1557 subprocess.check_call([self.javac, '-d', myDir, javaSource]) | 1560 if not exists(join(outDir, 'ClasspathDump.class')): |
1558 self._bootclasspath, self._extdirs, self._endorseddirs = [x if x != 'null' else None for x in subprocess.check_output([self.java, '-cp', myDir, 'ClasspathDump']).split('|')] | 1561 subprocess.check_call([self.javac, '-d', outDir, javaSource]) |
1562 self._bootclasspath, self._extdirs, self._endorseddirs = [x if x != 'null' else None for x in subprocess.check_output([self.java, '-cp', outDir, 'ClasspathDump']).split('|')] | |
1559 if not self._bootclasspath or not self._extdirs or not self._endorseddirs: | 1563 if not self._bootclasspath or not self._extdirs or not self._endorseddirs: |
1560 warn("Could not find all classpaths: boot='" + str(self._bootclasspath) + "' extdirs='" + str(self._extdirs) + "' endorseddirs='" + str(self._endorseddirs) + "'") | 1564 warn("Could not find all classpaths: boot='" + str(self._bootclasspath) + "' extdirs='" + str(self._extdirs) + "' endorseddirs='" + str(self._endorseddirs) + "'") |
1561 self._bootclasspath = _filter_non_existant_paths(self._bootclasspath) | 1565 self._bootclasspath = _filter_non_existant_paths(self._bootclasspath) |
1562 self._extdirs = _filter_non_existant_paths(self._extdirs) | 1566 self._extdirs = _filter_non_existant_paths(self._extdirs) |
1563 self._endorseddirs = _filter_non_existant_paths(self._endorseddirs) | 1567 self._endorseddirs = _filter_non_existant_paths(self._endorseddirs) |
1788 except IOError as e: | 1792 except IOError as e: |
1789 abort('Error while writing to ' + path + ': ' + str(e)) | 1793 abort('Error while writing to ' + path + ': ' + str(e)) |
1790 | 1794 |
1791 # Builtin commands | 1795 # Builtin commands |
1792 | 1796 |
1793 def build(args, parser=None): | 1797 def _defaultEcjPath(): |
1794 """compile the Java and C sources, linking the latter | 1798 return get_env('JDT', join(_primary_suite.mxDir, 'ecj.jar')) |
1795 | 1799 |
1796 Compile all the Java source code using the appropriate compilers | 1800 class JavaCompileTask: |
1797 and linkers for the various source code types.""" | 1801 def __init__(self, args, proj, reason, javafilelist, jdk, outputDir, deps): |
1798 | 1802 self.proj = proj |
1799 suppliedParser = parser is not None | 1803 self.reason = reason |
1800 if not suppliedParser: | 1804 self.javafilelist = javafilelist |
1801 parser = ArgumentParser(prog='mx build') | 1805 self.deps = deps |
1802 | 1806 self.jdk = jdk |
1803 defaultEcjPath = get_env('JDT', join(_primary_suite.mxDir, 'ecj.jar')) | 1807 self.outputDir = outputDir |
1804 | 1808 self.done = False |
1805 parser = parser if parser is not None else ArgumentParser(prog='mx build') | 1809 self.args = args |
1806 parser.add_argument('-f', action='store_true', dest='force', help='force build (disables timestamp checking)') | 1810 |
1807 parser.add_argument('-c', action='store_true', dest='clean', help='removes existing build output') | 1811 def __str__(self): |
1808 parser.add_argument('--source', dest='compliance', help='Java compliance level for projects without an explicit one') | 1812 return self.proj.name |
1809 parser.add_argument('--Wapi', action='store_true', dest='warnAPI', help='show warnings about using internal APIs') | 1813 |
1810 parser.add_argument('--projects', action='store', help='comma separated projects to build (omit to build all projects)') | 1814 def logCompilation(self, compiler): |
1811 parser.add_argument('--only', action='store', help='comma separated projects to build, without checking their dependencies (omit to build all projects)') | 1815 log('Compiling Java sources for {} with {}... [{}]'.format(self.proj.name, compiler, self.reason)) |
1812 parser.add_argument('--no-java', action='store_false', dest='java', help='do not build Java projects') | 1816 |
1813 parser.add_argument('--no-native', action='store_false', dest='native', help='do not build native projects') | 1817 def execute(self): |
1814 parser.add_argument('--jdt-warning-as-error', action='store_true', help='convert all Eclipse batch compiler warnings to errors') | 1818 argfileName = join(self.proj.dir, 'javafilelist.txt') |
1815 parser.add_argument('--jdt-show-task-tags', action='store_true', help='show task tags as Eclipse batch compiler warnings') | |
1816 compilerSelect = parser.add_mutually_exclusive_group() | |
1817 compilerSelect.add_argument('--error-prone', dest='error_prone', help='path to error-prone.jar', metavar='<path>') | |
1818 compilerSelect.add_argument('--jdt', help='path to ecj.jar, the Eclipse batch compiler (default: ' + defaultEcjPath + ')', default=defaultEcjPath, metavar='<path>') | |
1819 compilerSelect.add_argument('--force-javac', action='store_true', dest='javac', help='use javac despite ecj.jar is found or not') | |
1820 | |
1821 | |
1822 if suppliedParser: | |
1823 parser.add_argument('remainder', nargs=REMAINDER, metavar='...') | |
1824 | |
1825 args = parser.parse_args(args) | |
1826 | |
1827 jdtJar = None | |
1828 if not args.javac and args.jdt is not None: | |
1829 if not args.jdt.endswith('.jar'): | |
1830 abort('Path for Eclipse batch compiler does not look like a jar file: ' + args.jdt) | |
1831 jdtJar = args.jdt | |
1832 if not exists(jdtJar): | |
1833 if os.path.abspath(jdtJar) == os.path.abspath(defaultEcjPath) and get_env('JDT', None) is None: | |
1834 # Silently ignore JDT if default location is used but does not exist | |
1835 jdtJar = None | |
1836 else: | |
1837 abort('Eclipse batch compiler jar does not exist: ' + args.jdt) | |
1838 | |
1839 built = set() | |
1840 | |
1841 if args.only is not None: | |
1842 # N.B. This build will not include dependencies including annotation processor dependencies | |
1843 sortedProjects = [project(name) for name in args.only.split(',')] | |
1844 else: | |
1845 if args.projects is not None: | |
1846 projectNames = args.projects.split(',') | |
1847 else: | |
1848 projectNames = None | |
1849 | |
1850 projects = _projects_opt_limit_to_suites(projects_from_names(projectNames)) | |
1851 # N.B. Limiting to a suite only affects the starting set of projects. Dependencies in other suites will still be compiled | |
1852 sortedProjects = sorted_project_deps(projects, includeAnnotationProcessors=True) | |
1853 | |
1854 if args.java: | |
1855 ideinit([], refreshOnly=True, buildProcessorJars=False) | |
1856 | |
1857 def prepareOutputDirs(p, clean): | |
1858 outputDir = p.output_dir() | |
1859 if exists(outputDir): | |
1860 if clean: | |
1861 log('Cleaning {0}...'.format(outputDir)) | |
1862 shutil.rmtree(outputDir) | |
1863 os.mkdir(outputDir) | |
1864 else: | |
1865 os.mkdir(outputDir) | |
1866 genDir = p.source_gen_dir() | |
1867 if genDir != '' and exists(genDir) and clean: | |
1868 log('Cleaning {0}...'.format(genDir)) | |
1869 for f in os.listdir(genDir): | |
1870 shutil.rmtree(join(genDir, f)) | |
1871 return outputDir | |
1872 | |
1873 for p in sortedProjects: | |
1874 if p.native: | |
1875 if args.native: | |
1876 log('Calling GNU make {0}...'.format(p.dir)) | |
1877 | |
1878 if args.clean: | |
1879 run([gmake_cmd(), 'clean'], cwd=p.dir) | |
1880 | |
1881 run([gmake_cmd()], cwd=p.dir) | |
1882 built.add(p.name) | |
1883 continue | |
1884 else: | |
1885 if not args.java: | |
1886 continue | |
1887 if exists(join(p.dir, 'plugin.xml')): # eclipse plugin project | |
1888 continue | |
1889 | |
1890 # skip building this Java project if its Java compliance level is "higher" than the configured JDK | |
1891 requiredCompliance = p.javaCompliance if p.javaCompliance else JavaCompliance(args.compliance) if args.compliance else None | |
1892 jdk = java(requiredCompliance) | |
1893 if not jdk: | |
1894 log('Excluding {0} from build (Java compliance level {1} required)'.format(p.name, requiredCompliance)) | |
1895 continue | |
1896 compliance = str(jdk.javaCompliance) | |
1897 | |
1898 outputDir = prepareOutputDirs(p, args.clean) | |
1899 | |
1900 cp = classpath(p.name, includeSelf=True) | |
1901 sourceDirs = p.source_dirs() | |
1902 buildReason = 'forced build' if args.force else None | |
1903 if not buildReason: | |
1904 for dep in p.all_deps([], False): | |
1905 if dep.name in built: | |
1906 buildReason = dep.name + ' rebuilt' | |
1907 | |
1908 jasminAvailable = None | |
1909 javafilelist = [] | |
1910 for sourceDir in sourceDirs: | |
1911 for root, _, files in os.walk(sourceDir): | |
1912 javafiles = [join(root, name) for name in files if name.endswith('.java') and name != 'package-info.java'] | |
1913 javafilelist += javafiles | |
1914 | |
1915 # Copy all non Java resources or assemble Jasmin files | |
1916 nonjavafilelist = [join(root, name) for name in files if not name.endswith('.java')] | |
1917 for src in nonjavafilelist: | |
1918 if src.endswith('.jasm'): | |
1919 className = None | |
1920 with open(src) as f: | |
1921 for line in f: | |
1922 if line.startswith('.class '): | |
1923 className = line.split()[-1] | |
1924 break | |
1925 | |
1926 if className is not None: | |
1927 jasminOutputDir = p.jasmin_output_dir() | |
1928 classFile = join(jasminOutputDir, className.replace('/', os.sep) + '.class') | |
1929 if exists(dirname(classFile)) and (not exists(classFile) or os.path.getmtime(classFile) < os.path.getmtime(src)): | |
1930 if jasminAvailable is None: | |
1931 try: | |
1932 with open(os.devnull) as devnull: | |
1933 subprocess.call('jasmin', stdout=devnull, stderr=subprocess.STDOUT) | |
1934 jasminAvailable = True | |
1935 except OSError: | |
1936 jasminAvailable = False | |
1937 | |
1938 if jasminAvailable: | |
1939 log('Assembling Jasmin file ' + src) | |
1940 run(['jasmin', '-d', jasminOutputDir, src]) | |
1941 else: | |
1942 log('The jasmin executable could not be found - skipping ' + src) | |
1943 with file(classFile, 'a'): | |
1944 os.utime(classFile, None) | |
1945 | |
1946 else: | |
1947 log('could not file .class directive in Jasmin source: ' + src) | |
1948 else: | |
1949 dst = join(outputDir, src[len(sourceDir) + 1:]) | |
1950 if not exists(dirname(dst)): | |
1951 os.makedirs(dirname(dst)) | |
1952 if exists(dirname(dst)) and (not exists(dst) or os.path.getmtime(dst) < os.path.getmtime(src)): | |
1953 shutil.copyfile(src, dst) | |
1954 | |
1955 if not buildReason: | |
1956 for javafile in javafiles: | |
1957 classfile = TimeStampFile(outputDir + javafile[len(sourceDir):-len('java')] + 'class') | |
1958 if not classfile.exists() or classfile.isOlderThan(javafile): | |
1959 buildReason = 'class file(s) out of date' | |
1960 break | |
1961 | |
1962 aps = p.annotation_processors() | |
1963 apsOutOfDate = p.update_current_annotation_processors_file() | |
1964 if apsOutOfDate: | |
1965 buildReason = 'annotation processor(s) changed' | |
1966 | |
1967 if not buildReason: | |
1968 logv('[all class files for {0} are up to date - skipping]'.format(p.name)) | |
1969 continue | |
1970 | |
1971 if len(javafilelist) == 0: | |
1972 logv('[no Java sources for {0} - skipping]'.format(p.name)) | |
1973 continue | |
1974 | |
1975 # Ensure that the output directories are clean | |
1976 # prepareOutputDirs(p, True) | |
1977 | |
1978 built.add(p.name) | |
1979 | |
1980 argfileName = join(p.dir, 'javafilelist.txt') | |
1981 argfile = open(argfileName, 'wb') | 1819 argfile = open(argfileName, 'wb') |
1982 argfile.write('\n'.join(javafilelist)) | 1820 argfile.write('\n'.join(self.javafilelist)) |
1983 argfile.close() | 1821 argfile.close() |
1984 | 1822 |
1985 processorArgs = [] | 1823 processorArgs = [] |
1986 | 1824 |
1825 aps = self.proj.annotation_processors() | |
1987 if len(aps) > 0: | 1826 if len(aps) > 0: |
1988 processorPath = classpath(aps, resolve=True) | 1827 processorPath = classpath(aps, resolve=True) |
1989 genDir = p.source_gen_dir() | 1828 genDir = self.proj.source_gen_dir() |
1990 if exists(genDir): | 1829 if exists(genDir): |
1991 shutil.rmtree(genDir) | 1830 shutil.rmtree(genDir) |
1992 os.mkdir(genDir) | 1831 os.mkdir(genDir) |
1993 processorArgs += ['-processorpath', join(processorPath), '-s', genDir] | 1832 processorArgs += ['-processorpath', join(processorPath), '-s', genDir] |
1994 else: | 1833 else: |
1995 processorArgs += ['-proc:none'] | 1834 processorArgs += ['-proc:none'] |
1996 | 1835 |
1836 args = self.args | |
1837 jdk = self.jdk | |
1838 outputDir = self.outputDir | |
1839 compliance = str(jdk.javaCompliance) | |
1840 cp = classpath(self.proj.name, includeSelf=True) | |
1997 toBeDeleted = [argfileName] | 1841 toBeDeleted = [argfileName] |
1842 | |
1843 jdtJar = None | |
1844 if not args.javac and args.jdt is not None: | |
1845 if not args.jdt.endswith('.jar'): | |
1846 abort('Path for Eclipse batch compiler does not look like a jar file: ' + args.jdt) | |
1847 jdtJar = args.jdt | |
1848 if not exists(jdtJar): | |
1849 if os.path.abspath(jdtJar) == os.path.abspath(_defaultEcjPath()) and get_env('JDT', None) is None: | |
1850 # Silently ignore JDT if default location is used but does not exist | |
1851 jdtJar = None | |
1852 else: | |
1853 abort('Eclipse batch compiler jar does not exist: ' + args.jdt) | |
1854 | |
1998 try: | 1855 try: |
1999 | |
2000 def logCompilation(p, compiler, reason): | |
2001 log('Compiling Java sources for {} with {}... [{}]'.format(p.name, compiler, reason)) | |
2002 | |
2003 if not jdtJar: | 1856 if not jdtJar: |
2004 mainJava = java() | 1857 mainJava = java() |
2005 if not args.error_prone: | 1858 if not args.error_prone: |
2006 logCompilation(p, 'javac', buildReason) | 1859 self.logCompilation('javac') |
2007 javacCmd = [mainJava.javac, '-g', '-J-Xmx1g', '-source', compliance, '-target', compliance, '-classpath', cp, '-d', outputDir, '-bootclasspath', jdk.bootclasspath(), '-endorseddirs', jdk.endorseddirs(), '-extdirs', jdk.extdirs()] | 1860 javacCmd = [mainJava.javac, '-g', '-J-Xmx1g', '-source', compliance, '-target', compliance, '-classpath', cp, '-d', outputDir, '-bootclasspath', jdk.bootclasspath(), '-endorseddirs', jdk.endorseddirs(), '-extdirs', jdk.extdirs()] |
2008 if jdk.debug_port is not None: | 1861 if jdk.debug_port is not None: |
2009 javacCmd += ['-J-Xdebug', '-J-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=' + str(jdk.debug_port)] | 1862 javacCmd += ['-J-Xdebug', '-J-Xrunjdwp:transport=dt_socket,server=y,suspend=y,address=' + str(jdk.debug_port)] |
2010 javacCmd += processorArgs | 1863 javacCmd += processorArgs |
2011 javacCmd += ['@' + argfile.name] | 1864 javacCmd += ['@' + argfile.name] |
2012 | 1865 |
2013 if not args.warnAPI: | 1866 if not args.warnAPI: |
2014 javacCmd.append('-XDignore.symbol.file') | 1867 javacCmd.append('-XDignore.symbol.file') |
2015 run(javacCmd) | 1868 run(javacCmd) |
2016 else: | 1869 else: |
2017 logCompilation(p, 'javac (with error-prone)', buildReason) | 1870 self.logCompilation('javac (with error-prone)') |
2018 javaArgs = ['-Xmx1g'] | 1871 javaArgs = ['-Xmx1g'] |
2019 javacArgs = ['-g', '-source', compliance, '-target', compliance, '-classpath', cp, '-d', outputDir, '-bootclasspath', jdk.bootclasspath(), '-endorseddirs', jdk.endorseddirs(), '-extdirs', jdk.extdirs()] | 1872 javacArgs = ['-g', '-source', compliance, '-target', compliance, '-classpath', cp, '-d', outputDir, '-bootclasspath', jdk.bootclasspath(), '-endorseddirs', jdk.endorseddirs(), '-extdirs', jdk.extdirs()] |
2020 javacArgs += processorArgs | 1873 javacArgs += processorArgs |
2021 javacArgs += ['@' + argfile.name] | 1874 javacArgs += ['@' + argfile.name] |
2022 if not args.warnAPI: | 1875 if not args.warnAPI: |
2023 javacArgs.append('-XDignore.symbol.file') | 1876 javacArgs.append('-XDignore.symbol.file') |
2024 run_java(javaArgs + ['-cp', os.pathsep.join([mainJava.toolsjar, args.error_prone]), 'com.google.errorprone.ErrorProneCompiler'] + javacArgs) | 1877 run_java(javaArgs + ['-cp', os.pathsep.join([mainJava.toolsjar, args.error_prone]), 'com.google.errorprone.ErrorProneCompiler'] + javacArgs) |
2025 else: | 1878 else: |
2026 logCompilation(p, 'JDT', buildReason) | 1879 self.logCompilation('JDT') |
2027 | 1880 |
2028 jdtVmArgs = ['-Xmx1g', '-jar', jdtJar] | 1881 jdtVmArgs = ['-Xmx1g', '-jar', jdtJar] |
2029 | 1882 |
2030 jdtArgs = ['-' + compliance, | 1883 jdtArgs = ['-' + compliance, |
2031 '-cp', cp, '-g', '-enableJavadoc', | 1884 '-cp', cp, '-g', '-enableJavadoc', |
2033 '-bootclasspath', jdk.bootclasspath(), | 1886 '-bootclasspath', jdk.bootclasspath(), |
2034 '-endorseddirs', jdk.endorseddirs(), | 1887 '-endorseddirs', jdk.endorseddirs(), |
2035 '-extdirs', jdk.extdirs()] | 1888 '-extdirs', jdk.extdirs()] |
2036 jdtArgs += processorArgs | 1889 jdtArgs += processorArgs |
2037 | 1890 |
2038 | 1891 jdtProperties = join(self.proj.dir, '.settings', 'org.eclipse.jdt.core.prefs') |
2039 jdtProperties = join(p.dir, '.settings', 'org.eclipse.jdt.core.prefs') | 1892 rootJdtProperties = join(self.proj.suite.mxDir, 'eclipse-settings', 'org.eclipse.jdt.core.prefs') |
2040 rootJdtProperties = join(p.suite.mxDir, 'eclipse-settings', 'org.eclipse.jdt.core.prefs') | |
2041 if not exists(jdtProperties) or os.path.getmtime(jdtProperties) < os.path.getmtime(rootJdtProperties): | 1893 if not exists(jdtProperties) or os.path.getmtime(jdtProperties) < os.path.getmtime(rootJdtProperties): |
2042 # Try to fix a missing properties file by running eclipseinit | 1894 # Try to fix a missing properties file by running eclipseinit |
2043 eclipseinit([], buildProcessorJars=False) | 1895 eclipseinit([], buildProcessorJars=False) |
2044 if not exists(jdtProperties): | 1896 if not exists(jdtProperties): |
2045 log('JDT properties file {0} not found'.format(jdtProperties)) | 1897 log('JDT properties file {0} not found'.format(jdtProperties)) |
2063 | 1915 |
2064 run_java(jdtVmArgs + jdtArgs) | 1916 run_java(jdtVmArgs + jdtArgs) |
2065 finally: | 1917 finally: |
2066 for n in toBeDeleted: | 1918 for n in toBeDeleted: |
2067 os.remove(n) | 1919 os.remove(n) |
1920 self.done = True | |
1921 | |
1922 def build(args, parser=None): | |
1923 """compile the Java and C sources, linking the latter | |
1924 | |
1925 Compile all the Java source code using the appropriate compilers | |
1926 and linkers for the various source code types.""" | |
1927 | |
1928 suppliedParser = parser is not None | |
1929 if not suppliedParser: | |
1930 parser = ArgumentParser(prog='mx build') | |
1931 | |
1932 parser = parser if parser is not None else ArgumentParser(prog='mx build') | |
1933 parser.add_argument('-f', action='store_true', dest='force', help='force build (disables timestamp checking)') | |
1934 parser.add_argument('-c', action='store_true', dest='clean', help='removes existing build output') | |
1935 parser.add_argument('-p', action='store_true', dest='parallelize', help='parallelizes Java compilation') | |
1936 parser.add_argument('--source', dest='compliance', help='Java compliance level for projects without an explicit one') | |
1937 parser.add_argument('--Wapi', action='store_true', dest='warnAPI', help='show warnings about using internal APIs') | |
1938 parser.add_argument('--projects', action='store', help='comma separated projects to build (omit to build all projects)') | |
1939 parser.add_argument('--only', action='store', help='comma separated projects to build, without checking their dependencies (omit to build all projects)') | |
1940 parser.add_argument('--no-java', action='store_false', dest='java', help='do not build Java projects') | |
1941 parser.add_argument('--no-native', action='store_false', dest='native', help='do not build native projects') | |
1942 parser.add_argument('--jdt-warning-as-error', action='store_true', help='convert all Eclipse batch compiler warnings to errors') | |
1943 parser.add_argument('--jdt-show-task-tags', action='store_true', help='show task tags as Eclipse batch compiler warnings') | |
1944 compilerSelect = parser.add_mutually_exclusive_group() | |
1945 compilerSelect.add_argument('--error-prone', dest='error_prone', help='path to error-prone.jar', metavar='<path>') | |
1946 compilerSelect.add_argument('--jdt', help='path to ecj.jar, the Eclipse batch compiler', default=_defaultEcjPath(), metavar='<path>') | |
1947 compilerSelect.add_argument('--force-javac', action='store_true', dest='javac', help='use javac despite ecj.jar is found or not') | |
1948 | |
1949 | |
1950 if suppliedParser: | |
1951 parser.add_argument('remainder', nargs=REMAINDER, metavar='...') | |
1952 | |
1953 args = parser.parse_args(args) | |
1954 | |
1955 if args.only is not None: | |
1956 # N.B. This build will not include dependencies including annotation processor dependencies | |
1957 sortedProjects = [project(name) for name in args.only.split(',')] | |
1958 else: | |
1959 if args.projects is not None: | |
1960 projectNames = args.projects.split(',') | |
1961 else: | |
1962 projectNames = None | |
1963 | |
1964 projects = _projects_opt_limit_to_suites(projects_from_names(projectNames)) | |
1965 # N.B. Limiting to a suite only affects the starting set of projects. Dependencies in other suites will still be compiled | |
1966 sortedProjects = sorted_project_deps(projects, includeAnnotationProcessors=True) | |
1967 | |
1968 if args.java: | |
1969 ideinit([], refreshOnly=True, buildProcessorJars=False) | |
1970 | |
1971 def prepareOutputDirs(p, clean): | |
1972 outputDir = p.output_dir() | |
1973 if exists(outputDir): | |
1974 if clean: | |
1975 log('Cleaning {0}...'.format(outputDir)) | |
1976 shutil.rmtree(outputDir) | |
1977 os.mkdir(outputDir) | |
1978 else: | |
1979 os.mkdir(outputDir) | |
1980 genDir = p.source_gen_dir() | |
1981 if genDir != '' and exists(genDir) and clean: | |
1982 log('Cleaning {0}...'.format(genDir)) | |
1983 for f in os.listdir(genDir): | |
1984 shutil.rmtree(join(genDir, f)) | |
1985 return outputDir | |
1986 | |
1987 tasks = {} | |
1988 for p in sortedProjects: | |
1989 if p.native: | |
1990 if args.native: | |
1991 log('Calling GNU make {0}...'.format(p.dir)) | |
1992 | |
1993 if args.clean: | |
1994 run([gmake_cmd(), 'clean'], cwd=p.dir) | |
1995 | |
1996 run([gmake_cmd()], cwd=p.dir) | |
1997 continue | |
1998 else: | |
1999 if not args.java: | |
2000 continue | |
2001 if exists(join(p.dir, 'plugin.xml')): # eclipse plugin project | |
2002 continue | |
2003 | |
2004 # skip building this Java project if its Java compliance level is "higher" than the configured JDK | |
2005 requiredCompliance = p.javaCompliance if p.javaCompliance else JavaCompliance(args.compliance) if args.compliance else None | |
2006 jdk = java(requiredCompliance) | |
2007 if not jdk: | |
2008 log('Excluding {0} from build (Java compliance level {1} required)'.format(p.name, requiredCompliance)) | |
2009 continue | |
2010 | |
2011 outputDir = prepareOutputDirs(p, args.clean) | |
2012 | |
2013 sourceDirs = p.source_dirs() | |
2014 buildReason = 'forced build' if args.force else None | |
2015 taskDeps = [] | |
2016 if not buildReason: | |
2017 for dep in p.all_deps([], includeLibs=False, includeAnnotationProcessors=True): | |
2018 taskDep = tasks.get(dep.name) | |
2019 if taskDep: | |
2020 if not buildReason: | |
2021 buildReason = dep.name + ' rebuilt' | |
2022 taskDeps.append(taskDep) | |
2023 | |
2024 jasminAvailable = None | |
2025 javafilelist = [] | |
2026 for sourceDir in sourceDirs: | |
2027 for root, _, files in os.walk(sourceDir): | |
2028 javafiles = [join(root, name) for name in files if name.endswith('.java') and name != 'package-info.java'] | |
2029 javafilelist += javafiles | |
2030 | |
2031 # Copy all non Java resources or assemble Jasmin files | |
2032 nonjavafilelist = [join(root, name) for name in files if not name.endswith('.java')] | |
2033 for src in nonjavafilelist: | |
2034 if src.endswith('.jasm'): | |
2035 className = None | |
2036 with open(src) as f: | |
2037 for line in f: | |
2038 if line.startswith('.class '): | |
2039 className = line.split()[-1] | |
2040 break | |
2041 | |
2042 if className is not None: | |
2043 jasminOutputDir = p.jasmin_output_dir() | |
2044 classFile = join(jasminOutputDir, className.replace('/', os.sep) + '.class') | |
2045 if exists(dirname(classFile)) and (not exists(classFile) or os.path.getmtime(classFile) < os.path.getmtime(src)): | |
2046 if jasminAvailable is None: | |
2047 try: | |
2048 with open(os.devnull) as devnull: | |
2049 subprocess.call('jasmin', stdout=devnull, stderr=subprocess.STDOUT) | |
2050 jasminAvailable = True | |
2051 except OSError: | |
2052 jasminAvailable = False | |
2053 | |
2054 if jasminAvailable: | |
2055 log('Assembling Jasmin file ' + src) | |
2056 run(['jasmin', '-d', jasminOutputDir, src]) | |
2057 else: | |
2058 log('The jasmin executable could not be found - skipping ' + src) | |
2059 with file(classFile, 'a'): | |
2060 os.utime(classFile, None) | |
2061 | |
2062 else: | |
2063 log('could not file .class directive in Jasmin source: ' + src) | |
2064 else: | |
2065 dst = join(outputDir, src[len(sourceDir) + 1:]) | |
2066 if not exists(dirname(dst)): | |
2067 os.makedirs(dirname(dst)) | |
2068 if exists(dirname(dst)) and (not exists(dst) or os.path.getmtime(dst) < os.path.getmtime(src)): | |
2069 shutil.copyfile(src, dst) | |
2070 | |
2071 if not buildReason: | |
2072 for javafile in javafiles: | |
2073 classfile = TimeStampFile(outputDir + javafile[len(sourceDir):-len('java')] + 'class') | |
2074 if not classfile.exists() or classfile.isOlderThan(javafile): | |
2075 buildReason = 'class file(s) out of date' | |
2076 break | |
2077 | |
2078 apsOutOfDate = p.update_current_annotation_processors_file() | |
2079 if apsOutOfDate: | |
2080 buildReason = 'annotation processor(s) changed' | |
2081 | |
2082 if not buildReason: | |
2083 logv('[all class files for {0} are up to date - skipping]'.format(p.name)) | |
2084 continue | |
2085 | |
2086 if len(javafilelist) == 0: | |
2087 logv('[no Java sources for {0} - skipping]'.format(p.name)) | |
2088 continue | |
2089 | |
2090 task = JavaCompileTask(args, p, buildReason, javafilelist, jdk, outputDir, taskDeps) | |
2091 | |
2092 if args.parallelize: | |
2093 # Best to initialize class paths on main process | |
2094 jdk.bootclasspath() | |
2095 task.proc = None | |
2096 tasks[p.name] = task | |
2097 else: | |
2098 task.execute() | |
2099 | |
2100 if args.parallelize: | |
2101 | |
2102 def joinTasks(tasks): | |
2103 failed = [] | |
2104 for t in tasks: | |
2105 t.proc.join() | |
2106 if t.proc.exitcode != 0: | |
2107 failed.append(t) | |
2108 return failed | |
2109 | |
2110 def checkTasks(tasks): | |
2111 active = [] | |
2112 for t in tasks: | |
2113 if t.proc.is_alive(): | |
2114 active.append(t) | |
2115 else: | |
2116 if t.proc.exitcode != 0: | |
2117 return ([], joinTasks(tasks)) | |
2118 return (active, []) | |
2119 | |
2120 def remainingDepsDepth(task): | |
2121 if task._d is None: | |
2122 incompleteDeps = [d for d in task.deps if d.proc is None or d.proc.is_alive()] | |
2123 if len(incompleteDeps) == 0: | |
2124 task._d = 0 | |
2125 else: | |
2126 task._d = max([remainingDepsDepth(t) for t in incompleteDeps]) + 1 | |
2127 return task._d | |
2128 | |
2129 def sortWorklist(tasks): | |
2130 for t in tasks: | |
2131 t._d = None | |
2132 return sorted(tasks, lambda x, y : remainingDepsDepth(x) - remainingDepsDepth(y)) | |
2133 | |
2134 import multiprocessing | |
2135 cpus = multiprocessing.cpu_count() | |
2136 worklist = sortWorklist(tasks.values()) | |
2137 active = [] | |
2138 while len(worklist) != 0: | |
2139 while True: | |
2140 active, failed = checkTasks(active) | |
2141 if len(failed) != 0: | |
2142 assert not active, active | |
2143 break | |
2144 if len(active) == cpus: | |
2145 # Sleep for 1 second | |
2146 time.sleep(1) | |
2147 else: | |
2148 break | |
2149 | |
2150 def executeTask(task): | |
2151 task.execute() | |
2152 | |
2153 def depsDone(task): | |
2154 for d in task.deps: | |
2155 if d.proc is None or d.proc.exitcode is None: | |
2156 return False | |
2157 return True | |
2158 | |
2159 for task in worklist: | |
2160 if depsDone(task): | |
2161 worklist.remove(task) | |
2162 task.proc = multiprocessing.Process(target=executeTask, args=(task,)) | |
2163 task.proc.start() | |
2164 active.append(task) | |
2165 if len(active) == cpus: | |
2166 break | |
2167 | |
2168 worklist = sortWorklist(worklist) | |
2169 joinTasks(active) | |
2068 | 2170 |
2069 for dist in _dists.values(): | 2171 for dist in _dists.values(): |
2070 archive(['@' + dist.name]) | 2172 archive(['@' + dist.name]) |
2071 | 2173 |
2072 if suppliedParser: | 2174 if suppliedParser: |