# HG changeset patch # User Doug Simon # Date 1378218821 -7200 # Node ID dc3c8df5590565dabbc8b30a64fa09146ab6c8a0 # Parent 8d4e5e08d83f2b62f932713027ca690b3f682275 added support for pylint and fixed errors/warnings it found diff -r 8d4e5e08d83f -r dc3c8df55905 mx/.pylintrc --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/mx/.pylintrc Tue Sep 03 16:33:41 2013 +0200 @@ -0,0 +1,275 @@ +[MASTER] + +# Specify a configuration file. +#rcfile= + +# Python code to execute, usually for sys.path manipulation such as +# pygtk.require(). +#init-hook= + +# Profiled execution. +profile=no + +# Add files or directories to the blacklist. They should be base names, not +# paths. +ignore=CVS + +# Pickle collected data for later comparisons. +persistent=yes + +# List of plugins (as comma separated values of python modules names) to load, +# usually to register additional checkers. +load-plugins= + + +[MESSAGES CONTROL] + +# Enable the message, report, category or checker with the given id(s). You can +# either give multiple identifier separated by comma (,) or put this option +# multiple time. See also the "--disable" option for examples. +#enable= + +# Disable the message, report, category or checker with the given id(s). You +# can either give multiple identifiers separated by comma (,) or put this +# option multiple times (only on the command line, not in the configuration +# file where it should appear only once).You can also use "--disable=all" to +# disable everything first and then reenable specific checks. For example, if +# you want to run only the similarities checker, you can use "--disable=all +# --enable=similarities". If you want to run only the classes checker, but have +# no Warning level messages displayed, use"--disable=all --enable=classes +# --disable=W" +disable=attribute-defined-outside-init,arguments-differ, + bare-except,global-statement,protected-access,redefined-outer-name, + unused-argument,star-args,pointless-string-statement,old-style-class, + too-many-lines,missing-docstring,no-init + +[REPORTS] + +# Set the output format. Available formats are text, parseable, colorized, msvs +# (visual studio) and html. You can also give a reporter class, eg +# mypackage.mymodule.MyReporterClass. +output-format=text + +# Put messages in a separate file for each module / package specified on the +# command line instead of printing them on stdout. Reports (if any) will be +# written in a file name "pylint_global.[txt|html]". +files-output=no + +# Tells whether to display a full report or only the messages +reports=no + +# Python expression which should return a note less than 10 (10 is the highest +# note). You have access to the variables errors warning, statement which +# respectively contain the number of errors / warnings messages and the total +# number of statements analyzed. This is used by the global evaluation report +# (RP0004). +evaluation=10.0 - ((float(5 * error + warning + refactor + convention) / statement) * 10) + +# Add a comment according to your evaluation note. This is used by the global +# evaluation report (RP0004). +comment=no + +# Template used to display messages. This is a python new-style format string +# used to format the massage information. See doc for all details +#msg-template= + + +[BASIC] + +# Required attributes for module, separated by a comma +required-attributes= + +# List of builtins function names that should not be used, separated by a comma +bad-functions=filter,apply,input + +# Regular expression which should only match correct module names +module-rgx=(([a-z_][a-z0-9_]*)|([A-Z][a-zA-Z0-9]+))$ + +# Regular expression which should only match correct module level names +const-rgx=[a-zA-Z0-9_]{2,30}$ + +# Regular expression which should only match correct class names +class-rgx=[A-Z_][a-zA-Z0-9]+$ + +# Regular expression which should only match correct function names +function-rgx=[a-z_][a-zA-Z0-9_]{1,30}$ + +# Regular expression which should only match correct method names +method-rgx=[a-z_][a-zA-Z0-9_]{2,30}$ + +# Regular expression which should only match correct instance attribute names +attr-rgx=[a-z_][a-zA-Z0-9_]{1,30}$ + +# Regular expression which should only match correct argument names +argument-rgx=[a-z_][a-zA-Z0-9_]{0,30}$ + +# Regular expression which should only match correct variable names +variable-rgx=[a-z_][a-zA-Z0-9_]{0,30}$ + +# Regular expression which should only match correct attribute names in class +# bodies +class-attribute-rgx=([A-Za-z_][A-Za-z0-9_]{2,30}|(__.*__))$ + +# Regular expression which should only match correct list comprehension / +# generator expression variable names +inlinevar-rgx=[A-Za-z_][A-Za-z0-9_]*$ + +# Good variable names which should always be accepted, separated by a comma +good-names=i,j,k,ex,Run,_ + +# Bad variable names which should always be refused, separated by a comma +bad-names=foo,bar,baz,toto,tutu,tata + +# Regular expression which should only match function or class names that do +# not require a docstring. +no-docstring-rgx=.* + +# Minimum line length for functions/classes that require docstrings, shorter +# ones are exempt. +docstring-min-length=-1 + + +[FORMAT] + +# Maximum number of characters on a single line. +max-line-length=300 + +# Regexp for a line that is allowed to be longer than the limit. +ignore-long-lines=^\s*(# )??$ + +# Maximum number of lines in a module +max-module-lines=1000 + +# String used as indentation unit. This is usually " " (4 spaces) or "\t" (1 +# tab). +indent-string=' ' + + +[MISCELLANEOUS] + +# List of note tags to take in consideration, separated by a comma. +notes=FIXME + + +[SIMILARITIES] + +# Minimum lines number of a similarity. +min-similarity-lines=4 + +# Ignore comments when computing similarities. +ignore-comments=yes + +# Ignore docstrings when computing similarities. +ignore-docstrings=yes + +# Ignore imports when computing similarities. +ignore-imports=no + + +[TYPECHECK] + +# Tells whether missing members accessed in mixin class should be ignored. A +# mixin class is detected if its name ends with "mixin" (case insensitive). +ignore-mixin-members=yes + +# List of classes names for which member attributes should not be checked +# (useful for classes with attributes dynamically set). +ignored-classes=SQLObject + +# When zope mode is activated, add a predefined set of Zope acquired attributes +# to generated-members. +zope=no + +# List of members which are set dynamically and missed by pylint inference +# system, and so shouldn't trigger E0201 when accessed. Python regular +# expressions are accepted. +generated-members=REQUEST,acl_users,aq_parent + + +[VARIABLES] + +# Tells whether we should check for unused import in __init__ files. +init-import=no + +# A regular expression matching the beginning of the name of dummy variables +# (i.e. not used). +dummy-variables-rgx=_$|dummy + +# List of additional names supposed to be defined in builtins. Remember that +# you should avoid to define new builtins when possible. +additional-builtins= + + +[CLASSES] + +# List of interface methods to ignore, separated by a comma. This is used for +# instance to not check methods defines in Zope's Interface base class. +ignore-iface-methods=isImplementedBy,deferred,extends,names,namesAndDescriptions,queryDescriptionFor,getBases,getDescriptionFor,getDoc,getName,getTaggedValue,getTaggedValueTags,isEqualOrExtendedBy,setTaggedValue,isImplementedByInstancesOf,adaptWith,is_implemented_by + +# List of method names used to declare (i.e. assign) instance attributes. +defining-attr-methods=__init__,__new__,setUp + +# List of valid names for the first argument in a class method. +valid-classmethod-first-arg=cls + +# List of valid names for the first argument in a metaclass class method. +valid-metaclass-classmethod-first-arg=mcs + + +[DESIGN] + +# Maximum number of arguments for function / method +max-args=5 + +# Argument names that match this expression will be ignored. Default to name +# with leading underscore +ignored-argument-names=_.* + +# Maximum number of locals for function / method body +max-locals=15 + +# Maximum number of return / yield for function / method body +max-returns=6 + +# Maximum number of branch for function / method body +max-branches=12 + +# Maximum number of statements in function / method body +max-statements=50 + +# Maximum number of parents for a class (see R0901). +max-parents=7 + +# Maximum number of attributes for a class (see R0902). +max-attributes=7 + +# Minimum number of public methods for a class (see R0903). +min-public-methods=2 + +# Maximum number of public methods for a class (see R0904). +max-public-methods=20 + + +[IMPORTS] + +# Deprecated modules which should not be used, separated by a comma +deprecated-modules=regsub,TERMIOS,Bastion,rexec + +# Create a graph of every (i.e. internal and external) dependencies in the +# given file (report RP0402 must not be disabled) +import-graph= + +# Create a graph of external dependencies in the given file (report RP0402 must +# not be disabled) +ext-import-graph= + +# Create a graph of internal dependencies in the given file (report RP0402 must +# not be disabled) +int-import-graph= + + +[EXCEPTIONS] + +# Exceptions that will emit a warning when being caught. Defaults to +# "Exception" +overgeneral-exceptions=Exception diff -r 8d4e5e08d83f -r dc3c8df55905 mx/commands.py --- a/mx/commands.py Tue Sep 03 14:14:09 2013 +0200 +++ b/mx/commands.py Tue Sep 03 16:33:41 2013 +0200 @@ -45,10 +45,10 @@ _vmChoices = { 'graal' : 'All compilation is performed with Graal. This includes bootstrapping Graal itself unless -XX:-BootstrapGraal is used.', 'server' : 'Normal compilation is performed with the tiered system (i.e., client + server), Truffle compilation is performed with Graal. Use this for optimal Truffle performance.', - 'client' : None, # normal compilation with client compiler, explicit compilation (e.g., by Truffle) with Graal - 'server-nograal' : None, # all compilation with tiered system (i.e., client + server), Graal omitted - 'client-nograal' : None, # all compilation with client compiler, Graal omitted - 'original' : None, # default VM copied from bootstrap JDK + 'client' : None, # normal compilation with client compiler, explicit compilation (e.g., by Truffle) with Graal + 'server-nograal' : None, # all compilation with tiered system (i.e., client + server), Graal omitted + 'client-nograal' : None, # all compilation with client compiler, Graal omitted + 'original' : None, # default VM copied from bootstrap JDK } """ The VM that will be run by the 'vm' command and built by default by the 'build' command. @@ -101,7 +101,7 @@ _vm = vm return vm -""" +""" A context manager that can be used with the 'with' statement to set the VM used by all VM executions within the scope of the 'with' statement. For example: @@ -116,17 +116,17 @@ self.build = build if build else _vmbuild self.previousVm = _vm self.previousBuild = _vmbuild - + def __enter__(self): global _vm, _vmbuild _vm = self.vm _vmbuild = self.build - + def __exit__(self, exc_type, exc_value, traceback): global _vm, _vmbuild _vm = self.previousVm _vmbuild = self.previousBuild - + def _chmodDir(chmodFlags, dirname, fnames): os.chmod(dirname, chmodFlags) for name in fnames: @@ -144,7 +144,7 @@ shutil.rmtree(name) elif os.path.isfile(name): os.unlink(name) - + rmIfExists(join(_graal_home, 'build')) rmIfExists(join(_graal_home, 'build-nograal')) rmIfExists(_jdksDir()) @@ -153,7 +153,7 @@ def export(args): """create a GraalVM zip file for distribution""" - parser = ArgumentParser(prog='mx export'); + parser = ArgumentParser(prog='mx export') parser.add_argument('--omit-vm-build', action='store_false', dest='vmbuild', help='omit VM build step') parser.add_argument('--omit-dist-init', action='store_false', dest='distInit', help='omit class files and IDE configurations from distribution') parser.add_argument('zipfile', nargs=REMAINDER, metavar='zipfile') @@ -200,13 +200,13 @@ vmOpts, benchmarksAndOptions = _extract_VM_args(args, useDoubleDash=availableBenchmarks is None) if availableBenchmarks is None: - harnessArgs = benchmarksAndOptions + harnessArgs = benchmarksAndOptions return runBenchmark(None, harnessArgs, vmOpts) if len(benchmarksAndOptions) == 0: mx.abort('at least one benchmark name or "all" must be specified') benchmarks = list(itertools.takewhile(lambda x: not x.startswith('-'), benchmarksAndOptions)) - harnessArgs = benchmarksAndOptions[len(benchmarks):] + harnessArgs = benchmarksAndOptions[len(benchmarks):] if 'all' in benchmarks: benchmarks = availableBenchmarks @@ -228,7 +228,7 @@ def launcher(bm, harnessArgs, extraVmOpts): return sanitycheck.getDacapo(bm, harnessArgs).test(_get_vm(), extraVmOpts=extraVmOpts) - + _run_benchmark(args, sanitycheck.dacapoSanityWarmup.keys(), launcher) def scaladacapo(args): @@ -332,7 +332,7 @@ if mx.get_os() != 'windows': chmodRecursive(jdk, 0755) shutil.move(join(_vmLibDirInJdk(jdk), defaultVM), join(_vmLibDirInJdk(jdk), 'original')) - + with open(jvmCfg, 'w') as fp: for line in jvmCfgLines: @@ -348,10 +348,10 @@ if _installed_jdks and mx._opts.verbose: mx.log("Could not find JDK directory at " + jdk) _handle_missing_VM(build, vmToCheck if vmToCheck else 'graal') - + if installGraalJar: _installGraalJarInJdks(mx.distribution('GRAAL')) - + if vmToCheck is not None: jvmCfg = _vmCfgInJdk(jdk) found = False @@ -362,7 +362,7 @@ break if not found: _handle_missing_VM(build, vmToCheck) - + return jdk def _installGraalJarInJdks(graalDist): @@ -378,25 +378,27 @@ shutil.copyfile(graalJar, tmp) os.close(fd) shutil.move(tmp, join(jreLibDir, 'graal.jar')) - + if exists(graalOptions): shutil.copy(graalOptions, join(jreLibDir, 'graal.options')) # run a command in the windows SDK Debug Shell -def _runInDebugShell(cmd, workingDir, logFile=None, findInOutput=None, respondTo={}): +def _runInDebugShell(cmd, workingDir, logFile=None, findInOutput=None, respondTo=None): + if respondTo is None: + respondTo = {} newLine = os.linesep - STARTTOKEN = 'RUNINDEBUGSHELL_STARTSEQUENCE' - ENDTOKEN = 'RUNINDEBUGSHELL_ENDSEQUENCE' + startToken = 'RUNINDEBUGSHELL_STARTSEQUENCE' + endToken = 'RUNINDEBUGSHELL_ENDSEQUENCE' winSDK = mx.get_env('WIN_SDK', 'C:\\Program Files\\Microsoft SDKs\\Windows\\v7.1\\') - + if not exists(winSDK): mx.abort("Could not find Windows SDK : '" + winSDK + "' does not exist") - + if not exists(join(winSDK, 'Bin', 'SetEnv.cmd')): mx.abort("Invalid Windows SDK path (" + winSDK + ") : could not find Bin/SetEnv.cmd (you can use the WIN_SDK environment variable to specify an other path)") - p = subprocess.Popen('cmd.exe /E:ON /V:ON /K ""' + winSDK + '/Bin/SetEnv.cmd" & echo ' + STARTTOKEN + '"', \ + p = subprocess.Popen('cmd.exe /E:ON /V:ON /K ""' + winSDK + '/Bin/SetEnv.cmd" & echo ' + startToken + '"', \ shell=True, stdin=subprocess.PIPE, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, creationflags=subprocess.CREATE_NEW_PROCESS_GROUP) stdout = p.stdout stdin = p.stdin @@ -404,20 +406,20 @@ log = open(logFile, 'w') ret = False while True: - + # encoding may be None on windows plattforms if sys.stdout.encoding is None: encoding = 'utf-8' else: encoding = sys.stdout.encoding - + line = stdout.readline().decode(encoding) if logFile: log.write(line.encode('utf-8')) line = line.strip() mx.log(line) - if line == STARTTOKEN: - stdin.write('cd /D ' + workingDir + ' & ' + cmd + ' & echo ' + ENDTOKEN + newLine) + if line == startToken: + stdin.write('cd /D ' + workingDir + ' & ' + cmd + ' & echo ' + endToken + newLine) for regex in respondTo.keys(): match = regex.search(line) if match: @@ -426,7 +428,7 @@ match = findInOutput.search(line) if match: ret = True - if line == ENDTOKEN: + if line == endToken: if not findInOutput: stdin.write('echo ERRXXX%errorlevel%' + newLine) else: @@ -434,7 +436,7 @@ if line.startswith('ERRXXX'): if line == 'ERRXXX0': ret = True - break; + break stdin.write('exit' + newLine) if logFile: log.close() @@ -456,16 +458,16 @@ 'INSTALL' : 'Install the built VM into the JDK? (default: y)', 'ZIP_DEBUGINFO_FILES' : 'Install zipped debug symbols file? (default: 0)', } - + mx.log('HotSpot build variables that can be set by the -D option to "mx build":') mx.log('') for n in sorted(buildVars.iterkeys()): mx.log(n) mx.log(textwrap.fill(buildVars[n], initial_indent=' ', subsequent_indent=' ', width=200)) - + mx.log('') mx.log('Note that these variables can be given persistent values in the file ' + join(_graal_home, 'mx', 'env') + ' (see \'mx about\').') - + def build(args, vm=None): """build the VM binary @@ -486,7 +488,7 @@ return result # Call mx.build to compile the Java sources - parser=AP() + parser = AP() parser.add_argument('--export-dir', help='directory to which graal.jar and graal.options will be copied', metavar='') parser.add_argument('-D', action='append', help='set a HotSpot build variable (run \'mx buildvars\' to list variables)', metavar='name=value') opts2 = mx.build(['--source', '1.7'] + args, parser=parser) @@ -520,11 +522,11 @@ else: assert vm == 'graal', vm buildSuffix = 'graal' - + if _installed_jdks: - if not mx.ask_yes_no("You are going to build while --installed-jdks is set (" + _installed_jdks + ") are you sure you want to continue", 'n'): + if not mx.ask_yes_no("You are going to build because --installed-jdks is set (" + _installed_jdks + ") - are you sure you want to continue", 'n'): mx.abort(1) - + for build in builds: if build == 'ide-build-target': build = os.environ.get('IDE_BUILD_TARGET', None) @@ -579,7 +581,7 @@ variant = {'client': 'compiler1', 'server': 'compiler2'}.get(vm, vm) project_config = variant + '_' + build _runInDebugShell('msbuild ' + _graal_home + r'\build\vs-amd64\jvm.vcproj /p:Configuration=' + project_config + ' /target:clean', _graal_home) - winCompileCmd = r'set HotSpotMksHome=' + mksHome + r'& set OUT_DIR=' + jdk + r'& set JAVA_HOME=' + jdk + r'& set path=%JAVA_HOME%\bin;%path%;%HotSpotMksHome%& cd /D "' +_graal_home + r'\make\windows"& call create.bat ' + _graal_home + winCompileCmd = r'set HotSpotMksHome=' + mksHome + r'& set OUT_DIR=' + jdk + r'& set JAVA_HOME=' + jdk + r'& set path=%JAVA_HOME%\bin;%path%;%HotSpotMksHome%& cd /D "' + _graal_home + r'\make\windows"& call create.bat ' + _graal_home print(winCompileCmd) winCompileSuccess = re.compile(r"^Writing \.vcxproj file:") if not _runInDebugShell(winCompileCmd, _graal_home, compilelogfile, winCompileSuccess): @@ -592,14 +594,14 @@ else: cpus = multiprocessing.cpu_count() runCmd = [mx.gmake_cmd()] - runCmd.append(build + buildSuffix) + runCmd.append(build + buildSuffix) env = os.environ.copy() - + if opts2.D: for nv in opts2.D: name, value = nv.split('=', 1) env[name.strip()] = value - + env.setdefault('ARCH_DATA_MODEL', '64') env.setdefault('LANG', 'C') env.setdefault('HOTSPOT_BUILD_JOBS', str(cpus)) @@ -647,7 +649,7 @@ if line.strip() == vmKnown.strip(): found = True lines.append(line) - + if not found: mx.log('Appending "' + prefix + 'KNOWN" to ' + jvmCfg) if mx.get_os() != 'windows': @@ -701,7 +703,7 @@ for p in mx.projects(): excludes += _find_classes_with_annotations(p, None, ['@Snippet', '@ClassSubstitution', '@Test'], includeInnerClasses=True).keys() excludes += p.find_classes_with_matching_source_line(None, lambda line: 'JaCoCo Exclude' in line, includeInnerClasses=True).keys() - + includes = ['com.oracle.graal.*'] agentOptions = { 'append' : 'true' if _jacoco == 'append' else 'false', @@ -716,12 +718,12 @@ exe = join(jdk, 'bin', mx.exe_suffix('java')) pfx = _vm_prefix.split() if _vm_prefix is not None else [] - + if '-version' in args: - ignoredArgs = args[args.index('-version')+1:] + ignoredArgs = args[args.index('-version') + 1:] if len(ignoredArgs) > 0: mx.log("Warning: The following options will be ignored by the vm because they come after the '-version' argument: " + ' '.join(ignoredArgs)) - + return mx.run(pfx + [exe, '-' + vm] + args, nonZeroIsFatal=nonZeroIsFatal, out=out, err=err, cwd=cwd, timeout=timeout) def _find_classes_with_annotations(p, pkgRoot, annotations, includeInnerClasses=False): @@ -730,7 +732,7 @@ (ignoring preceding whitespace) and return the fully qualified class name for each Java source file matched in a list. """ - + matches = lambda line : len([a for a in annotations if line == a or line.startswith(a + '(')]) != 0 return p.find_classes_with_matching_source_line(pkgRoot, matches, includeInnerClasses) @@ -742,7 +744,7 @@ if useDoubleDash: if args[i] == '--': vmArgs = args[:i] - remainder = args[i + 1:] + remainder = args[i + 1:] return vmArgs, remainder else: if not args[i].startswith('-'): @@ -752,25 +754,19 @@ else: continue vmArgs = args[:i] - remainder = args[i:] + remainder = args[i:] return vmArgs, remainder - + return args, [] - + def _run_tests(args, harness, annotations, testfile): - - + + vmArgs, tests = _extract_VM_args(args) for t in tests: if t.startswith('-'): mx.abort('VM option ' + t + ' must precede ' + tests[0]) - def containsAny(c, substrings): - for s in substrings: - if s in c: - return True - return False - candidates = [] for p in mx.projects(): if mx.java().javaCompliance < p.javaCompliance: @@ -835,14 +831,14 @@ If filters are supplied, only tests whose fully qualified name includes a filter as a substring are run. - + For example, this command line: - + mx unittest -G:Dump= -G:MethodFilter=BC_aload.* -G:+PrintCFG BC_aload - + will run all JUnit test classes that contain 'BC_aload' in their - fully qualified name and will pass these options to the VM: - + fully qualified name and will pass these options to the VM: + -G:Dump= -G:MethodFilter=BC_aload.* -G:+PrintCFG To get around command line length limitations on some OSes, the @@ -855,7 +851,7 @@ As with all other commands, using the global '-v' before 'unittest' command will cause mx to show the complete command line it uses to run the VM. -""" +""" def unittest(args): """run the JUnit tests (all testcases){0}""" @@ -877,8 +873,8 @@ vmsDefault = ','.join(_vmChoices.keys()) vmbuildsDefault = ','.join(_vmbuildChoices) - - parser = ArgumentParser(prog='mx buildvms'); + + parser = ArgumentParser(prog='mx buildvms') parser.add_argument('--vms', help='a comma separated list of VMs to build (default: ' + vmsDefault + ')', metavar='', default=vmsDefault) parser.add_argument('--builds', help='a comma separated list of build types (default: ' + vmbuildsDefault + ')', metavar='', default=vmbuildsDefault) parser.add_argument('-n', '--no-check', action='store_true', help='omit running "java -version" after each build') @@ -899,7 +895,8 @@ start = time.time() mx.log('BEGIN: ' + v + '-' + vmbuild + '\t(see: ' + logFile + ')') # Run as subprocess so that output can be directed to a file - subprocess.check_call([sys.executable, '-u', join('mxtool', 'mx.py'), '--vm', v, '--vmbuild', vmbuild, 'build'], cwd=_graal_home, stdout=log, stderr=subprocess.STDOUT) + subprocess.check_call([sys.executable, '-u', join('mxtool', 'mx.py'), '--vm', v, '--vmbuild', + vmbuild, 'build'], cwd=_graal_home, stdout=log, stderr=subprocess.STDOUT) duration = datetime.timedelta(seconds=time.time() - start) mx.log('END: ' + v + '-' + vmbuild + '\t[' + str(duration) + ']') else: @@ -938,7 +935,7 @@ mx.abort(codeOrMessage) return self - parser = ArgumentParser(prog='mx gate'); + parser = ArgumentParser(prog='mx gate') parser.add_argument('-j', '--omit-java-clean', action='store_false', dest='cleanJava', help='omit cleaning Java native code') parser.add_argument('-n', '--omit-native-clean', action='store_false', dest='cleanNative', help='omit cleaning and building native code') parser.add_argument('-g', '--only-build-graalvm', action='store_false', dest='buildNonGraal', help='only build the Graal VM') @@ -947,7 +944,7 @@ args = parser.parse_args(args) global _jacoco - + tasks = [] total = Task('Gate') try: @@ -978,7 +975,7 @@ if mx.canonicalizeprojects([]) != 0: t.abort('Rerun "mx canonicalizeprojects" and check-in the modified mx/projects files.') tasks.append(t.stop()) - + t = Task('BuildJava') build(['--no-native', '--jdt-warning-as-error']) tasks.append(t.stop()) @@ -987,10 +984,10 @@ if mx.checkstyle([]) != 0: t.abort('Checkstyle warnings were found') tasks.append(t.stop()) - + if exists('jacoco.exec'): os.unlink('jacoco.exec') - + if args.jacocout is not None: _jacoco = 'append' else: @@ -1009,10 +1006,10 @@ t = Task('BootstrapWithGCVerification:product') vm(['-XX:+UnlockDiagnosticVMOptions', '-XX:+VerifyBeforeGC', '-XX:+VerifyAfterGC', '-version']) tasks.append(t.stop()) - + with VM('graal', 'product'): t = Task('BootstrapWithG1GCVerification:product') - vm(['-XX:+UnlockDiagnosticVMOptions', '-XX:-UseSerialGC','-XX:+UseG1GC','-XX:+UseNewCode','-XX:+VerifyBeforeGC', '-XX:+VerifyAfterGC', '-version']) + vm(['-XX:+UnlockDiagnosticVMOptions', '-XX:-UseSerialGC', '-XX:+UseG1GC', '-XX:+UseNewCode', '-XX:+VerifyBeforeGC', '-XX:+VerifyAfterGC', '-version']) tasks.append(t.stop()) with VM('graal', 'product'): @@ -1025,7 +1022,7 @@ vm(['-G:+AOTCompilation', '-G:+VerifyPhases', '-esa', '-version']) tasks.append(t.stop()) - with VM('server', 'product'): # hosted mode + with VM('server', 'product'): # hosted mode t = Task('UnitTests:hosted-product') unittest([]) tasks.append(t.stop()) @@ -1039,7 +1036,7 @@ if args.jacocout is not None: jacocoreport([args.jacocout]) - + _jacoco = 'off' t = Task('CleanAndBuildGraalVisualizer') @@ -1060,7 +1057,7 @@ t = Task('DaCapo_pmd:' + theVm + ':' + vmbuild) dacapo(['pmd']) tasks.append(t.stop()) - + t = Task('UnitTests:' + theVm + ':' + vmbuild) unittest(['-XX:CompileCommand=exclude,*::run*', 'graal.api']) tasks.append(t.stop()) @@ -1080,7 +1077,7 @@ mx.log(' ' + str(t.duration) + '\t' + t.title) mx.log(' =======') mx.log(' ' + str(total.duration)) - + def deoptalot(args): """bootstrap a fastdebug Graal VM with DeoptimizeALot and VerifyOops on @@ -1090,15 +1087,15 @@ if len(args) > 0 and args[0].isdigit(): count = int(args[0]) del args[0] - + for _ in range(count): if not vm(['-XX:+DeoptimizeALot', '-XX:+VerifyOops'] + args + ['-version'], vmbuild='fastdebug') == 0: mx.abort("Failed") - + def longtests(args): - + deoptalot(['15', '-Xmx48m']) - + dacapo(['100', 'eclipse', '-esa']) def gv(args): @@ -1133,7 +1130,7 @@ vm = _get_vm() if len(args) is 0: args = ['all'] - + vmArgs = [arg for arg in args if arg.startswith('-')] def benchmarks_in_group(group): @@ -1142,7 +1139,7 @@ results = {} benchmarks = [] - #DaCapo + # DaCapo if ('dacapo' in args or 'all' in args): benchmarks += sanitycheck.getDacapos(level=sanitycheck.SanityCheckLevel.Benchmark) else: @@ -1165,23 +1162,23 @@ if (iterations > 0): benchmarks += [sanitycheck.getScalaDacapo(scaladacapo, ['-n', str(iterations)])] - #Bootstrap + # Bootstrap if ('bootstrap' in args or 'all' in args): benchmarks += sanitycheck.getBootstraps() - #SPECjvm2008 + # SPECjvm2008 if ('specjvm2008' in args or 'all' in args): benchmarks += [sanitycheck.getSPECjvm2008(['-ikv', '-wt', '120', '-it', '120'])] else: specjvms = benchmarks_in_group('specjvm2008') for specjvm in specjvms: benchmarks += [sanitycheck.getSPECjvm2008(['-ikv', '-wt', '120', '-it', '120', specjvm])] - + if ('specjbb2005' in args or 'all' in args): benchmarks += [sanitycheck.getSPECjbb2005()] - - if ('specjbb2013' in args): # or 'all' in args //currently not in default set + + if ('specjbb2013' in args): # or 'all' in args //currently not in default set benchmarks += [sanitycheck.getSPECjbb2013()] - + if ('ctw-full' in args): benchmarks.append(sanitycheck.getCTW(vm, sanitycheck.CTWMode.Full)) if ('ctw-noinline' in args): @@ -1200,10 +1197,10 @@ def specjvm2008(args): """run one or more SPECjvm2008 benchmarks""" - + def launcher(bm, harnessArgs, extraVmOpts): return sanitycheck.getSPECjvm2008(harnessArgs + [bm]).bench(_get_vm(), extraVmOpts=extraVmOpts) - + availableBenchmarks = set(sanitycheck.specjvm2008Names) for name in sanitycheck.specjvm2008Names: parts = name.rsplit('.', 1) @@ -1214,23 +1211,23 @@ availableBenchmarks.add(group) _run_benchmark(args, sorted(availableBenchmarks), launcher) - + def specjbb2013(args): """runs the composite SPECjbb2013 benchmark""" - + def launcher(bm, harnessArgs, extraVmOpts): assert bm is None return sanitycheck.getSPECjbb2013(harnessArgs).bench(_get_vm(), extraVmOpts=extraVmOpts) - + _run_benchmark(args, None, launcher) def specjbb2005(args): """runs the composite SPECjbb2005 benchmark""" - + def launcher(bm, harnessArgs, extraVmOpts): assert bm is None return sanitycheck.getSPECjbb2005(harnessArgs).bench(_get_vm(), extraVmOpts=extraVmOpts) - + _run_benchmark(args, None, launcher) def hsdis(args, copyToDir=None): @@ -1254,17 +1251,17 @@ Run a tool over the input files to convert all embedded HexCodeFiles to a disassembled format.""" - parser = ArgumentParser(prog='mx hcfdis'); + parser = ArgumentParser(prog='mx hcfdis') parser.add_argument('-m', '--map', help='address to symbol map applied to disassembler output') parser.add_argument('files', nargs=REMAINDER, metavar='files...') args = parser.parse_args(args) - + path = join(_graal_home, 'lib', 'hcfdis-1.jar') if not exists(path): mx.download(path, ['http://lafo.ssw.uni-linz.ac.at/hcfdis-1.jar']) mx.run_java(['-jar', path] + args.files) - + if args.map is not None: addressRE = re.compile(r'0[xX]([A-Fa-f0-9]+)') with open(args.map) as fp: @@ -1273,8 +1270,8 @@ for l in lines: addressAndSymbol = l.split(' ', 1) if len(addressAndSymbol) == 2: - address, symbol = addressAndSymbol; - if address.startswith('0x'): + address, symbol = addressAndSymbol + if address.startswith('0x'): address = long(address, 16) symbols[address] = symbol for f in args.files: @@ -1364,7 +1361,7 @@ if (_vmSourcesAvailable): mx.add_argument('--vm', action='store', dest='vm', choices=_vmChoices.keys(), help='the VM type to build/run') - mx.add_argument('--vmbuild', action='store', dest='vmbuild', choices=_vmbuildChoices, help='the VM build to build/run (default: ' + _vmbuildChoices[0] +')') + mx.add_argument('--vmbuild', action='store', dest='vmbuild', choices=_vmbuildChoices, help='the VM build to build/run (default: ' + _vmbuildChoices[0] + ')') mx.add_argument('--ecl', action='store_true', dest='make_eclipse_launch', help='create launch configuration for running VM execution(s) in Eclipse') mx.add_argument('--vmprefix', action='store', dest='vm_prefix', help='prefix for running the VM (e.g. "/usr/bin/gdb --args")', metavar='') mx.add_argument('--gdb', action='store_const', const='/usr/bin/gdb --args', dest='vm_prefix', help='alias for --vmprefix "/usr/bin/gdb --args"') @@ -1373,9 +1370,9 @@ 'export': [export, '[-options] [zipfile]'], }) - mx.commands.update(commands) + mx._commands.update(commands) -def mx_post_parse_cmd_line(opts):# +def mx_post_parse_cmd_line(opts): # # TODO _minVersion check could probably be part of a Suite in mx? if (mx.java().version < _minVersion) : mx.abort('Requires Java version ' + str(_minVersion) + ' or greater, got version ' + str(mx.java().version)) diff -r 8d4e5e08d83f -r dc3c8df55905 mx/outputparser.py --- a/mx/outputparser.py Tue Sep 03 14:14:09 2013 +0200 +++ b/mx/outputparser.py Tue Sep 03 16:33:41 2013 +0200 @@ -26,13 +26,13 @@ import re class OutputParser: - + def __init__(self): self.matchers = [] - + def addMatcher(self, matcher): self.matchers.append(matcher) - + def parse(self, output): valueMaps = [] for matcher in self.matchers: @@ -47,12 +47,12 @@ given as the group name enclosed in '<' and '>'. """ class ValuesMatcher: - + def __init__(self, regex, valuesTemplate): assert isinstance(valuesTemplate, dict) self.regex = regex self.valuesTemplate = valuesTemplate - + def parse(self, text, valueMaps): for match in self.regex.finditer(text): valueMap = {} @@ -62,10 +62,10 @@ assert not valueMap.has_key(key), key valueMap[key] = value valueMaps.append(valueMap) - + def get_template_value(self, match, template): def replace_var(m): groupName = m.group(1) return match.group(groupName) - + return re.sub(r'<([\w]+)>', replace_var, template) diff -r 8d4e5e08d83f -r dc3c8df55905 mx/sanitycheck.py --- a/mx/sanitycheck.py Tue Sep 03 14:14:09 2013 +0200 +++ b/mx/sanitycheck.py Tue Sep 03 16:33:41 2013 +0200 @@ -27,31 +27,31 @@ import re, mx, commands, os, sys, StringIO, subprocess from os.path import isfile, join, exists -gc='UseSerialGC' +gc = 'UseSerialGC' dacapoSanityWarmup = { - 'avrora': [0, 0, 3, 6, 13], - 'batik': [0, 0, 5, 5, 20], - 'eclipse': [2, 4, 5, 10, 16], + 'avrora': [0, 0, 3, 6, 13], + 'batik': [0, 0, 5, 5, 20], + 'eclipse': [2, 4, 5, 10, 16], 'fop': [4, 8, 10, 20, 30], - 'h2': [0, 0, 5, 5, 8], - 'jython': [0, 0, 5, 10, 13], - 'luindex': [0, 0, 5, 10, 10], - 'lusearch': [0, 4, 5, 5, 8], - 'pmd': [0, 0, 5, 10, 13], - 'sunflow': [0, 2, 5, 10, 15], - 'tomcat': [0, 0, 5, 10, 15], - 'tradebeans': [0, 0, 5, 10, 13], - 'tradesoap': [2, 4, 5, 10, 15], - 'xalan': [0, 0, 5, 10, 18], + 'h2': [0, 0, 5, 5, 8], + 'jython': [0, 0, 5, 10, 13], + 'luindex': [0, 0, 5, 10, 10], + 'lusearch': [0, 4, 5, 5, 8], + 'pmd': [0, 0, 5, 10, 13], + 'sunflow': [0, 2, 5, 10, 15], + 'tomcat': [0, 0, 5, 10, 15], + 'tradebeans': [0, 0, 5, 10, 13], + 'tradesoap': [2, 4, 5, 10, 15], + 'xalan': [0, 0, 5, 10, 18], } dacapoScalaSanityWarmup = { # (tw) actors sometimes fails verification; hardly reproducible - 'actors': [0, 0, 0, 0, 0], + 'actors': [0, 0, 0, 0, 0], # (lstadler) apparat was disabled due to a deadlock which I think is the benchmarks fault. - 'apparat': [0, 0, 0, 0, 0], - 'factorie': [0, 0, 2, 5, 5], + 'apparat': [0, 0, 0, 0, 0], + 'factorie': [0, 0, 2, 5, 5], 'kiama': [0, 0, 3, 13, 15], 'scalac': [0, 0, 5, 15, 20], 'scaladoc': [0, 0, 5, 15, 15], @@ -59,8 +59,8 @@ 'scalariform':[0, 0, 6, 15, 20], 'scalatest': [0, 0, 2, 10, 12], 'scalaxb': [0, 0, 5, 15, 25], -#(gdub) specs sometimes returns a non-zero value event though there is no apparent failure - 'specs': [0, 0, 0, 0, 0], +# (gdub) specs sometimes returns a non-zero value event though there is no apparent failure + 'specs': [0, 0, 0, 0, 0], 'tmt': [0, 0, 3, 10, 12] } @@ -137,60 +137,68 @@ 'xml.validation' ] +def _noneAsEmptyList(a): + if a is None: + return [] + return a + class SanityCheckLevel: Fast, Gate, Normal, Extensive, Benchmark = range(5) - -def getSPECjbb2005(benchArgs = []): - + +def getSPECjbb2005(benchArgs=None): + benchArgs = [] if benchArgs is None else benchArgs + specjbb2005 = mx.get_env('SPECJBB2005') if specjbb2005 is None or not exists(join(specjbb2005, 'jbb.jar')): mx.abort('Please set the SPECJBB2005 environment variable to a SPECjbb2005 directory') - + score = re.compile(r"^Valid run, Score is (?P[0-9]+)$", re.MULTILINE) error = re.compile(r"VALIDATION ERROR") success = re.compile(r"^Valid run, Score is [0-9]+$", re.MULTILINE) matcher = ValuesMatcher(score, {'group' : 'SPECjbb2005', 'name' : 'score', 'score' : ''}) classpath = ['jbb.jar', 'check.jar'] - return Test("SPECjbb2005", ['spec.jbb.JBBmain', '-propfile', 'SPECjbb.props'] + benchArgs, [success], [error], [matcher], vmOpts=['-Xms3g', '-XX:+'+gc, '-XX:-UseCompressedOops', '-cp', os.pathsep.join(classpath)], defaultCwd=specjbb2005) + return Test("SPECjbb2005", ['spec.jbb.JBBmain', '-propfile', 'SPECjbb.props'] + benchArgs, [success], [error], [matcher], vmOpts=['-Xms3g', '-XX:+' + gc, '-XX:-UseCompressedOops', '-cp', os.pathsep.join(classpath)], defaultCwd=specjbb2005) -def getSPECjbb2013(benchArgs = []): - +def getSPECjbb2013(benchArgs=None): + specjbb2013 = mx.get_env('SPECJBB2013') if specjbb2013 is None or not exists(join(specjbb2013, 'specjbb2013.jar')): mx.abort('Please set the SPECJBB2013 environment variable to a SPECjbb2013 directory') - + jops = re.compile(r"^RUN RESULT: hbIR \(max attempted\) = [0-9]+, hbIR \(settled\) = [0-9]+, max-jOPS = (?P[0-9]+), critical-jOPS = (?P[0-9]+)$", re.MULTILINE) - #error? + # error? success = re.compile(r"org.spec.jbb.controller: Run finished", re.MULTILINE) matcherMax = ValuesMatcher(jops, {'group' : 'SPECjbb2013', 'name' : 'max', 'score' : ''}) matcherCritical = ValuesMatcher(jops, {'group' : 'SPECjbb2013', 'name' : 'critical', 'score' : ''}) - return Test("SPECjbb2013", ['-jar', 'specjbb2013.jar', '-m', 'composite'] + benchArgs, [success], [], [matcherCritical, matcherMax], vmOpts=['-Xmx6g', '-Xms6g', '-Xmn3g', '-XX:+UseParallelOldGC', '-XX:-UseAdaptiveSizePolicy', '-XX:-UseBiasedLocking', '-XX:-UseCompressedOops'], defaultCwd=specjbb2013) - -def getSPECjvm2008(benchArgs=[]): - + return Test("SPECjbb2013", ['-jar', 'specjbb2013.jar', '-m', 'composite'] + + _noneAsEmptyList(benchArgs), [success], [], [matcherCritical, matcherMax], + vmOpts=['-Xmx6g', '-Xms6g', '-Xmn3g', '-XX:+UseParallelOldGC', '-XX:-UseAdaptiveSizePolicy', '-XX:-UseBiasedLocking', '-XX:-UseCompressedOops'], defaultCwd=specjbb2013) + +def getSPECjvm2008(benchArgs=None): + specjvm2008 = mx.get_env('SPECJVM2008') if specjvm2008 is None or not exists(join(specjvm2008, 'SPECjvm2008.jar')): mx.abort('Please set the SPECJVM2008 environment variable to a SPECjvm2008 directory') - + score = re.compile(r"^(Score on|Noncompliant) (?P[a-zA-Z0-9\._]+)( result)?: (?P[0-9]+((,|\.)[0-9]+)?)( SPECjvm2008 Base)? ops/m$", re.MULTILINE) error = re.compile(r"^Errors in benchmark: ", re.MULTILINE) # The ' ops/m' at the end of the success string is important : it's how you can tell valid and invalid runs apart success = re.compile(r"^(Noncompliant c|C)omposite result: [0-9]+((,|\.)[0-9]+)?( SPECjvm2008 (Base|Peak))? ops/m$", re.MULTILINE) matcher = ValuesMatcher(score, {'group' : 'SPECjvm2008', 'name' : '', 'score' : ''}) - - return Test("SPECjvm2008", ['-jar', 'SPECjvm2008.jar'] + benchArgs, [success], [error], [matcher], vmOpts=['-Xms3g', '-XX:+'+gc, '-XX:-UseCompressedOops'], defaultCwd=specjvm2008) + + return Test("SPECjvm2008", ['-jar', 'SPECjvm2008.jar'] + _noneAsEmptyList(benchArgs), [success], [error], [matcher], vmOpts=['-Xms3g', '-XX:+' + gc, '-XX:-UseCompressedOops'], defaultCwd=specjvm2008) -def getDacapos(level=SanityCheckLevel.Normal, gateBuildLevel=None, dacapoArgs=[]): +def getDacapos(level=SanityCheckLevel.Normal, gateBuildLevel=None, dacapoArgs=None): checks = [] - + for (bench, ns) in dacapoSanityWarmup.items(): if ns[level] > 0: if gateBuildLevel is None or gateBuildLevel in dacapoGateBuildLevels[bench]: - checks.append(getDacapo(bench, ['-n', str(ns[level])] + dacapoArgs)) - + checks.append(getDacapo(bench, ['-n', str(ns[level])] + _noneAsEmptyList(dacapoArgs))) + return checks -def getDacapo(name, dacapoArgs=[]): +def getDacapo(name, dacapoArgs=None): dacapo = mx.get_env('DACAPO_CP') if dacapo is None: l = mx.library('DACAPO', False) @@ -198,31 +206,31 @@ dacapo = l.get_path(True) else: mx.abort('DaCapo 9.12 jar file must be specified with DACAPO_CP environment variable or as DACAPO library') - + if not isfile(dacapo) or not dacapo.endswith('.jar'): mx.abort('Specified DaCapo jar file does not exist or is not a jar file: ' + dacapo) - + dacapoSuccess = re.compile(r"^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) PASSED in ([0-9]+) msec =====$", re.MULTILINE) dacapoFail = re.compile(r"^===== DaCapo 9\.12 ([a-zA-Z0-9_]+) FAILED (warmup|) =====$", re.MULTILINE) dacapoTime = re.compile(r"===== DaCapo 9\.12 (?P[a-zA-Z0-9_]+) PASSED in (?P