Mercurial > hg > graal-compiler

comparison mxtool/mx.py @ 17159:ef5212ce8091

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
support for new projects file format including support for automatically updating from old format
author Doug Simon <doug.simon@oracle.com>
date Fri, 19 Sep 2014 13:48:53 +0200
parents 2f64a6c771d5
children 30dda118ef3d
comparison
equal deleted inserted replaced
17158:ae749a34de49 17159:ef5212ce8091
41 import hashlib 41 import hashlib
42 import xml.parsers.expat 42 import xml.parsers.expat
43 import shutil, re, xml.dom.minidom 43 import shutil, re, xml.dom.minidom
44 import pipes 44 import pipes
45 import difflib 45 import difflib
46 from collections import Callable 46 from collections import Callable, OrderedDict
47 from threading import Thread 47 from threading import Thread
48 from argparse import ArgumentParser, REMAINDER 48 from argparse import ArgumentParser, REMAINDER
49 from os.path import join, basename, dirname, exists, getmtime, isabs, expandvars, isdir, isfile 49 from os.path import join, basename, dirname, exists, getmtime, isabs, expandvars, isdir, isfile
50 50
51 _projects = dict() 51 _projects = dict()
785 if abortOnError: 785 if abortOnError:
786 abort('failed to get status') 786 abort('failed to get status')
787 else: 787 else:
788 return None 788 return None
789 789
790 # TODO: remove this function once all repos have transitioned
791 # to the new project format
792 def _read_projects_file(projectsFile):
793 suite = OrderedDict()
794
795 suite['projects'] = OrderedDict()
796 suite['libraries'] = OrderedDict()
797 suite['jrelibraries'] = OrderedDict()
798 suite['distributions'] = OrderedDict()
799
800 with open(projectsFile) as f:
801 prefix = ''
802 lineNum = 0
803
804 def error(message):
805 abort(projectsFile + ':' + str(lineNum) + ': ' + message)
806
807 for line in f:
808 lineNum = lineNum + 1
809 line = line.strip()
810 if line.endswith('\\'):
811 prefix = prefix + line[:-1]
812 continue
813 if len(prefix) != 0:
814 line = prefix + line
815 prefix = ''
816 if len(line) != 0 and line[0] != '#':
817 if '=' not in line:
818 error('non-comment line does not contain an "=" character')
819 key, value = line.split('=', 1)
820
821 parts = key.split('@')
822
823 if len(parts) == 1:
824 if parts[0] == 'suite':
825 suite['name'] = value
826 elif parts[0] == 'mxversion':
827 suite['mxversion'] = value
828 else:
829 error('Single part property must be "suite": ' + key)
830
831 continue
832 if len(parts) != 3:
833 error('Property name does not have 3 parts separated by "@": ' + key)
834 kind, name, attr = parts
835 if kind == 'project':
836 m = suite['projects']
837 elif kind == 'library':
838 m = suite['libraries']
839 elif kind == 'jrelibrary':
840 m = suite['jrelibraries']
841 elif kind == 'distribution':
842 m = suite['distributions']
843 else:
844 error('Property name does not start with "project@", "library@" or "distribution@": ' + key)
845
846 attrs = m.get(name)
847 if attrs is None:
848 attrs = OrderedDict()
849 m[name] = attrs
850 value = expandvars_in_property(value)
851 attrs[attr] = value
852 return suite
853
854 # TODO: remove this command once all repos have transitioned
855 # to the new project format
856 def convertprojects(args, verbose=True):
857 """convert old style projects file to projects*.py file(s)"""
858
859 class Printer:
860 def __init__(self, fp, indent):
861 self.fp = fp
862 self.indent = indent
863 self.prefix = ''
864 def println(self, s):
865 if len(s) == 0:
866 print >> self.fp, s
867 else:
868 print >> self.fp, self.prefix + s
869 def inc(self):
870 self.prefix = ''.rjust(len(self.prefix) + self.indent)
871 def dec(self):
872 self.prefix = ''.rjust(len(self.prefix) - self.indent)
873
874 for projectsFile in args:
875 suite = _read_projects_file(projectsFile)
876 def print_attrs(p, name, attrs, listKeys, is_last=False):
877 p.println('"' + name + '" : {')
878 p.inc()
879 for n, v in attrs.iteritems():
880 if n in listKeys:
881 if len(v) == 0:
882 p.println('"{}" : [],'.format(n))
883 else:
884 p.println('"{}" : ['.format(n))
885 p.inc()
886 for e in v.split(','):
887 p.println('"' + e.strip() + '",')
888 p.dec()
889 p.println('],')
890 else:
891 p.println('"{}" : "{}",'.format(n, v))
892 p.dec()
893 if is_last:
894 p.println('}')
895 else:
896 p.println('},')
897 p.println('')
898
899 def print_section(p, sname, suite, is_last=False):
900 section = suite.get(sname)
901 if section:
902 p.println('"' + sname + '" : {')
903 p.inc()
904 i = 0
905 for name, attrs in section.iteritems():
906 i = i + 1
907 print_attrs(p, name, attrs, ['urls', 'dependencies', 'sourceUrls'], i == len(section))
908
909 p.dec()
910 if is_last:
911 p.println('}')
912 else:
913 p.println('},')
914 p.println('')
915
916 existing, projectsPyFile = _load_suite_dict(dirname(projectsFile))
917 if existing:
918 assert existing['name'] == suite.pop('name')
919 assert existing['mxversion'] == suite.pop('mxversion')
920 for s in ['projects', 'libraries', 'jrelibraries', 'distributions']:
921 for k in existing[s].iterkeys():
922 suite[s].pop(k)
923 if len(suite[s]) == 0:
924 suite.pop(s)
925
926 if len(suite):
927 out = StringIO.StringIO()
928 p = Printer(out, 2)
929 p.println(('extra' if existing else 'suite') + ' = {')
930 p.inc()
931 if not existing:
932 p.println('"mxversion" : "' + suite['mxversion'] + '",')
933 p.println('"name" : "' + suite['name'] + '",')
934 print_section(p, 'libraries', suite)
935 print_section(p, 'jrelibraries', suite)
936 print_section(p, 'projects', suite)
937 print_section(p, 'distributions', suite, is_last=True)
938 p.dec()
939 p.println('}')
940
941 with open(projectsPyFile, 'w') as fp:
942 fp.write(out.getvalue())
943 if verbose:
944 print 'created: ' + projectsPyFile
945
946 def _load_suite_dict(mxDir):
947
948 suffix = 1
949 suite = None
950 dictName = 'suite'
951
952 moduleName = 'projects'
953 modulePath = join(mxDir, moduleName + '.py')
954 while exists(modulePath):
955
956 savedModule = sys.modules.get(moduleName)
957 if savedModule:
958 warn(modulePath + ' conflicts with ' + savedModule.__file__)
959 # temporarily extend the Python path
960 sys.path.insert(0, mxDir)
961
962 snapshot = frozenset(sys.modules.viewkeys())
963 module = __import__(moduleName)
964
965 if savedModule:
966 # restore the old module into the module name space
967 sys.modules[moduleName] = savedModule
968 else:
969 # remove moduleName from the module name space
970 sys.modules.pop(moduleName)
971
972 # For now fail fast if extra modules were loaded.
973 # This can later be relaxed to simply remove the extra modules
974 # from the sys.modules name space if necessary.
975 extraModules = snapshot - sys.modules.viewkeys()
976 assert len(extraModules) == 0, 'loading ' + modulePath + ' caused extra modules to be loaded: ' + ', '.join([m.__file__ for m in extraModules])
977
978 # revert the Python path
979 del sys.path[0]
980
981 if not hasattr(module, dictName):
982 abort(modulePath + ' must define a variable named "' + dictName + '"')
983 d = getattr(module, dictName)
984 sections = ['projects', 'libraries', 'jrelibraries', 'distributions'] + ([] if suite else ['name', 'mxversion'])
985 unknown = d.viewkeys() - sections
986 if unknown:
987 abort(modulePath + ' defines unsupported suite sections: ' + ', '.join(unknown))
988
989 if suite is None:
990 suite = d
991 else:
992 for s in sections:
993 existing = suite.get(s)
994 additional = d.get(s)
995 if additional:
996 if not existing:
997 suite[s] = additional
998 else:
999 conflicting = additional.viewkeys() & existing.viewkeys()
1000 if conflicting:
1001 abort(modulePath + ' redefines: ' + ', '.join(conflicting))
1002 existing.update(additional)
1003
1004 dictName = 'extra'
1005 moduleName = 'projects' + str(suffix)
1006 modulePath = join(mxDir, moduleName + '.py')
1007 suffix = suffix + 1
1008
1009 return suite, modulePath
1010
790 class Suite: 1011 class Suite:
791 def __init__(self, mxDir, primary, load=True): 1012 def __init__(self, mxDir, primary, load=True):
792 self.dir = dirname(mxDir) 1013 self.dir = dirname(mxDir)
793 self.mxDir = mxDir 1014 self.mxDir = mxDir
794 self.projects = [] 1015 self.projects = []
808 1029
809 def __str__(self): 1030 def __str__(self):
810 return self.name 1031 return self.name
811 1032
812 def _load_projects(self): 1033 def _load_projects(self):
813 libsMap = dict() 1034 # TODO: remove once mx/projects has been deprecated
814 jreLibsMap = dict()
815 projsMap = dict()
816 distsMap = dict()
817 projectsFile = join(self.mxDir, 'projects') 1035 projectsFile = join(self.mxDir, 'projects')
818 if not exists(projectsFile): 1036 if exists(projectsFile):
1037 convertprojects([projectsFile], verbose=False)
1038
1039 projectsPyFile = join(self.mxDir, 'projects.py')
1040 if not exists(projectsPyFile):
819 return 1041 return
820 1042
821 with open(projectsFile) as f: 1043 suiteDict, _ = _load_suite_dict(self.mxDir)
822 prefix = '' 1044
823 lineNum = 0 1045 if suiteDict.get('name') is not None and suiteDict.get('name') != self.name:
824 1046 abort('suite name in project file does not match ' + _suitename(self.mxDir))
825 def error(message): 1047
826 abort(projectsFile + ':' + str(lineNum) + ': ' + message) 1048 if suiteDict.has_key('mxversion'):
827 1049 try:
828 for line in f: 1050 self.requiredMxVersion = VersionSpec(suiteDict['mxversion'])
829 lineNum = lineNum + 1 1051 except AssertionError as ae:
830 line = line.strip() 1052 abort('Exception while parsing "mxversion" in project file: ' + str(ae))
831 if line.endswith('\\'): 1053
832 prefix = prefix + line[:-1] 1054 libsMap = suiteDict['libraries']
833 continue 1055 jreLibsMap = suiteDict['jrelibraries']
834 if len(prefix) != 0: 1056 projsMap = suiteDict['projects']
835 line = prefix + line 1057 distsMap = suiteDict['distributions']
836 prefix = ''
837 if len(line) != 0 and line[0] != '#':
838 if '=' not in line:
839 error('non-comment line does not contain an "=" character')
840 key, value = line.split('=', 1)
841
842 parts = key.split('@')
843
844 if len(parts) == 1:
845 if parts[0] == 'suite':
846 if self.name != value:
847 error('suite name in project file does not match ' + _suitename(self.mxDir))
848 elif parts[0] == 'mxversion':
849 try:
850 self.requiredMxVersion = VersionSpec(value)
851 except AssertionError as ae:
852 error('Exception while parsing "mxversion" in project file: ' + str(ae))
853 else:
854 error('Single part property must be "suite": ' + key)
855
856 continue
857 if len(parts) != 3:
858 error('Property name does not have 3 parts separated by "@": ' + key)
859 kind, name, attr = parts
860 if kind == 'project':
861 m = projsMap
862 elif kind == 'library':
863 m = libsMap
864 elif kind == 'jrelibrary':
865 m = jreLibsMap
866 elif kind == 'distribution':
867 m = distsMap
868 else:
869 error('Property name does not start with "project@", "library@" or "distribution@": ' + key)
870
871 attrs = m.get(name)
872 if attrs is None:
873 attrs = dict()
874 m[name] = attrs
875 value = expandvars_in_property(value)
876 attrs[attr] = value
877 1058
878 def pop_list(attrs, name): 1059 def pop_list(attrs, name):
879 v = attrs.pop(name, None) 1060 v = attrs.pop(name, None)
1061 if isinstance(v, list):
1062 return v
880 if v is None or len(v.strip()) == 0: 1063 if v is None or len(v.strip()) == 0:
881 return [] 1064 return []
882 return [n.strip() for n in v.split(',')] 1065 return [n.strip() for n in v.split(',')]
883 1066
884 for name, attrs in projsMap.iteritems(): 1067 for name, attrs in projsMap.iteritems():
894 workingSets = attrs.pop('workingSets', None) 1077 workingSets = attrs.pop('workingSets', None)
895 p = Project(self, name, srcDirs, deps, javaCompliance, workingSets, d) 1078 p = Project(self, name, srcDirs, deps, javaCompliance, workingSets, d)
896 p.checkstyleProj = attrs.pop('checkstyle', name) 1079 p.checkstyleProj = attrs.pop('checkstyle', name)
897 p.native = attrs.pop('native', '') == 'true' 1080 p.native = attrs.pop('native', '') == 'true'
898 if not p.native and p.javaCompliance is None: 1081 if not p.native and p.javaCompliance is None:
899 error('javaCompliance property required for non-native project ' + name) 1082 abort('javaCompliance property required for non-native project ' + name)
900 if len(ap) > 0: 1083 if len(ap) > 0:
901 p._declaredAnnotationProcessors = ap 1084 p._declaredAnnotationProcessors = ap
902 p.__dict__.update(attrs) 1085 p.__dict__.update(attrs)
903 self.projects.append(p) 1086 self.projects.append(p)
904 1087
909 l = JreLibrary(self, name, jar, optional) 1092 l = JreLibrary(self, name, jar, optional)
910 self.jreLibs.append(l) 1093 self.jreLibs.append(l)
911 1094
912 for name, attrs in libsMap.iteritems(): 1095 for name, attrs in libsMap.iteritems():
913 if "|" in name: 1096 if "|" in name:
914 assert name.count("|") == 2, "syntax: libname|os-platform|architecture" 1097 if name.count('|') != 2:
1098 abort("Format error in library name: " + name + "\nsyntax: libname|os-platform|architecture")
915 name, platform, architecture = name.split("|") 1099 name, platform, architecture = name.split("|")
916 if platform != get_os() or architecture != get_arch(): 1100 if platform != get_os() or architecture != get_arch():
917 continue 1101 continue
918 path = attrs.pop('path') 1102 path = attrs.pop('path')
919 urls = pop_list(attrs, 'urls') 1103 urls = pop_list(attrs, 'urls')
2864 3048
2865 logv("generated archives: " + str(archives)) 3049 logv("generated archives: " + str(archives))
2866 return archives 3050 return archives
2867 3051
2868 def canonicalizeprojects(args): 3052 def canonicalizeprojects(args):
2869 """process all project files to canonicalize the dependencies 3053 """check all project specifications for canonical dependencies
2870 3054
2871 The exit code of this command reflects how many files were updated.""" 3055 The exit code of this command reflects how many projects have non-canonical dependencies."""
2872 3056
2873 changedFiles = 0 3057 nonCanonical = []
2874 for s in suites(True): 3058 for s in suites(True):
2875 projectsFile = join(s.mxDir, 'projects') 3059 projectsPyFile = join(s.mxDir, 'projects')
2876 if not exists(projectsFile): 3060 if not exists(projectsPyFile):
2877 continue 3061 continue
2878 with open(projectsFile) as f: 3062
2879 out = StringIO.StringIO() 3063 for p in s.projects:
2880 pattern = re.compile('project@([^@]+)@dependencies=.*') 3064 for pkg in p.defined_java_packages():
2881 lineNo = 1 3065 if not pkg.startswith(p.name):
2882 for line in f: 3066 abort('package in {0} does not have prefix matching project name: {1}'.format(p, pkg))
2883 line = line.strip() 3067
2884 m = pattern.match(line) 3068 ignoredDeps = set([name for name in p.deps if project(name, False) is not None])
2885 p = project(m.group(1), fatalIfMissing=False) if m else None 3069 for pkg in p.imported_java_packages():
2886 if m is None or p is None: 3070 for name in p.deps:
2887 out.write(line + '\n') 3071 dep = project(name, False)
2888 else: 3072 if dep is None:
2889 for pkg in p.defined_java_packages(): 3073 ignoredDeps.discard(name)
2890 if not pkg.startswith(p.name): 3074 else:
2891 abort('package in {0} does not have prefix matching project name: {1}'.format(p, pkg)) 3075 if pkg in dep.defined_java_packages():
2892 3076 ignoredDeps.discard(name)
2893 ignoredDeps = set([name for name in p.deps if project(name, False) is not None]) 3077 if pkg in dep.extended_java_packages():
2894 for pkg in p.imported_java_packages(): 3078 ignoredDeps.discard(name)
2895 for name in p.deps: 3079 if len(ignoredDeps) != 0:
2896 dep = project(name, False) 3080 candidates = set()
2897 if dep is None: 3081 # Compute dependencies based on projects required by p
2898 ignoredDeps.discard(name) 3082 for d in sorted_deps():
2899 else: 3083 if not d.defined_java_packages().isdisjoint(p.imported_java_packages()):
2900 if pkg in dep.defined_java_packages(): 3084 candidates.add(d)
2901 ignoredDeps.discard(name) 3085 # Remove non-canonical candidates
2902 if pkg in dep.extended_java_packages(): 3086 for c in list(candidates):
2903 ignoredDeps.discard(name) 3087 candidates.difference_update(c.all_deps([], False, False))
2904 if len(ignoredDeps) != 0: 3088 candidates = [d.name for d in candidates]
2905 candidates = set() 3089
2906 # Compute dependencies based on projects required by p 3090 abort('{} does not use any packages defined in these projects: {}\nComputed project dependencies: {}'.format(
2907 for d in sorted_deps(): 3091 p, ', '.join(ignoredDeps), ','.join(candidates)))
2908 if not d.defined_java_packages().isdisjoint(p.imported_java_packages()): 3092
2909 candidates.add(d) 3093 excess = frozenset(p.deps) - set(p.canonical_deps())
2910 # Remove non-canonical candidates 3094 if len(excess) != 0:
2911 for c in list(candidates): 3095 nonCanonical.append(p)
2912 candidates.difference_update(c.all_deps([], False, False)) 3096 if len(nonCanonical) != 0:
2913 candidates = [d.name for d in candidates] 3097 for p in nonCanonical:
2914 3098 canonicalDeps = p.canonical_deps()
2915 abort('{0}:{1}: {2} does not use any packages defined in these projects: {3}\nComputed project dependencies: {4}'.format( 3099 if len(canonicalDeps) != 0:
2916 projectsFile, lineNo, p, ', '.join(ignoredDeps), ','.join(candidates))) 3100 log('Canonical dependencies for project ' + p.name + ' are: [')
2917 3101 for d in canonicalDeps:
2918 out.write('project@' + m.group(1) + '@dependencies=' + ','.join(p.canonical_deps()) + '\n') 3102 log(' "' + d + '",')
2919 lineNo = lineNo + 1 3103 log(' ],')
2920 content = out.getvalue() 3104 else:
2921 if update_file(projectsFile, content): 3105 log('Canonical dependencies for project ' + p.name + ' are: []')
2922 changedFiles += 1 3106 return len(nonCanonical)
2923 return changedFiles
2924 3107
2925 class TimeStampFile: 3108 class TimeStampFile:
2926 def __init__(self, path): 3109 def __init__(self, path):
2927 self.path = path 3110 self.path = path
2928 self.timestamp = os.path.getmtime(path) if exists(path) else None 3111 self.timestamp = os.path.getmtime(path) if exists(path) else None
4950 # Suite extensions should not update this table directly, but use update_commands 5133 # Suite extensions should not update this table directly, but use update_commands
4951 _commands = { 5134 _commands = {
4952 'about': [about, ''], 5135 'about': [about, ''],
4953 'build': [build, '[options]'], 5136 'build': [build, '[options]'],
4954 'checkstyle': [checkstyle, ''], 5137 'checkstyle': [checkstyle, ''],
5138 'convertprojects' : [convertprojects, ''],
4955 'canonicalizeprojects': [canonicalizeprojects, ''], 5139 'canonicalizeprojects': [canonicalizeprojects, ''],
4956 'clean': [clean, ''], 5140 'clean': [clean, ''],
4957 'eclipseinit': [eclipseinit, ''], 5141 'eclipseinit': [eclipseinit, ''],
4958 'eclipseformat': [eclipseformat, ''], 5142 'eclipseformat': [eclipseformat, ''],
4959 'exportlibs': [exportlibs, ''], 5143 'exportlibs': [exportlibs, ''],