Mercurial > hg > truffle
comparison mxtool/mx.py @ 17159:ef5212ce8091
support for new projects file format including support for automatically updating from old format
author | Doug Simon <doug.simon@oracle.com> |
---|---|
date | Fri, 19 Sep 2014 13:48:53 +0200 |
parents | 2f64a6c771d5 |
children | 30dda118ef3d |
comparison
equal
deleted
inserted
replaced
17158:ae749a34de49 | 17159:ef5212ce8091 |
---|---|
41 import hashlib | 41 import hashlib |
42 import xml.parsers.expat | 42 import xml.parsers.expat |
43 import shutil, re, xml.dom.minidom | 43 import shutil, re, xml.dom.minidom |
44 import pipes | 44 import pipes |
45 import difflib | 45 import difflib |
46 from collections import Callable | 46 from collections import Callable, OrderedDict |
47 from threading import Thread | 47 from threading import Thread |
48 from argparse import ArgumentParser, REMAINDER | 48 from argparse import ArgumentParser, REMAINDER |
49 from os.path import join, basename, dirname, exists, getmtime, isabs, expandvars, isdir, isfile | 49 from os.path import join, basename, dirname, exists, getmtime, isabs, expandvars, isdir, isfile |
50 | 50 |
51 _projects = dict() | 51 _projects = dict() |
785 if abortOnError: | 785 if abortOnError: |
786 abort('failed to get status') | 786 abort('failed to get status') |
787 else: | 787 else: |
788 return None | 788 return None |
789 | 789 |
790 # TODO: remove this function once all repos have transitioned | |
791 # to the new project format | |
792 def _read_projects_file(projectsFile): | |
793 suite = OrderedDict() | |
794 | |
795 suite['projects'] = OrderedDict() | |
796 suite['libraries'] = OrderedDict() | |
797 suite['jrelibraries'] = OrderedDict() | |
798 suite['distributions'] = OrderedDict() | |
799 | |
800 with open(projectsFile) as f: | |
801 prefix = '' | |
802 lineNum = 0 | |
803 | |
804 def error(message): | |
805 abort(projectsFile + ':' + str(lineNum) + ': ' + message) | |
806 | |
807 for line in f: | |
808 lineNum = lineNum + 1 | |
809 line = line.strip() | |
810 if line.endswith('\\'): | |
811 prefix = prefix + line[:-1] | |
812 continue | |
813 if len(prefix) != 0: | |
814 line = prefix + line | |
815 prefix = '' | |
816 if len(line) != 0 and line[0] != '#': | |
817 if '=' not in line: | |
818 error('non-comment line does not contain an "=" character') | |
819 key, value = line.split('=', 1) | |
820 | |
821 parts = key.split('@') | |
822 | |
823 if len(parts) == 1: | |
824 if parts[0] == 'suite': | |
825 suite['name'] = value | |
826 elif parts[0] == 'mxversion': | |
827 suite['mxversion'] = value | |
828 else: | |
829 error('Single part property must be "suite": ' + key) | |
830 | |
831 continue | |
832 if len(parts) != 3: | |
833 error('Property name does not have 3 parts separated by "@": ' + key) | |
834 kind, name, attr = parts | |
835 if kind == 'project': | |
836 m = suite['projects'] | |
837 elif kind == 'library': | |
838 m = suite['libraries'] | |
839 elif kind == 'jrelibrary': | |
840 m = suite['jrelibraries'] | |
841 elif kind == 'distribution': | |
842 m = suite['distributions'] | |
843 else: | |
844 error('Property name does not start with "project@", "library@" or "distribution@": ' + key) | |
845 | |
846 attrs = m.get(name) | |
847 if attrs is None: | |
848 attrs = OrderedDict() | |
849 m[name] = attrs | |
850 value = expandvars_in_property(value) | |
851 attrs[attr] = value | |
852 return suite | |
853 | |
854 # TODO: remove this command once all repos have transitioned | |
855 # to the new project format | |
856 def convertprojects(args, verbose=True): | |
857 """convert old style projects file to projects*.py file(s)""" | |
858 | |
859 class Printer: | |
860 def __init__(self, fp, indent): | |
861 self.fp = fp | |
862 self.indent = indent | |
863 self.prefix = '' | |
864 def println(self, s): | |
865 if len(s) == 0: | |
866 print >> self.fp, s | |
867 else: | |
868 print >> self.fp, self.prefix + s | |
869 def inc(self): | |
870 self.prefix = ''.rjust(len(self.prefix) + self.indent) | |
871 def dec(self): | |
872 self.prefix = ''.rjust(len(self.prefix) - self.indent) | |
873 | |
874 for projectsFile in args: | |
875 suite = _read_projects_file(projectsFile) | |
876 def print_attrs(p, name, attrs, listKeys, is_last=False): | |
877 p.println('"' + name + '" : {') | |
878 p.inc() | |
879 for n, v in attrs.iteritems(): | |
880 if n in listKeys: | |
881 if len(v) == 0: | |
882 p.println('"{}" : [],'.format(n)) | |
883 else: | |
884 p.println('"{}" : ['.format(n)) | |
885 p.inc() | |
886 for e in v.split(','): | |
887 p.println('"' + e.strip() + '",') | |
888 p.dec() | |
889 p.println('],') | |
890 else: | |
891 p.println('"{}" : "{}",'.format(n, v)) | |
892 p.dec() | |
893 if is_last: | |
894 p.println('}') | |
895 else: | |
896 p.println('},') | |
897 p.println('') | |
898 | |
899 def print_section(p, sname, suite, is_last=False): | |
900 section = suite.get(sname) | |
901 if section: | |
902 p.println('"' + sname + '" : {') | |
903 p.inc() | |
904 i = 0 | |
905 for name, attrs in section.iteritems(): | |
906 i = i + 1 | |
907 print_attrs(p, name, attrs, ['urls', 'dependencies', 'sourceUrls'], i == len(section)) | |
908 | |
909 p.dec() | |
910 if is_last: | |
911 p.println('}') | |
912 else: | |
913 p.println('},') | |
914 p.println('') | |
915 | |
916 existing, projectsPyFile = _load_suite_dict(dirname(projectsFile)) | |
917 if existing: | |
918 assert existing['name'] == suite.pop('name') | |
919 assert existing['mxversion'] == suite.pop('mxversion') | |
920 for s in ['projects', 'libraries', 'jrelibraries', 'distributions']: | |
921 for k in existing[s].iterkeys(): | |
922 suite[s].pop(k) | |
923 if len(suite[s]) == 0: | |
924 suite.pop(s) | |
925 | |
926 if len(suite): | |
927 out = StringIO.StringIO() | |
928 p = Printer(out, 2) | |
929 p.println(('extra' if existing else 'suite') + ' = {') | |
930 p.inc() | |
931 if not existing: | |
932 p.println('"mxversion" : "' + suite['mxversion'] + '",') | |
933 p.println('"name" : "' + suite['name'] + '",') | |
934 print_section(p, 'libraries', suite) | |
935 print_section(p, 'jrelibraries', suite) | |
936 print_section(p, 'projects', suite) | |
937 print_section(p, 'distributions', suite, is_last=True) | |
938 p.dec() | |
939 p.println('}') | |
940 | |
941 with open(projectsPyFile, 'w') as fp: | |
942 fp.write(out.getvalue()) | |
943 if verbose: | |
944 print 'created: ' + projectsPyFile | |
945 | |
946 def _load_suite_dict(mxDir): | |
947 | |
948 suffix = 1 | |
949 suite = None | |
950 dictName = 'suite' | |
951 | |
952 moduleName = 'projects' | |
953 modulePath = join(mxDir, moduleName + '.py') | |
954 while exists(modulePath): | |
955 | |
956 savedModule = sys.modules.get(moduleName) | |
957 if savedModule: | |
958 warn(modulePath + ' conflicts with ' + savedModule.__file__) | |
959 # temporarily extend the Python path | |
960 sys.path.insert(0, mxDir) | |
961 | |
962 snapshot = frozenset(sys.modules.viewkeys()) | |
963 module = __import__(moduleName) | |
964 | |
965 if savedModule: | |
966 # restore the old module into the module name space | |
967 sys.modules[moduleName] = savedModule | |
968 else: | |
969 # remove moduleName from the module name space | |
970 sys.modules.pop(moduleName) | |
971 | |
972 # For now fail fast if extra modules were loaded. | |
973 # This can later be relaxed to simply remove the extra modules | |
974 # from the sys.modules name space if necessary. | |
975 extraModules = snapshot - sys.modules.viewkeys() | |
976 assert len(extraModules) == 0, 'loading ' + modulePath + ' caused extra modules to be loaded: ' + ', '.join([m.__file__ for m in extraModules]) | |
977 | |
978 # revert the Python path | |
979 del sys.path[0] | |
980 | |
981 if not hasattr(module, dictName): | |
982 abort(modulePath + ' must define a variable named "' + dictName + '"') | |
983 d = getattr(module, dictName) | |
984 sections = ['projects', 'libraries', 'jrelibraries', 'distributions'] + ([] if suite else ['name', 'mxversion']) | |
985 unknown = d.viewkeys() - sections | |
986 if unknown: | |
987 abort(modulePath + ' defines unsupported suite sections: ' + ', '.join(unknown)) | |
988 | |
989 if suite is None: | |
990 suite = d | |
991 else: | |
992 for s in sections: | |
993 existing = suite.get(s) | |
994 additional = d.get(s) | |
995 if additional: | |
996 if not existing: | |
997 suite[s] = additional | |
998 else: | |
999 conflicting = additional.viewkeys() & existing.viewkeys() | |
1000 if conflicting: | |
1001 abort(modulePath + ' redefines: ' + ', '.join(conflicting)) | |
1002 existing.update(additional) | |
1003 | |
1004 dictName = 'extra' | |
1005 moduleName = 'projects' + str(suffix) | |
1006 modulePath = join(mxDir, moduleName + '.py') | |
1007 suffix = suffix + 1 | |
1008 | |
1009 return suite, modulePath | |
1010 | |
790 class Suite: | 1011 class Suite: |
791 def __init__(self, mxDir, primary, load=True): | 1012 def __init__(self, mxDir, primary, load=True): |
792 self.dir = dirname(mxDir) | 1013 self.dir = dirname(mxDir) |
793 self.mxDir = mxDir | 1014 self.mxDir = mxDir |
794 self.projects = [] | 1015 self.projects = [] |
808 | 1029 |
809 def __str__(self): | 1030 def __str__(self): |
810 return self.name | 1031 return self.name |
811 | 1032 |
812 def _load_projects(self): | 1033 def _load_projects(self): |
813 libsMap = dict() | 1034 # TODO: remove once mx/projects has been deprecated |
814 jreLibsMap = dict() | |
815 projsMap = dict() | |
816 distsMap = dict() | |
817 projectsFile = join(self.mxDir, 'projects') | 1035 projectsFile = join(self.mxDir, 'projects') |
818 if not exists(projectsFile): | 1036 if exists(projectsFile): |
1037 convertprojects([projectsFile], verbose=False) | |
1038 | |
1039 projectsPyFile = join(self.mxDir, 'projects.py') | |
1040 if not exists(projectsPyFile): | |
819 return | 1041 return |
820 | 1042 |
821 with open(projectsFile) as f: | 1043 suiteDict, _ = _load_suite_dict(self.mxDir) |
822 prefix = '' | 1044 |
823 lineNum = 0 | 1045 if suiteDict.get('name') is not None and suiteDict.get('name') != self.name: |
824 | 1046 abort('suite name in project file does not match ' + _suitename(self.mxDir)) |
825 def error(message): | 1047 |
826 abort(projectsFile + ':' + str(lineNum) + ': ' + message) | 1048 if suiteDict.has_key('mxversion'): |
827 | 1049 try: |
828 for line in f: | 1050 self.requiredMxVersion = VersionSpec(suiteDict['mxversion']) |
829 lineNum = lineNum + 1 | 1051 except AssertionError as ae: |
830 line = line.strip() | 1052 abort('Exception while parsing "mxversion" in project file: ' + str(ae)) |
831 if line.endswith('\\'): | 1053 |
832 prefix = prefix + line[:-1] | 1054 libsMap = suiteDict['libraries'] |
833 continue | 1055 jreLibsMap = suiteDict['jrelibraries'] |
834 if len(prefix) != 0: | 1056 projsMap = suiteDict['projects'] |
835 line = prefix + line | 1057 distsMap = suiteDict['distributions'] |
836 prefix = '' | |
837 if len(line) != 0 and line[0] != '#': | |
838 if '=' not in line: | |
839 error('non-comment line does not contain an "=" character') | |
840 key, value = line.split('=', 1) | |
841 | |
842 parts = key.split('@') | |
843 | |
844 if len(parts) == 1: | |
845 if parts[0] == 'suite': | |
846 if self.name != value: | |
847 error('suite name in project file does not match ' + _suitename(self.mxDir)) | |
848 elif parts[0] == 'mxversion': | |
849 try: | |
850 self.requiredMxVersion = VersionSpec(value) | |
851 except AssertionError as ae: | |
852 error('Exception while parsing "mxversion" in project file: ' + str(ae)) | |
853 else: | |
854 error('Single part property must be "suite": ' + key) | |
855 | |
856 continue | |
857 if len(parts) != 3: | |
858 error('Property name does not have 3 parts separated by "@": ' + key) | |
859 kind, name, attr = parts | |
860 if kind == 'project': | |
861 m = projsMap | |
862 elif kind == 'library': | |
863 m = libsMap | |
864 elif kind == 'jrelibrary': | |
865 m = jreLibsMap | |
866 elif kind == 'distribution': | |
867 m = distsMap | |
868 else: | |
869 error('Property name does not start with "project@", "library@" or "distribution@": ' + key) | |
870 | |
871 attrs = m.get(name) | |
872 if attrs is None: | |
873 attrs = dict() | |
874 m[name] = attrs | |
875 value = expandvars_in_property(value) | |
876 attrs[attr] = value | |
877 | 1058 |
878 def pop_list(attrs, name): | 1059 def pop_list(attrs, name): |
879 v = attrs.pop(name, None) | 1060 v = attrs.pop(name, None) |
1061 if isinstance(v, list): | |
1062 return v | |
880 if v is None or len(v.strip()) == 0: | 1063 if v is None or len(v.strip()) == 0: |
881 return [] | 1064 return [] |
882 return [n.strip() for n in v.split(',')] | 1065 return [n.strip() for n in v.split(',')] |
883 | 1066 |
884 for name, attrs in projsMap.iteritems(): | 1067 for name, attrs in projsMap.iteritems(): |
894 workingSets = attrs.pop('workingSets', None) | 1077 workingSets = attrs.pop('workingSets', None) |
895 p = Project(self, name, srcDirs, deps, javaCompliance, workingSets, d) | 1078 p = Project(self, name, srcDirs, deps, javaCompliance, workingSets, d) |
896 p.checkstyleProj = attrs.pop('checkstyle', name) | 1079 p.checkstyleProj = attrs.pop('checkstyle', name) |
897 p.native = attrs.pop('native', '') == 'true' | 1080 p.native = attrs.pop('native', '') == 'true' |
898 if not p.native and p.javaCompliance is None: | 1081 if not p.native and p.javaCompliance is None: |
899 error('javaCompliance property required for non-native project ' + name) | 1082 abort('javaCompliance property required for non-native project ' + name) |
900 if len(ap) > 0: | 1083 if len(ap) > 0: |
901 p._declaredAnnotationProcessors = ap | 1084 p._declaredAnnotationProcessors = ap |
902 p.__dict__.update(attrs) | 1085 p.__dict__.update(attrs) |
903 self.projects.append(p) | 1086 self.projects.append(p) |
904 | 1087 |
909 l = JreLibrary(self, name, jar, optional) | 1092 l = JreLibrary(self, name, jar, optional) |
910 self.jreLibs.append(l) | 1093 self.jreLibs.append(l) |
911 | 1094 |
912 for name, attrs in libsMap.iteritems(): | 1095 for name, attrs in libsMap.iteritems(): |
913 if "|" in name: | 1096 if "|" in name: |
914 assert name.count("|") == 2, "syntax: libname|os-platform|architecture" | 1097 if name.count('|') != 2: |
1098 abort("Format error in library name: " + name + "\nsyntax: libname|os-platform|architecture") | |
915 name, platform, architecture = name.split("|") | 1099 name, platform, architecture = name.split("|") |
916 if platform != get_os() or architecture != get_arch(): | 1100 if platform != get_os() or architecture != get_arch(): |
917 continue | 1101 continue |
918 path = attrs.pop('path') | 1102 path = attrs.pop('path') |
919 urls = pop_list(attrs, 'urls') | 1103 urls = pop_list(attrs, 'urls') |
2864 | 3048 |
2865 logv("generated archives: " + str(archives)) | 3049 logv("generated archives: " + str(archives)) |
2866 return archives | 3050 return archives |
2867 | 3051 |
2868 def canonicalizeprojects(args): | 3052 def canonicalizeprojects(args): |
2869 """process all project files to canonicalize the dependencies | 3053 """check all project specifications for canonical dependencies |
2870 | 3054 |
2871 The exit code of this command reflects how many files were updated.""" | 3055 The exit code of this command reflects how many projects have non-canonical dependencies.""" |
2872 | 3056 |
2873 changedFiles = 0 | 3057 nonCanonical = [] |
2874 for s in suites(True): | 3058 for s in suites(True): |
2875 projectsFile = join(s.mxDir, 'projects') | 3059 projectsPyFile = join(s.mxDir, 'projects') |
2876 if not exists(projectsFile): | 3060 if not exists(projectsPyFile): |
2877 continue | 3061 continue |
2878 with open(projectsFile) as f: | 3062 |
2879 out = StringIO.StringIO() | 3063 for p in s.projects: |
2880 pattern = re.compile('project@([^@]+)@dependencies=.*') | 3064 for pkg in p.defined_java_packages(): |
2881 lineNo = 1 | 3065 if not pkg.startswith(p.name): |
2882 for line in f: | 3066 abort('package in {0} does not have prefix matching project name: {1}'.format(p, pkg)) |
2883 line = line.strip() | 3067 |
2884 m = pattern.match(line) | 3068 ignoredDeps = set([name for name in p.deps if project(name, False) is not None]) |
2885 p = project(m.group(1), fatalIfMissing=False) if m else None | 3069 for pkg in p.imported_java_packages(): |
2886 if m is None or p is None: | 3070 for name in p.deps: |
2887 out.write(line + '\n') | 3071 dep = project(name, False) |
2888 else: | 3072 if dep is None: |
2889 for pkg in p.defined_java_packages(): | 3073 ignoredDeps.discard(name) |
2890 if not pkg.startswith(p.name): | 3074 else: |
2891 abort('package in {0} does not have prefix matching project name: {1}'.format(p, pkg)) | 3075 if pkg in dep.defined_java_packages(): |
2892 | 3076 ignoredDeps.discard(name) |
2893 ignoredDeps = set([name for name in p.deps if project(name, False) is not None]) | 3077 if pkg in dep.extended_java_packages(): |
2894 for pkg in p.imported_java_packages(): | 3078 ignoredDeps.discard(name) |
2895 for name in p.deps: | 3079 if len(ignoredDeps) != 0: |
2896 dep = project(name, False) | 3080 candidates = set() |
2897 if dep is None: | 3081 # Compute dependencies based on projects required by p |
2898 ignoredDeps.discard(name) | 3082 for d in sorted_deps(): |
2899 else: | 3083 if not d.defined_java_packages().isdisjoint(p.imported_java_packages()): |
2900 if pkg in dep.defined_java_packages(): | 3084 candidates.add(d) |
2901 ignoredDeps.discard(name) | 3085 # Remove non-canonical candidates |
2902 if pkg in dep.extended_java_packages(): | 3086 for c in list(candidates): |
2903 ignoredDeps.discard(name) | 3087 candidates.difference_update(c.all_deps([], False, False)) |
2904 if len(ignoredDeps) != 0: | 3088 candidates = [d.name for d in candidates] |
2905 candidates = set() | 3089 |
2906 # Compute dependencies based on projects required by p | 3090 abort('{} does not use any packages defined in these projects: {}\nComputed project dependencies: {}'.format( |
2907 for d in sorted_deps(): | 3091 p, ', '.join(ignoredDeps), ','.join(candidates))) |
2908 if not d.defined_java_packages().isdisjoint(p.imported_java_packages()): | 3092 |
2909 candidates.add(d) | 3093 excess = frozenset(p.deps) - set(p.canonical_deps()) |
2910 # Remove non-canonical candidates | 3094 if len(excess) != 0: |
2911 for c in list(candidates): | 3095 nonCanonical.append(p) |
2912 candidates.difference_update(c.all_deps([], False, False)) | 3096 if len(nonCanonical) != 0: |
2913 candidates = [d.name for d in candidates] | 3097 for p in nonCanonical: |
2914 | 3098 canonicalDeps = p.canonical_deps() |
2915 abort('{0}:{1}: {2} does not use any packages defined in these projects: {3}\nComputed project dependencies: {4}'.format( | 3099 if len(canonicalDeps) != 0: |
2916 projectsFile, lineNo, p, ', '.join(ignoredDeps), ','.join(candidates))) | 3100 log('Canonical dependencies for project ' + p.name + ' are: [') |
2917 | 3101 for d in canonicalDeps: |
2918 out.write('project@' + m.group(1) + '@dependencies=' + ','.join(p.canonical_deps()) + '\n') | 3102 log(' "' + d + '",') |
2919 lineNo = lineNo + 1 | 3103 log(' ],') |
2920 content = out.getvalue() | 3104 else: |
2921 if update_file(projectsFile, content): | 3105 log('Canonical dependencies for project ' + p.name + ' are: []') |
2922 changedFiles += 1 | 3106 return len(nonCanonical) |
2923 return changedFiles | |
2924 | 3107 |
2925 class TimeStampFile: | 3108 class TimeStampFile: |
2926 def __init__(self, path): | 3109 def __init__(self, path): |
2927 self.path = path | 3110 self.path = path |
2928 self.timestamp = os.path.getmtime(path) if exists(path) else None | 3111 self.timestamp = os.path.getmtime(path) if exists(path) else None |
4950 # Suite extensions should not update this table directly, but use update_commands | 5133 # Suite extensions should not update this table directly, but use update_commands |
4951 _commands = { | 5134 _commands = { |
4952 'about': [about, ''], | 5135 'about': [about, ''], |
4953 'build': [build, '[options]'], | 5136 'build': [build, '[options]'], |
4954 'checkstyle': [checkstyle, ''], | 5137 'checkstyle': [checkstyle, ''], |
5138 'convertprojects' : [convertprojects, ''], | |
4955 'canonicalizeprojects': [canonicalizeprojects, ''], | 5139 'canonicalizeprojects': [canonicalizeprojects, ''], |
4956 'clean': [clean, ''], | 5140 'clean': [clean, ''], |
4957 'eclipseinit': [eclipseinit, ''], | 5141 'eclipseinit': [eclipseinit, ''], |
4958 'eclipseformat': [eclipseformat, ''], | 5142 'eclipseformat': [eclipseformat, ''], |
4959 'exportlibs': [exportlibs, ''], | 5143 'exportlibs': [exportlibs, ''], |