Mercurial > hg > truffle
comparison src/share/vm/opto/superword.cpp @ 667:78af5ae8e731
6636138: UseSuperWord enabled failure
Summary: Fixed SuperWord scheduling of memory operations.
Reviewed-by: kvn, never
author | cfang |
---|---|
date | Tue, 24 Mar 2009 12:19:47 -0700 |
parents | 7bb995fbd3c0 |
children | ace8397c8563 |
comparison
equal
deleted
inserted
replaced
666:ebebd376f657 | 667:78af5ae8e731 |
---|---|
452 if (out->is_MergeMem() && !in_bb(out)) { | 452 if (out->is_MergeMem() && !in_bb(out)) { |
453 // Either unrolling is causing a memory edge not to disappear, | 453 // Either unrolling is causing a memory edge not to disappear, |
454 // or need to run igvn.optimize() again before SLP | 454 // or need to run igvn.optimize() again before SLP |
455 } else if (out->is_Phi() && out->bottom_type() == Type::MEMORY && !in_bb(out)) { | 455 } else if (out->is_Phi() && out->bottom_type() == Type::MEMORY && !in_bb(out)) { |
456 // Ditto. Not sure what else to check further. | 456 // Ditto. Not sure what else to check further. |
457 } else if (out->Opcode() == Op_StoreCM && out->in(4) == n) { | 457 } else if (out->Opcode() == Op_StoreCM && out->in(MemNode::OopStore) == n) { |
458 // StoreCM has an input edge used as a precedence edge. | 458 // StoreCM has an input edge used as a precedence edge. |
459 // Maybe an issue when oop stores are vectorized. | 459 // Maybe an issue when oop stores are vectorized. |
460 } else if( out->is_MergeMem() && prev && | |
461 prev->Opcode() == Op_StoreCM && out == prev->in(MemNode::OopStore)) { | |
462 // Oop store is a MergeMem! This should not happen. Temporarily remove the assertion | |
463 // for this case because it could not be superwordized anyway. | |
460 } else { | 464 } else { |
461 assert(out == prev || prev == NULL, "no branches off of store slice"); | 465 assert(out == prev || prev == NULL, "no branches off of store slice"); |
462 } | 466 } |
463 } | 467 } |
464 } | 468 } |
910 for (int i = 0; i < _packset.length(); i++) { | 914 for (int i = 0; i < _packset.length(); i++) { |
911 co_locate_pack(_packset.at(i)); | 915 co_locate_pack(_packset.at(i)); |
912 } | 916 } |
913 } | 917 } |
914 | 918 |
915 //------------------------------co_locate_pack--------------------------- | 919 //-------------------------------remove_and_insert------------------- |
916 // Within a pack, move stores down to the last executed store, | 920 //remove "current" from its current position in the memory graph and insert |
917 // and move loads up to the first executed load. | 921 //it after the appropriate insertion point (lip or uip) |
922 void SuperWord::remove_and_insert(MemNode *current, MemNode *prev, MemNode *lip, | |
923 Node *uip, Unique_Node_List &sched_before) { | |
924 Node* my_mem = current->in(MemNode::Memory); | |
925 _igvn.hash_delete(current); | |
926 _igvn.hash_delete(my_mem); | |
927 | |
928 //remove current_store from its current position in the memmory graph | |
929 for (DUIterator i = current->outs(); current->has_out(i); i++) { | |
930 Node* use = current->out(i); | |
931 if (use->is_Mem()) { | |
932 assert(use->in(MemNode::Memory) == current, "must be"); | |
933 _igvn.hash_delete(use); | |
934 if (use == prev) { // connect prev to my_mem | |
935 use->set_req(MemNode::Memory, my_mem); | |
936 } else if (sched_before.member(use)) { | |
937 _igvn.hash_delete(uip); | |
938 use->set_req(MemNode::Memory, uip); | |
939 } else { | |
940 _igvn.hash_delete(lip); | |
941 use->set_req(MemNode::Memory, lip); | |
942 } | |
943 _igvn._worklist.push(use); | |
944 --i; //deleted this edge; rescan position | |
945 } | |
946 } | |
947 | |
948 bool sched_up = sched_before.member(current); | |
949 Node *insert_pt = sched_up ? uip : lip; | |
950 _igvn.hash_delete(insert_pt); | |
951 | |
952 // all uses of insert_pt's memory state should use current's instead | |
953 for (DUIterator i = insert_pt->outs(); insert_pt->has_out(i); i++) { | |
954 Node* use = insert_pt->out(i); | |
955 if (use->is_Mem()) { | |
956 assert(use->in(MemNode::Memory) == insert_pt, "must be"); | |
957 _igvn.hash_delete(use); | |
958 use->set_req(MemNode::Memory, current); | |
959 _igvn._worklist.push(use); | |
960 --i; //deleted this edge; rescan position | |
961 } else if (!sched_up && use->is_Phi() && use->bottom_type() == Type::MEMORY) { | |
962 uint pos; //lip (lower insert point) must be the last one in the memory slice | |
963 _igvn.hash_delete(use); | |
964 for (pos=1; pos < use->req(); pos++) { | |
965 if (use->in(pos) == insert_pt) break; | |
966 } | |
967 use->set_req(pos, current); | |
968 _igvn._worklist.push(use); | |
969 --i; | |
970 } | |
971 } | |
972 | |
973 //connect current to insert_pt | |
974 current->set_req(MemNode::Memory, insert_pt); | |
975 _igvn._worklist.push(current); | |
976 } | |
977 | |
978 //------------------------------co_locate_pack---------------------------------- | |
979 // To schedule a store pack, we need to move any sandwiched memory ops either before | |
980 // or after the pack, based upon dependence information: | |
981 // (1) If any store in the pack depends on the sandwiched memory op, the | |
982 // sandwiched memory op must be scheduled BEFORE the pack; | |
983 // (2) If a sandwiched memory op depends on any store in the pack, the | |
984 // sandwiched memory op must be scheduled AFTER the pack; | |
985 // (3) If a sandwiched memory op (say, memA) depends on another sandwiched | |
986 // memory op (say memB), memB must be scheduled before memA. So, if memA is | |
987 // scheduled before the pack, memB must also be scheduled before the pack; | |
988 // (4) If there is no dependence restriction for a sandwiched memory op, we simply | |
989 // schedule this store AFTER the pack | |
990 // (5) We know there is no dependence cycle, so there in no other case; | |
991 // (6) Finally, all memory ops in another single pack should be moved in the same direction. | |
992 // | |
993 // To schedule a load pack: the memory edge of every loads in the pack must be | |
994 // the same as the memory edge of the last executed load in the pack | |
918 void SuperWord::co_locate_pack(Node_List* pk) { | 995 void SuperWord::co_locate_pack(Node_List* pk) { |
919 if (pk->at(0)->is_Store()) { | 996 if (pk->at(0)->is_Store()) { |
920 // Push Stores down towards last executed pack member | |
921 MemNode* first = executed_first(pk)->as_Mem(); | 997 MemNode* first = executed_first(pk)->as_Mem(); |
922 MemNode* last = executed_last(pk)->as_Mem(); | 998 MemNode* last = executed_last(pk)->as_Mem(); |
923 MemNode* insert_pt = last; | 999 Unique_Node_List schedule_before_pack; |
1000 Unique_Node_List memops; | |
1001 | |
924 MemNode* current = last->in(MemNode::Memory)->as_Mem(); | 1002 MemNode* current = last->in(MemNode::Memory)->as_Mem(); |
1003 MemNode* previous = last; | |
925 while (true) { | 1004 while (true) { |
926 assert(in_bb(current), "stay in block"); | 1005 assert(in_bb(current), "stay in block"); |
1006 memops.push(previous); | |
1007 for (DUIterator i = current->outs(); current->has_out(i); i++) { | |
1008 Node* use = current->out(i); | |
1009 if (use->is_Mem() && use != previous) | |
1010 memops.push(use); | |
1011 } | |
1012 if(current == first) break; | |
1013 previous = current; | |
1014 current = current->in(MemNode::Memory)->as_Mem(); | |
1015 } | |
1016 | |
1017 // determine which memory operations should be scheduled before the pack | |
1018 for (uint i = 1; i < memops.size(); i++) { | |
1019 Node *s1 = memops.at(i); | |
1020 if (!in_pack(s1, pk) && !schedule_before_pack.member(s1)) { | |
1021 for (uint j = 0; j< i; j++) { | |
1022 Node *s2 = memops.at(j); | |
1023 if (!independent(s1, s2)) { | |
1024 if (in_pack(s2, pk) || schedule_before_pack.member(s2)) { | |
1025 schedule_before_pack.push(s1); //s1 must be scheduled before | |
1026 Node_List* mem_pk = my_pack(s1); | |
1027 if (mem_pk != NULL) { | |
1028 for (uint ii = 0; ii < mem_pk->size(); ii++) { | |
1029 Node* s = mem_pk->at(ii); // follow partner | |
1030 if (memops.member(s) && !schedule_before_pack.member(s)) | |
1031 schedule_before_pack.push(s); | |
1032 } | |
1033 } | |
1034 } | |
1035 } | |
1036 } | |
1037 } | |
1038 } | |
1039 | |
1040 MemNode* lower_insert_pt = last; | |
1041 Node* upper_insert_pt = first->in(MemNode::Memory); | |
1042 previous = last; //previous store in pk | |
1043 current = last->in(MemNode::Memory)->as_Mem(); | |
1044 | |
1045 //start scheduling from "last" to "first" | |
1046 while (true) { | |
1047 assert(in_bb(current), "stay in block"); | |
1048 assert(in_pack(previous, pk), "previous stays in pack"); | |
927 Node* my_mem = current->in(MemNode::Memory); | 1049 Node* my_mem = current->in(MemNode::Memory); |
1050 | |
928 if (in_pack(current, pk)) { | 1051 if (in_pack(current, pk)) { |
929 // Forward users of my memory state to my input memory state | 1052 // Forward users of my memory state (except "previous) to my input memory state |
930 _igvn.hash_delete(current); | 1053 _igvn.hash_delete(current); |
931 _igvn.hash_delete(my_mem); | |
932 for (DUIterator i = current->outs(); current->has_out(i); i++) { | 1054 for (DUIterator i = current->outs(); current->has_out(i); i++) { |
933 Node* use = current->out(i); | 1055 Node* use = current->out(i); |
934 if (use->is_Mem()) { | 1056 if (use->is_Mem() && use != previous) { |
935 assert(use->in(MemNode::Memory) == current, "must be"); | 1057 assert(use->in(MemNode::Memory) == current, "must be"); |
936 _igvn.hash_delete(use); | 1058 _igvn.hash_delete(use); |
937 use->set_req(MemNode::Memory, my_mem); | 1059 if (schedule_before_pack.member(use)) { |
1060 _igvn.hash_delete(upper_insert_pt); | |
1061 use->set_req(MemNode::Memory, upper_insert_pt); | |
1062 } else { | |
1063 _igvn.hash_delete(lower_insert_pt); | |
1064 use->set_req(MemNode::Memory, lower_insert_pt); | |
1065 } | |
938 _igvn._worklist.push(use); | 1066 _igvn._worklist.push(use); |
939 --i; // deleted this edge; rescan position | 1067 --i; // deleted this edge; rescan position |
940 } | 1068 } |
941 } | 1069 } |
942 // put current immediately before insert_pt | 1070 previous = current; |
943 current->set_req(MemNode::Memory, insert_pt->in(MemNode::Memory)); | 1071 } else { // !in_pack(current, pk) ==> a sandwiched store |
944 _igvn.hash_delete(insert_pt); | 1072 remove_and_insert(current, previous, lower_insert_pt, upper_insert_pt, schedule_before_pack); |
945 insert_pt->set_req(MemNode::Memory, current); | 1073 } |
946 _igvn._worklist.push(insert_pt); | 1074 |
947 _igvn._worklist.push(current); | |
948 insert_pt = current; | |
949 } | |
950 if (current == first) break; | 1075 if (current == first) break; |
951 current = my_mem->as_Mem(); | 1076 current = my_mem->as_Mem(); |
952 } | 1077 } // end while |
953 } else if (pk->at(0)->is_Load()) { | 1078 } else if (pk->at(0)->is_Load()) { //load |
954 // Pull Loads up towards first executed pack member | 1079 // all use the memory state that the last executed load uses |
955 LoadNode* first = executed_first(pk)->as_Load(); | 1080 LoadNode* last_load = executed_last(pk)->as_Load(); |
956 Node* first_mem = first->in(MemNode::Memory); | 1081 Node* last_mem = last_load->in(MemNode::Memory); |
957 _igvn.hash_delete(first_mem); | 1082 _igvn.hash_delete(last_mem); |
958 // Give each load same memory state as first | 1083 // Give each load same memory state as last |
959 for (uint i = 0; i < pk->size(); i++) { | 1084 for (uint i = 0; i < pk->size(); i++) { |
960 LoadNode* ld = pk->at(i)->as_Load(); | 1085 LoadNode* ld = pk->at(i)->as_Load(); |
961 _igvn.hash_delete(ld); | 1086 _igvn.hash_delete(ld); |
962 ld->set_req(MemNode::Memory, first_mem); | 1087 ld->set_req(MemNode::Memory, last_mem); |
963 _igvn._worklist.push(ld); | 1088 _igvn._worklist.push(ld); |
964 } | 1089 } |
965 } | 1090 } |
966 } | 1091 } |
967 | 1092 |