Mercurial > hg > truffle
annotate src/share/vm/opto/superword.cpp @ 4710:41406797186b
7113012: G1: rename not-fully-young GCs as "mixed"
Summary: Renamed partially-young GCs as mixed and fully-young GCs as young. Change all external output that includes those terms (GC log and GC ergo log) as well as any comments, fields, methods, etc. The changeset also includes very minor code tidying up (added some curly brackets).
Reviewed-by: johnc, brutisso
| author | tonyp |
|---|---|
| date | Fri, 16 Dec 2011 02:14:27 -0500 |
| parents | 6987871cfb9b |
| children | dc682d9431f3 |
| rev | line source |
|---|---|
| 0 | 1 /* |
| 1585 | 2 * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. |
| 0 | 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| 4 * | |
| 5 * This code is free software; you can redistribute it and/or modify it | |
| 6 * under the terms of the GNU General Public License version 2 only, as | |
| 7 * published by the Free Software Foundation. | |
| 8 * | |
| 9 * This code is distributed in the hope that it will be useful, but WITHOUT | |
| 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
| 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
| 12 * version 2 for more details (a copy is included in the LICENSE file that | |
| 13 * accompanied this code). | |
| 14 * | |
| 15 * You should have received a copy of the GNU General Public License version | |
| 16 * 2 along with this work; if not, write to the Free Software Foundation, | |
| 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |
| 18 * | |
|
1552
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1058
diff
changeset
|
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
|
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1058
diff
changeset
|
20 * or visit www.oracle.com if you need additional information or have any |
|
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1058
diff
changeset
|
21 * questions. |
| 0 | 22 */ |
| 23 | |
| 1972 | 24 #include "precompiled.hpp" |
| 25 #include "compiler/compileLog.hpp" | |
| 26 #include "libadt/vectset.hpp" | |
| 27 #include "memory/allocation.inline.hpp" | |
| 28 #include "opto/addnode.hpp" | |
| 29 #include "opto/callnode.hpp" | |
| 30 #include "opto/divnode.hpp" | |
| 31 #include "opto/matcher.hpp" | |
| 32 #include "opto/memnode.hpp" | |
| 33 #include "opto/mulnode.hpp" | |
| 34 #include "opto/opcodes.hpp" | |
| 35 #include "opto/superword.hpp" | |
| 36 #include "opto/vectornode.hpp" | |
| 0 | 37 |
| 38 // | |
| 39 // S U P E R W O R D T R A N S F O R M | |
| 40 //============================================================================= | |
| 41 | |
| 42 //------------------------------SuperWord--------------------------- | |
| 43 SuperWord::SuperWord(PhaseIdealLoop* phase) : | |
| 44 _phase(phase), | |
| 45 _igvn(phase->_igvn), | |
| 46 _arena(phase->C->comp_arena()), | |
| 47 _packset(arena(), 8, 0, NULL), // packs for the current block | |
| 48 _bb_idx(arena(), (int)(1.10 * phase->C->unique()), 0, 0), // node idx to index in bb | |
| 49 _block(arena(), 8, 0, NULL), // nodes in current block | |
| 50 _data_entry(arena(), 8, 0, NULL), // nodes with all inputs from outside | |
| 51 _mem_slice_head(arena(), 8, 0, NULL), // memory slice heads | |
| 52 _mem_slice_tail(arena(), 8, 0, NULL), // memory slice tails | |
| 53 _node_info(arena(), 8, 0, SWNodeInfo::initial), // info needed per node | |
| 54 _align_to_ref(NULL), // memory reference to align vectors to | |
| 55 _disjoint_ptrs(arena(), 8, 0, OrderedPair::initial), // runtime disambiguated pointer pairs | |
| 56 _dg(_arena), // dependence graph | |
| 57 _visited(arena()), // visited node set | |
| 58 _post_visited(arena()), // post visited node set | |
| 59 _n_idx_list(arena(), 8), // scratch list of (node,index) pairs | |
| 60 _stk(arena(), 8, 0, NULL), // scratch stack of nodes | |
| 61 _nlist(arena(), 8, 0, NULL), // scratch list of nodes | |
| 62 _lpt(NULL), // loop tree node | |
| 63 _lp(NULL), // LoopNode | |
| 64 _bb(NULL), // basic block | |
| 65 _iv(NULL) // induction var | |
| 66 {} | |
| 67 | |
| 68 //------------------------------transform_loop--------------------------- | |
| 69 void SuperWord::transform_loop(IdealLoopTree* lpt) { | |
| 70 assert(lpt->_head->is_CountedLoop(), "must be"); | |
| 71 CountedLoopNode *cl = lpt->_head->as_CountedLoop(); | |
| 72 | |
|
3850
6987871cfb9b
7077439: Possible reference through NULL in loopPredicate.cpp:726
kvn
parents:
3842
diff
changeset
|
73 if (!cl->is_valid_counted_loop()) return; // skip malformed counted loop |
|
6987871cfb9b
7077439: Possible reference through NULL in loopPredicate.cpp:726
kvn
parents:
3842
diff
changeset
|
74 |
| 0 | 75 if (!cl->is_main_loop() ) return; // skip normal, pre, and post loops |
| 76 | |
| 77 // Check for no control flow in body (other than exit) | |
| 78 Node *cl_exit = cl->loopexit(); | |
| 79 if (cl_exit->in(0) != lpt->_head) return; | |
| 80 | |
|
105
a7d0f95410bd
6646020: assert(in_bb(n),"must be in block") in -Xcomp mode
never
parents:
72
diff
changeset
|
81 // Make sure the are no extra control users of the loop backedge |
|
a7d0f95410bd
6646020: assert(in_bb(n),"must be in block") in -Xcomp mode
never
parents:
72
diff
changeset
|
82 if (cl->back_control()->outcnt() != 1) { |
|
a7d0f95410bd
6646020: assert(in_bb(n),"must be in block") in -Xcomp mode
never
parents:
72
diff
changeset
|
83 return; |
|
a7d0f95410bd
6646020: assert(in_bb(n),"must be in block") in -Xcomp mode
never
parents:
72
diff
changeset
|
84 } |
|
a7d0f95410bd
6646020: assert(in_bb(n),"must be in block") in -Xcomp mode
never
parents:
72
diff
changeset
|
85 |
| 0 | 86 // Check for pre-loop ending with CountedLoopEnd(Bool(Cmp(x,Opaque1(limit)))) |
| 87 CountedLoopEndNode* pre_end = get_pre_loop_end(cl); | |
| 88 if (pre_end == NULL) return; | |
| 89 Node *pre_opaq1 = pre_end->limit(); | |
| 90 if (pre_opaq1->Opcode() != Op_Opaque1) return; | |
| 91 | |
| 92 // Do vectors exist on this architecture? | |
| 93 if (vector_width_in_bytes() == 0) return; | |
| 94 | |
| 95 init(); // initialize data structures | |
| 96 | |
| 97 set_lpt(lpt); | |
| 98 set_lp(cl); | |
| 99 | |
| 100 // For now, define one block which is the entire loop body | |
| 101 set_bb(cl); | |
| 102 | |
| 103 assert(_packset.length() == 0, "packset must be empty"); | |
| 104 SLP_extract(); | |
| 105 } | |
| 106 | |
| 107 //------------------------------SLP_extract--------------------------- | |
| 108 // Extract the superword level parallelism | |
| 109 // | |
| 110 // 1) A reverse post-order of nodes in the block is constructed. By scanning | |
| 111 // this list from first to last, all definitions are visited before their uses. | |
| 112 // | |
| 113 // 2) A point-to-point dependence graph is constructed between memory references. | |
| 114 // This simplies the upcoming "independence" checker. | |
| 115 // | |
| 116 // 3) The maximum depth in the node graph from the beginning of the block | |
| 117 // to each node is computed. This is used to prune the graph search | |
| 118 // in the independence checker. | |
| 119 // | |
| 120 // 4) For integer types, the necessary bit width is propagated backwards | |
| 121 // from stores to allow packed operations on byte, char, and short | |
| 122 // integers. This reverses the promotion to type "int" that javac | |
| 123 // did for operations like: char c1,c2,c3; c1 = c2 + c3. | |
| 124 // | |
| 125 // 5) One of the memory references is picked to be an aligned vector reference. | |
| 126 // The pre-loop trip count is adjusted to align this reference in the | |
| 127 // unrolled body. | |
| 128 // | |
| 129 // 6) The initial set of pack pairs is seeded with memory references. | |
| 130 // | |
| 131 // 7) The set of pack pairs is extended by following use->def and def->use links. | |
| 132 // | |
| 133 // 8) The pairs are combined into vector sized packs. | |
| 134 // | |
| 135 // 9) Reorder the memory slices to co-locate members of the memory packs. | |
| 136 // | |
| 137 // 10) Generate ideal vector nodes for the final set of packs and where necessary, | |
| 138 // inserting scalar promotion, vector creation from multiple scalars, and | |
| 139 // extraction of scalar values from vectors. | |
| 140 // | |
| 141 void SuperWord::SLP_extract() { | |
| 142 | |
| 143 // Ready the block | |
| 144 | |
| 145 construct_bb(); | |
| 146 | |
| 147 dependence_graph(); | |
| 148 | |
| 149 compute_max_depth(); | |
| 150 | |
| 151 compute_vector_element_type(); | |
| 152 | |
| 153 // Attempt vectorization | |
| 154 | |
| 155 find_adjacent_refs(); | |
| 156 | |
| 157 extend_packlist(); | |
| 158 | |
| 159 combine_packs(); | |
| 160 | |
| 161 construct_my_pack_map(); | |
| 162 | |
| 163 filter_packs(); | |
| 164 | |
| 165 schedule(); | |
| 166 | |
| 167 output(); | |
| 168 } | |
| 169 | |
| 170 //------------------------------find_adjacent_refs--------------------------- | |
| 171 // Find the adjacent memory references and create pack pairs for them. | |
| 172 // This is the initial set of packs that will then be extended by | |
| 173 // following use->def and def->use links. The align positions are | |
| 174 // assigned relative to the reference "align_to_ref" | |
| 175 void SuperWord::find_adjacent_refs() { | |
| 176 // Get list of memory operations | |
| 177 Node_List memops; | |
| 178 for (int i = 0; i < _block.length(); i++) { | |
| 179 Node* n = _block.at(i); | |
|
29
d5fc211aea19
6633953: type2aelembytes{T_ADDRESS} should be 8 bytes in 64 bit VM
kvn
parents:
0
diff
changeset
|
180 if (n->is_Mem() && in_bb(n) && |
|
d5fc211aea19
6633953: type2aelembytes{T_ADDRESS} should be 8 bytes in 64 bit VM
kvn
parents:
0
diff
changeset
|
181 is_java_primitive(n->as_Mem()->memory_type())) { |
| 0 | 182 int align = memory_alignment(n->as_Mem(), 0); |
| 183 if (align != bottom_align) { | |
| 184 memops.push(n); | |
| 185 } | |
| 186 } | |
| 187 } | |
| 188 if (memops.size() == 0) return; | |
| 189 | |
| 190 // Find a memory reference to align to. The pre-loop trip count | |
| 191 // is modified to align this reference to a vector-aligned address | |
| 192 find_align_to_ref(memops); | |
| 193 if (align_to_ref() == NULL) return; | |
| 194 | |
| 195 SWPointer align_to_ref_p(align_to_ref(), this); | |
| 196 int offset = align_to_ref_p.offset_in_bytes(); | |
| 197 int scale = align_to_ref_p.scale_in_bytes(); | |
| 198 int vw = vector_width_in_bytes(); | |
| 199 int stride_sign = (scale * iv_stride()) > 0 ? 1 : -1; | |
| 200 int iv_adjustment = (stride_sign * vw - (offset % vw)) % vw; | |
| 201 | |
| 202 #ifndef PRODUCT | |
| 203 if (TraceSuperWord) | |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
204 tty->print_cr("\noffset = %d iv_adjustment = %d elt_align = %d scale = %d iv_stride = %d", |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
205 offset, iv_adjustment, align_to_ref_p.memory_size(), align_to_ref_p.scale_in_bytes(), iv_stride()); |
| 0 | 206 #endif |
| 207 | |
| 208 // Set alignment relative to "align_to_ref" | |
| 209 for (int i = memops.size() - 1; i >= 0; i--) { | |
| 210 MemNode* s = memops.at(i)->as_Mem(); | |
| 211 SWPointer p2(s, this); | |
| 212 if (p2.comparable(align_to_ref_p)) { | |
| 213 int align = memory_alignment(s, iv_adjustment); | |
| 214 set_alignment(s, align); | |
| 215 } else { | |
| 216 memops.remove(i); | |
| 217 } | |
| 218 } | |
| 219 | |
| 220 // Create initial pack pairs of memory operations | |
| 221 for (uint i = 0; i < memops.size(); i++) { | |
| 222 Node* s1 = memops.at(i); | |
| 223 for (uint j = 0; j < memops.size(); j++) { | |
| 224 Node* s2 = memops.at(j); | |
| 225 if (s1 != s2 && are_adjacent_refs(s1, s2)) { | |
| 226 int align = alignment(s1); | |
| 227 if (stmts_can_pack(s1, s2, align)) { | |
| 228 Node_List* pair = new Node_List(); | |
| 229 pair->push(s1); | |
| 230 pair->push(s2); | |
| 231 _packset.append(pair); | |
| 232 } | |
| 233 } | |
| 234 } | |
| 235 } | |
| 236 | |
| 237 #ifndef PRODUCT | |
| 238 if (TraceSuperWord) { | |
| 239 tty->print_cr("\nAfter find_adjacent_refs"); | |
| 240 print_packset(); | |
| 241 } | |
| 242 #endif | |
| 243 } | |
| 244 | |
| 245 //------------------------------find_align_to_ref--------------------------- | |
| 246 // Find a memory reference to align the loop induction variable to. | |
| 247 // Looks first at stores then at loads, looking for a memory reference | |
| 248 // with the largest number of references similar to it. | |
| 249 void SuperWord::find_align_to_ref(Node_List &memops) { | |
| 250 GrowableArray<int> cmp_ct(arena(), memops.size(), memops.size(), 0); | |
| 251 | |
| 252 // Count number of comparable memory ops | |
| 253 for (uint i = 0; i < memops.size(); i++) { | |
| 254 MemNode* s1 = memops.at(i)->as_Mem(); | |
| 255 SWPointer p1(s1, this); | |
| 256 // Discard if pre loop can't align this reference | |
| 257 if (!ref_is_alignable(p1)) { | |
| 258 *cmp_ct.adr_at(i) = 0; | |
| 259 continue; | |
| 260 } | |
| 261 for (uint j = i+1; j < memops.size(); j++) { | |
| 262 MemNode* s2 = memops.at(j)->as_Mem(); | |
| 263 if (isomorphic(s1, s2)) { | |
| 264 SWPointer p2(s2, this); | |
| 265 if (p1.comparable(p2)) { | |
| 266 (*cmp_ct.adr_at(i))++; | |
| 267 (*cmp_ct.adr_at(j))++; | |
| 268 } | |
| 269 } | |
| 270 } | |
| 271 } | |
| 272 | |
| 273 // Find Store (or Load) with the greatest number of "comparable" references | |
| 274 int max_ct = 0; | |
| 275 int max_idx = -1; | |
| 276 int min_size = max_jint; | |
| 277 int min_iv_offset = max_jint; | |
| 278 for (uint j = 0; j < memops.size(); j++) { | |
| 279 MemNode* s = memops.at(j)->as_Mem(); | |
| 280 if (s->is_Store()) { | |
| 281 SWPointer p(s, this); | |
| 282 if (cmp_ct.at(j) > max_ct || | |
| 283 cmp_ct.at(j) == max_ct && (data_size(s) < min_size || | |
| 284 data_size(s) == min_size && | |
| 285 p.offset_in_bytes() < min_iv_offset)) { | |
| 286 max_ct = cmp_ct.at(j); | |
| 287 max_idx = j; | |
| 288 min_size = data_size(s); | |
| 289 min_iv_offset = p.offset_in_bytes(); | |
| 290 } | |
| 291 } | |
| 292 } | |
| 293 // If no stores, look at loads | |
| 294 if (max_ct == 0) { | |
| 295 for (uint j = 0; j < memops.size(); j++) { | |
| 296 MemNode* s = memops.at(j)->as_Mem(); | |
| 297 if (s->is_Load()) { | |
| 298 SWPointer p(s, this); | |
| 299 if (cmp_ct.at(j) > max_ct || | |
| 300 cmp_ct.at(j) == max_ct && (data_size(s) < min_size || | |
| 301 data_size(s) == min_size && | |
| 302 p.offset_in_bytes() < min_iv_offset)) { | |
| 303 max_ct = cmp_ct.at(j); | |
| 304 max_idx = j; | |
| 305 min_size = data_size(s); | |
| 306 min_iv_offset = p.offset_in_bytes(); | |
| 307 } | |
| 308 } | |
| 309 } | |
| 310 } | |
| 311 | |
| 312 if (max_ct > 0) | |
| 313 set_align_to_ref(memops.at(max_idx)->as_Mem()); | |
| 314 | |
| 315 #ifndef PRODUCT | |
| 316 if (TraceSuperWord && Verbose) { | |
| 317 tty->print_cr("\nVector memops after find_align_to_refs"); | |
| 318 for (uint i = 0; i < memops.size(); i++) { | |
| 319 MemNode* s = memops.at(i)->as_Mem(); | |
| 320 s->dump(); | |
| 321 } | |
| 322 } | |
| 323 #endif | |
| 324 } | |
| 325 | |
| 326 //------------------------------ref_is_alignable--------------------------- | |
| 327 // Can the preloop align the reference to position zero in the vector? | |
| 328 bool SuperWord::ref_is_alignable(SWPointer& p) { | |
| 329 if (!p.has_iv()) { | |
| 330 return true; // no induction variable | |
| 331 } | |
| 332 CountedLoopEndNode* pre_end = get_pre_loop_end(lp()->as_CountedLoop()); | |
| 333 assert(pre_end->stride_is_con(), "pre loop stride is constant"); | |
| 334 int preloop_stride = pre_end->stride_con(); | |
| 335 | |
| 336 int span = preloop_stride * p.scale_in_bytes(); | |
| 337 | |
| 338 // Stride one accesses are alignable. | |
| 339 if (ABS(span) == p.memory_size()) | |
| 340 return true; | |
| 341 | |
| 342 // If initial offset from start of object is computable, | |
| 343 // compute alignment within the vector. | |
| 344 int vw = vector_width_in_bytes(); | |
| 345 if (vw % span == 0) { | |
| 346 Node* init_nd = pre_end->init_trip(); | |
| 347 if (init_nd->is_Con() && p.invar() == NULL) { | |
| 348 int init = init_nd->bottom_type()->is_int()->get_con(); | |
| 349 | |
| 350 int init_offset = init * p.scale_in_bytes() + p.offset_in_bytes(); | |
| 351 assert(init_offset >= 0, "positive offset from object start"); | |
| 352 | |
| 353 if (span > 0) { | |
| 354 return (vw - (init_offset % vw)) % span == 0; | |
| 355 } else { | |
| 356 assert(span < 0, "nonzero stride * scale"); | |
| 357 return (init_offset % vw) % -span == 0; | |
| 358 } | |
| 359 } | |
| 360 } | |
| 361 return false; | |
| 362 } | |
| 363 | |
| 364 //---------------------------dependence_graph--------------------------- | |
| 365 // Construct dependency graph. | |
| 366 // Add dependence edges to load/store nodes for memory dependence | |
| 367 // A.out()->DependNode.in(1) and DependNode.out()->B.prec(x) | |
| 368 void SuperWord::dependence_graph() { | |
| 369 // First, assign a dependence node to each memory node | |
| 370 for (int i = 0; i < _block.length(); i++ ) { | |
| 371 Node *n = _block.at(i); | |
| 372 if (n->is_Mem() || n->is_Phi() && n->bottom_type() == Type::MEMORY) { | |
| 373 _dg.make_node(n); | |
| 374 } | |
| 375 } | |
| 376 | |
| 377 // For each memory slice, create the dependences | |
| 378 for (int i = 0; i < _mem_slice_head.length(); i++) { | |
| 379 Node* n = _mem_slice_head.at(i); | |
| 380 Node* n_tail = _mem_slice_tail.at(i); | |
| 381 | |
| 382 // Get slice in predecessor order (last is first) | |
| 383 mem_slice_preds(n_tail, n, _nlist); | |
| 384 | |
| 385 // Make the slice dependent on the root | |
| 386 DepMem* slice = _dg.dep(n); | |
| 387 _dg.make_edge(_dg.root(), slice); | |
| 388 | |
| 389 // Create a sink for the slice | |
| 390 DepMem* slice_sink = _dg.make_node(NULL); | |
| 391 _dg.make_edge(slice_sink, _dg.tail()); | |
| 392 | |
| 393 // Now visit each pair of memory ops, creating the edges | |
| 394 for (int j = _nlist.length() - 1; j >= 0 ; j--) { | |
| 395 Node* s1 = _nlist.at(j); | |
| 396 | |
| 397 // If no dependency yet, use slice | |
| 398 if (_dg.dep(s1)->in_cnt() == 0) { | |
| 399 _dg.make_edge(slice, s1); | |
| 400 } | |
| 401 SWPointer p1(s1->as_Mem(), this); | |
| 402 bool sink_dependent = true; | |
| 403 for (int k = j - 1; k >= 0; k--) { | |
| 404 Node* s2 = _nlist.at(k); | |
| 405 if (s1->is_Load() && s2->is_Load()) | |
| 406 continue; | |
| 407 SWPointer p2(s2->as_Mem(), this); | |
| 408 | |
| 409 int cmp = p1.cmp(p2); | |
| 410 if (SuperWordRTDepCheck && | |
| 411 p1.base() != p2.base() && p1.valid() && p2.valid()) { | |
| 412 // Create a runtime check to disambiguate | |
| 413 OrderedPair pp(p1.base(), p2.base()); | |
| 414 _disjoint_ptrs.append_if_missing(pp); | |
| 415 } else if (!SWPointer::not_equal(cmp)) { | |
| 416 // Possibly same address | |
| 417 _dg.make_edge(s1, s2); | |
| 418 sink_dependent = false; | |
| 419 } | |
| 420 } | |
| 421 if (sink_dependent) { | |
| 422 _dg.make_edge(s1, slice_sink); | |
| 423 } | |
| 424 } | |
| 425 #ifndef PRODUCT | |
| 426 if (TraceSuperWord) { | |
| 427 tty->print_cr("\nDependence graph for slice: %d", n->_idx); | |
| 428 for (int q = 0; q < _nlist.length(); q++) { | |
| 429 _dg.print(_nlist.at(q)); | |
| 430 } | |
| 431 tty->cr(); | |
| 432 } | |
| 433 #endif | |
| 434 _nlist.clear(); | |
| 435 } | |
| 436 | |
| 437 #ifndef PRODUCT | |
| 438 if (TraceSuperWord) { | |
| 439 tty->print_cr("\ndisjoint_ptrs: %s", _disjoint_ptrs.length() > 0 ? "" : "NONE"); | |
| 440 for (int r = 0; r < _disjoint_ptrs.length(); r++) { | |
| 441 _disjoint_ptrs.at(r).print(); | |
| 442 tty->cr(); | |
| 443 } | |
| 444 tty->cr(); | |
| 445 } | |
| 446 #endif | |
| 447 } | |
| 448 | |
| 449 //---------------------------mem_slice_preds--------------------------- | |
| 450 // Return a memory slice (node list) in predecessor order starting at "start" | |
| 451 void SuperWord::mem_slice_preds(Node* start, Node* stop, GrowableArray<Node*> &preds) { | |
| 452 assert(preds.length() == 0, "start empty"); | |
| 453 Node* n = start; | |
| 454 Node* prev = NULL; | |
| 455 while (true) { | |
| 456 assert(in_bb(n), "must be in block"); | |
| 457 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { | |
| 458 Node* out = n->fast_out(i); | |
| 459 if (out->is_Load()) { | |
| 460 if (in_bb(out)) { | |
| 461 preds.push(out); | |
| 462 } | |
| 463 } else { | |
| 464 // FIXME | |
| 465 if (out->is_MergeMem() && !in_bb(out)) { | |
| 466 // Either unrolling is causing a memory edge not to disappear, | |
| 467 // or need to run igvn.optimize() again before SLP | |
| 468 } else if (out->is_Phi() && out->bottom_type() == Type::MEMORY && !in_bb(out)) { | |
| 469 // Ditto. Not sure what else to check further. | |
| 667 | 470 } else if (out->Opcode() == Op_StoreCM && out->in(MemNode::OopStore) == n) { |
| 0 | 471 // StoreCM has an input edge used as a precedence edge. |
| 472 // Maybe an issue when oop stores are vectorized. | |
| 473 } else { | |
| 474 assert(out == prev || prev == NULL, "no branches off of store slice"); | |
| 475 } | |
| 476 } | |
| 477 } | |
| 478 if (n == stop) break; | |
| 479 preds.push(n); | |
| 480 prev = n; | |
| 481 n = n->in(MemNode::Memory); | |
| 482 } | |
| 483 } | |
| 484 | |
| 485 //------------------------------stmts_can_pack--------------------------- | |
| 605 | 486 // Can s1 and s2 be in a pack with s1 immediately preceding s2 and |
| 0 | 487 // s1 aligned at "align" |
| 488 bool SuperWord::stmts_can_pack(Node* s1, Node* s2, int align) { | |
|
987
00977607da34
6879921: CTW failure jdk6_18/hotspot/src/share/vm/utilities/globalDefinitions.cpp:268
cfang
parents:
985
diff
changeset
|
489 |
|
00977607da34
6879921: CTW failure jdk6_18/hotspot/src/share/vm/utilities/globalDefinitions.cpp:268
cfang
parents:
985
diff
changeset
|
490 // Do not use superword for non-primitives |
|
00977607da34
6879921: CTW failure jdk6_18/hotspot/src/share/vm/utilities/globalDefinitions.cpp:268
cfang
parents:
985
diff
changeset
|
491 if((s1->is_Mem() && !is_java_primitive(s1->as_Mem()->memory_type())) || |
|
00977607da34
6879921: CTW failure jdk6_18/hotspot/src/share/vm/utilities/globalDefinitions.cpp:268
cfang
parents:
985
diff
changeset
|
492 (s2->is_Mem() && !is_java_primitive(s2->as_Mem()->memory_type()))) |
|
00977607da34
6879921: CTW failure jdk6_18/hotspot/src/share/vm/utilities/globalDefinitions.cpp:268
cfang
parents:
985
diff
changeset
|
493 return false; |
|
00977607da34
6879921: CTW failure jdk6_18/hotspot/src/share/vm/utilities/globalDefinitions.cpp:268
cfang
parents:
985
diff
changeset
|
494 |
| 0 | 495 if (isomorphic(s1, s2)) { |
| 496 if (independent(s1, s2)) { | |
| 497 if (!exists_at(s1, 0) && !exists_at(s2, 1)) { | |
| 498 if (!s1->is_Mem() || are_adjacent_refs(s1, s2)) { | |
| 499 int s1_align = alignment(s1); | |
| 500 int s2_align = alignment(s2); | |
| 501 if (s1_align == top_align || s1_align == align) { | |
| 502 if (s2_align == top_align || s2_align == align + data_size(s1)) { | |
| 503 return true; | |
| 504 } | |
| 505 } | |
| 506 } | |
| 507 } | |
| 508 } | |
| 509 } | |
| 510 return false; | |
| 511 } | |
| 512 | |
| 513 //------------------------------exists_at--------------------------- | |
| 514 // Does s exist in a pack at position pos? | |
| 515 bool SuperWord::exists_at(Node* s, uint pos) { | |
| 516 for (int i = 0; i < _packset.length(); i++) { | |
| 517 Node_List* p = _packset.at(i); | |
| 518 if (p->at(pos) == s) { | |
| 519 return true; | |
| 520 } | |
| 521 } | |
| 522 return false; | |
| 523 } | |
| 524 | |
| 525 //------------------------------are_adjacent_refs--------------------------- | |
| 526 // Is s1 immediately before s2 in memory? | |
| 527 bool SuperWord::are_adjacent_refs(Node* s1, Node* s2) { | |
| 528 if (!s1->is_Mem() || !s2->is_Mem()) return false; | |
| 529 if (!in_bb(s1) || !in_bb(s2)) return false; | |
| 1585 | 530 |
| 531 // Do not use superword for non-primitives | |
| 532 if (!is_java_primitive(s1->as_Mem()->memory_type()) || | |
| 533 !is_java_primitive(s2->as_Mem()->memory_type())) { | |
| 534 return false; | |
| 535 } | |
| 536 | |
| 0 | 537 // FIXME - co_locate_pack fails on Stores in different mem-slices, so |
| 538 // only pack memops that are in the same alias set until that's fixed. | |
| 539 if (_phase->C->get_alias_index(s1->as_Mem()->adr_type()) != | |
| 540 _phase->C->get_alias_index(s2->as_Mem()->adr_type())) | |
| 541 return false; | |
| 542 SWPointer p1(s1->as_Mem(), this); | |
| 543 SWPointer p2(s2->as_Mem(), this); | |
| 544 if (p1.base() != p2.base() || !p1.comparable(p2)) return false; | |
| 545 int diff = p2.offset_in_bytes() - p1.offset_in_bytes(); | |
| 546 return diff == data_size(s1); | |
| 547 } | |
| 548 | |
| 549 //------------------------------isomorphic--------------------------- | |
| 550 // Are s1 and s2 similar? | |
| 551 bool SuperWord::isomorphic(Node* s1, Node* s2) { | |
| 552 if (s1->Opcode() != s2->Opcode()) return false; | |
| 553 if (s1->req() != s2->req()) return false; | |
| 554 if (s1->in(0) != s2->in(0)) return false; | |
| 555 if (velt_type(s1) != velt_type(s2)) return false; | |
| 556 return true; | |
| 557 } | |
| 558 | |
| 559 //------------------------------independent--------------------------- | |
| 560 // Is there no data path from s1 to s2 or s2 to s1? | |
| 561 bool SuperWord::independent(Node* s1, Node* s2) { | |
| 562 // assert(s1->Opcode() == s2->Opcode(), "check isomorphic first"); | |
| 563 int d1 = depth(s1); | |
| 564 int d2 = depth(s2); | |
| 565 if (d1 == d2) return s1 != s2; | |
| 566 Node* deep = d1 > d2 ? s1 : s2; | |
| 567 Node* shallow = d1 > d2 ? s2 : s1; | |
| 568 | |
| 569 visited_clear(); | |
| 570 | |
| 571 return independent_path(shallow, deep); | |
| 572 } | |
| 573 | |
| 574 //------------------------------independent_path------------------------------ | |
| 575 // Helper for independent | |
| 576 bool SuperWord::independent_path(Node* shallow, Node* deep, uint dp) { | |
| 577 if (dp >= 1000) return false; // stop deep recursion | |
| 578 visited_set(deep); | |
| 579 int shal_depth = depth(shallow); | |
| 580 assert(shal_depth <= depth(deep), "must be"); | |
| 581 for (DepPreds preds(deep, _dg); !preds.done(); preds.next()) { | |
| 582 Node* pred = preds.current(); | |
| 583 if (in_bb(pred) && !visited_test(pred)) { | |
| 584 if (shallow == pred) { | |
| 585 return false; | |
| 586 } | |
| 587 if (shal_depth < depth(pred) && !independent_path(shallow, pred, dp+1)) { | |
| 588 return false; | |
| 589 } | |
| 590 } | |
| 591 } | |
| 592 return true; | |
| 593 } | |
| 594 | |
| 595 //------------------------------set_alignment--------------------------- | |
| 596 void SuperWord::set_alignment(Node* s1, Node* s2, int align) { | |
| 597 set_alignment(s1, align); | |
| 598 set_alignment(s2, align + data_size(s1)); | |
| 599 } | |
| 600 | |
| 601 //------------------------------data_size--------------------------- | |
| 602 int SuperWord::data_size(Node* s) { | |
| 603 const Type* t = velt_type(s); | |
| 604 BasicType bt = t->array_element_basic_type(); | |
|
29
d5fc211aea19
6633953: type2aelembytes{T_ADDRESS} should be 8 bytes in 64 bit VM
kvn
parents:
0
diff
changeset
|
605 int bsize = type2aelembytes(bt); |
| 0 | 606 assert(bsize != 0, "valid size"); |
| 607 return bsize; | |
| 608 } | |
| 609 | |
| 610 //------------------------------extend_packlist--------------------------- | |
| 611 // Extend packset by following use->def and def->use links from pack members. | |
| 612 void SuperWord::extend_packlist() { | |
| 613 bool changed; | |
| 614 do { | |
| 615 changed = false; | |
| 616 for (int i = 0; i < _packset.length(); i++) { | |
| 617 Node_List* p = _packset.at(i); | |
| 618 changed |= follow_use_defs(p); | |
| 619 changed |= follow_def_uses(p); | |
| 620 } | |
| 621 } while (changed); | |
| 622 | |
| 623 #ifndef PRODUCT | |
| 624 if (TraceSuperWord) { | |
| 625 tty->print_cr("\nAfter extend_packlist"); | |
| 626 print_packset(); | |
| 627 } | |
| 628 #endif | |
| 629 } | |
| 630 | |
| 631 //------------------------------follow_use_defs--------------------------- | |
| 632 // Extend the packset by visiting operand definitions of nodes in pack p | |
| 633 bool SuperWord::follow_use_defs(Node_List* p) { | |
| 634 Node* s1 = p->at(0); | |
| 635 Node* s2 = p->at(1); | |
| 636 assert(p->size() == 2, "just checking"); | |
| 637 assert(s1->req() == s2->req(), "just checking"); | |
| 638 assert(alignment(s1) + data_size(s1) == alignment(s2), "just checking"); | |
| 639 | |
| 640 if (s1->is_Load()) return false; | |
| 641 | |
| 642 int align = alignment(s1); | |
| 643 bool changed = false; | |
| 644 int start = s1->is_Store() ? MemNode::ValueIn : 1; | |
| 645 int end = s1->is_Store() ? MemNode::ValueIn+1 : s1->req(); | |
| 646 for (int j = start; j < end; j++) { | |
| 647 Node* t1 = s1->in(j); | |
| 648 Node* t2 = s2->in(j); | |
| 649 if (!in_bb(t1) || !in_bb(t2)) | |
| 650 continue; | |
| 651 if (stmts_can_pack(t1, t2, align)) { | |
| 652 if (est_savings(t1, t2) >= 0) { | |
| 653 Node_List* pair = new Node_List(); | |
| 654 pair->push(t1); | |
| 655 pair->push(t2); | |
| 656 _packset.append(pair); | |
| 657 set_alignment(t1, t2, align); | |
| 658 changed = true; | |
| 659 } | |
| 660 } | |
| 661 } | |
| 662 return changed; | |
| 663 } | |
| 664 | |
| 665 //------------------------------follow_def_uses--------------------------- | |
| 666 // Extend the packset by visiting uses of nodes in pack p | |
| 667 bool SuperWord::follow_def_uses(Node_List* p) { | |
| 668 bool changed = false; | |
| 669 Node* s1 = p->at(0); | |
| 670 Node* s2 = p->at(1); | |
| 671 assert(p->size() == 2, "just checking"); | |
| 672 assert(s1->req() == s2->req(), "just checking"); | |
| 673 assert(alignment(s1) + data_size(s1) == alignment(s2), "just checking"); | |
| 674 | |
| 675 if (s1->is_Store()) return false; | |
| 676 | |
| 677 int align = alignment(s1); | |
| 678 int savings = -1; | |
| 679 Node* u1 = NULL; | |
| 680 Node* u2 = NULL; | |
| 681 for (DUIterator_Fast imax, i = s1->fast_outs(imax); i < imax; i++) { | |
| 682 Node* t1 = s1->fast_out(i); | |
| 683 if (!in_bb(t1)) continue; | |
| 684 for (DUIterator_Fast jmax, j = s2->fast_outs(jmax); j < jmax; j++) { | |
| 685 Node* t2 = s2->fast_out(j); | |
| 686 if (!in_bb(t2)) continue; | |
| 687 if (!opnd_positions_match(s1, t1, s2, t2)) | |
| 688 continue; | |
| 689 if (stmts_can_pack(t1, t2, align)) { | |
| 690 int my_savings = est_savings(t1, t2); | |
| 691 if (my_savings > savings) { | |
| 692 savings = my_savings; | |
| 693 u1 = t1; | |
| 694 u2 = t2; | |
| 695 } | |
| 696 } | |
| 697 } | |
| 698 } | |
| 699 if (savings >= 0) { | |
| 700 Node_List* pair = new Node_List(); | |
| 701 pair->push(u1); | |
| 702 pair->push(u2); | |
| 703 _packset.append(pair); | |
| 704 set_alignment(u1, u2, align); | |
| 705 changed = true; | |
| 706 } | |
| 707 return changed; | |
| 708 } | |
| 709 | |
| 710 //---------------------------opnd_positions_match------------------------- | |
| 711 // Is the use of d1 in u1 at the same operand position as d2 in u2? | |
| 712 bool SuperWord::opnd_positions_match(Node* d1, Node* u1, Node* d2, Node* u2) { | |
| 713 uint ct = u1->req(); | |
| 714 if (ct != u2->req()) return false; | |
| 715 uint i1 = 0; | |
| 716 uint i2 = 0; | |
| 717 do { | |
| 718 for (i1++; i1 < ct; i1++) if (u1->in(i1) == d1) break; | |
| 719 for (i2++; i2 < ct; i2++) if (u2->in(i2) == d2) break; | |
| 720 if (i1 != i2) { | |
| 721 return false; | |
| 722 } | |
| 723 } while (i1 < ct); | |
| 724 return true; | |
| 725 } | |
| 726 | |
| 727 //------------------------------est_savings--------------------------- | |
| 728 // Estimate the savings from executing s1 and s2 as a pack | |
| 729 int SuperWord::est_savings(Node* s1, Node* s2) { | |
| 730 int save = 2 - 1; // 2 operations per instruction in packed form | |
| 731 | |
| 732 // inputs | |
| 733 for (uint i = 1; i < s1->req(); i++) { | |
| 734 Node* x1 = s1->in(i); | |
| 735 Node* x2 = s2->in(i); | |
| 736 if (x1 != x2) { | |
| 737 if (are_adjacent_refs(x1, x2)) { | |
| 738 save += adjacent_profit(x1, x2); | |
| 739 } else if (!in_packset(x1, x2)) { | |
| 740 save -= pack_cost(2); | |
| 741 } else { | |
| 742 save += unpack_cost(2); | |
| 743 } | |
| 744 } | |
| 745 } | |
| 746 | |
| 747 // uses of result | |
| 748 uint ct = 0; | |
| 749 for (DUIterator_Fast imax, i = s1->fast_outs(imax); i < imax; i++) { | |
| 750 Node* s1_use = s1->fast_out(i); | |
| 751 for (int j = 0; j < _packset.length(); j++) { | |
| 752 Node_List* p = _packset.at(j); | |
| 753 if (p->at(0) == s1_use) { | |
| 754 for (DUIterator_Fast kmax, k = s2->fast_outs(kmax); k < kmax; k++) { | |
| 755 Node* s2_use = s2->fast_out(k); | |
| 756 if (p->at(p->size()-1) == s2_use) { | |
| 757 ct++; | |
| 758 if (are_adjacent_refs(s1_use, s2_use)) { | |
| 759 save += adjacent_profit(s1_use, s2_use); | |
| 760 } | |
| 761 } | |
| 762 } | |
| 763 } | |
| 764 } | |
| 765 } | |
| 766 | |
| 767 if (ct < s1->outcnt()) save += unpack_cost(1); | |
| 768 if (ct < s2->outcnt()) save += unpack_cost(1); | |
| 769 | |
| 770 return save; | |
| 771 } | |
| 772 | |
| 773 //------------------------------costs--------------------------- | |
| 774 int SuperWord::adjacent_profit(Node* s1, Node* s2) { return 2; } | |
| 775 int SuperWord::pack_cost(int ct) { return ct; } | |
| 776 int SuperWord::unpack_cost(int ct) { return ct; } | |
| 777 | |
| 778 //------------------------------combine_packs--------------------------- | |
| 779 // Combine packs A and B with A.last == B.first into A.first..,A.last,B.second,..B.last | |
| 780 void SuperWord::combine_packs() { | |
| 781 bool changed; | |
| 782 do { | |
| 783 changed = false; | |
| 784 for (int i = 0; i < _packset.length(); i++) { | |
| 785 Node_List* p1 = _packset.at(i); | |
| 786 if (p1 == NULL) continue; | |
| 787 for (int j = 0; j < _packset.length(); j++) { | |
| 788 Node_List* p2 = _packset.at(j); | |
| 789 if (p2 == NULL) continue; | |
| 790 if (p1->at(p1->size()-1) == p2->at(0)) { | |
| 791 for (uint k = 1; k < p2->size(); k++) { | |
| 792 p1->push(p2->at(k)); | |
| 793 } | |
| 794 _packset.at_put(j, NULL); | |
| 795 changed = true; | |
| 796 } | |
| 797 } | |
| 798 } | |
| 799 } while (changed); | |
| 800 | |
| 801 for (int i = _packset.length() - 1; i >= 0; i--) { | |
| 802 Node_List* p1 = _packset.at(i); | |
| 803 if (p1 == NULL) { | |
| 804 _packset.remove_at(i); | |
| 805 } | |
| 806 } | |
| 807 | |
| 808 #ifndef PRODUCT | |
| 809 if (TraceSuperWord) { | |
| 810 tty->print_cr("\nAfter combine_packs"); | |
| 811 print_packset(); | |
| 812 } | |
| 813 #endif | |
| 814 } | |
| 815 | |
| 816 //-----------------------------construct_my_pack_map-------------------------- | |
| 817 // Construct the map from nodes to packs. Only valid after the | |
| 818 // point where a node is only in one pack (after combine_packs). | |
| 819 void SuperWord::construct_my_pack_map() { | |
| 820 Node_List* rslt = NULL; | |
| 821 for (int i = 0; i < _packset.length(); i++) { | |
| 822 Node_List* p = _packset.at(i); | |
| 823 for (uint j = 0; j < p->size(); j++) { | |
| 824 Node* s = p->at(j); | |
| 825 assert(my_pack(s) == NULL, "only in one pack"); | |
| 826 set_my_pack(s, p); | |
| 827 } | |
| 828 } | |
| 829 } | |
| 830 | |
| 831 //------------------------------filter_packs--------------------------- | |
| 832 // Remove packs that are not implemented or not profitable. | |
| 833 void SuperWord::filter_packs() { | |
| 834 | |
| 835 // Remove packs that are not implemented | |
| 836 for (int i = _packset.length() - 1; i >= 0; i--) { | |
| 837 Node_List* pk = _packset.at(i); | |
| 838 bool impl = implemented(pk); | |
| 839 if (!impl) { | |
| 840 #ifndef PRODUCT | |
| 841 if (TraceSuperWord && Verbose) { | |
| 842 tty->print_cr("Unimplemented"); | |
| 843 pk->at(0)->dump(); | |
| 844 } | |
| 845 #endif | |
| 846 remove_pack_at(i); | |
| 847 } | |
| 848 } | |
| 849 | |
| 850 // Remove packs that are not profitable | |
| 851 bool changed; | |
| 852 do { | |
| 853 changed = false; | |
| 854 for (int i = _packset.length() - 1; i >= 0; i--) { | |
| 855 Node_List* pk = _packset.at(i); | |
| 856 bool prof = profitable(pk); | |
| 857 if (!prof) { | |
| 858 #ifndef PRODUCT | |
| 859 if (TraceSuperWord && Verbose) { | |
| 860 tty->print_cr("Unprofitable"); | |
| 861 pk->at(0)->dump(); | |
| 862 } | |
| 863 #endif | |
| 864 remove_pack_at(i); | |
| 865 changed = true; | |
| 866 } | |
| 867 } | |
| 868 } while (changed); | |
| 869 | |
| 870 #ifndef PRODUCT | |
| 871 if (TraceSuperWord) { | |
| 872 tty->print_cr("\nAfter filter_packs"); | |
| 873 print_packset(); | |
| 874 tty->cr(); | |
| 875 } | |
| 876 #endif | |
| 877 } | |
| 878 | |
| 879 //------------------------------implemented--------------------------- | |
| 880 // Can code be generated for pack p? | |
| 881 bool SuperWord::implemented(Node_List* p) { | |
| 882 Node* p0 = p->at(0); | |
| 883 int vopc = VectorNode::opcode(p0->Opcode(), p->size(), velt_type(p0)); | |
| 884 return vopc > 0 && Matcher::has_match_rule(vopc); | |
| 885 } | |
| 886 | |
| 887 //------------------------------profitable--------------------------- | |
| 888 // For pack p, are all operands and all uses (with in the block) vector? | |
| 889 bool SuperWord::profitable(Node_List* p) { | |
| 890 Node* p0 = p->at(0); | |
| 891 uint start, end; | |
| 892 vector_opd_range(p0, &start, &end); | |
| 893 | |
| 894 // Return false if some input is not vector and inside block | |
| 895 for (uint i = start; i < end; i++) { | |
| 896 if (!is_vector_use(p0, i)) { | |
| 897 // For now, return false if not scalar promotion case (inputs are the same.) | |
| 605 | 898 // Later, implement PackNode and allow differing, non-vector inputs |
| 0 | 899 // (maybe just the ones from outside the block.) |
| 900 Node* p0_def = p0->in(i); | |
| 901 for (uint j = 1; j < p->size(); j++) { | |
| 902 Node* use = p->at(j); | |
| 903 Node* def = use->in(i); | |
| 904 if (p0_def != def) | |
| 905 return false; | |
| 906 } | |
| 907 } | |
| 908 } | |
| 909 if (!p0->is_Store()) { | |
| 910 // For now, return false if not all uses are vector. | |
| 911 // Later, implement ExtractNode and allow non-vector uses (maybe | |
| 912 // just the ones outside the block.) | |
| 913 for (uint i = 0; i < p->size(); i++) { | |
| 914 Node* def = p->at(i); | |
| 915 for (DUIterator_Fast jmax, j = def->fast_outs(jmax); j < jmax; j++) { | |
| 916 Node* use = def->fast_out(j); | |
| 917 for (uint k = 0; k < use->req(); k++) { | |
| 918 Node* n = use->in(k); | |
| 919 if (def == n) { | |
| 920 if (!is_vector_use(use, k)) { | |
| 921 return false; | |
| 922 } | |
| 923 } | |
| 924 } | |
| 925 } | |
| 926 } | |
| 927 } | |
| 928 return true; | |
| 929 } | |
| 930 | |
| 931 //------------------------------schedule--------------------------- | |
| 932 // Adjust the memory graph for the packed operations | |
| 933 void SuperWord::schedule() { | |
| 934 | |
| 935 // Co-locate in the memory graph the members of each memory pack | |
| 936 for (int i = 0; i < _packset.length(); i++) { | |
| 937 co_locate_pack(_packset.at(i)); | |
| 938 } | |
| 939 } | |
| 940 | |
| 667 | 941 //-------------------------------remove_and_insert------------------- |
| 942 //remove "current" from its current position in the memory graph and insert | |
| 943 //it after the appropriate insertion point (lip or uip) | |
| 944 void SuperWord::remove_and_insert(MemNode *current, MemNode *prev, MemNode *lip, | |
| 945 Node *uip, Unique_Node_List &sched_before) { | |
| 946 Node* my_mem = current->in(MemNode::Memory); | |
| 947 _igvn.hash_delete(current); | |
| 948 _igvn.hash_delete(my_mem); | |
| 949 | |
| 950 //remove current_store from its current position in the memmory graph | |
| 951 for (DUIterator i = current->outs(); current->has_out(i); i++) { | |
| 952 Node* use = current->out(i); | |
| 953 if (use->is_Mem()) { | |
| 954 assert(use->in(MemNode::Memory) == current, "must be"); | |
| 955 _igvn.hash_delete(use); | |
| 956 if (use == prev) { // connect prev to my_mem | |
| 957 use->set_req(MemNode::Memory, my_mem); | |
| 958 } else if (sched_before.member(use)) { | |
| 959 _igvn.hash_delete(uip); | |
| 960 use->set_req(MemNode::Memory, uip); | |
| 961 } else { | |
| 962 _igvn.hash_delete(lip); | |
| 963 use->set_req(MemNode::Memory, lip); | |
| 964 } | |
| 965 _igvn._worklist.push(use); | |
| 966 --i; //deleted this edge; rescan position | |
| 967 } | |
| 968 } | |
| 969 | |
| 970 bool sched_up = sched_before.member(current); | |
| 971 Node *insert_pt = sched_up ? uip : lip; | |
| 972 _igvn.hash_delete(insert_pt); | |
| 973 | |
| 974 // all uses of insert_pt's memory state should use current's instead | |
| 975 for (DUIterator i = insert_pt->outs(); insert_pt->has_out(i); i++) { | |
| 976 Node* use = insert_pt->out(i); | |
| 977 if (use->is_Mem()) { | |
| 978 assert(use->in(MemNode::Memory) == insert_pt, "must be"); | |
| 979 _igvn.hash_delete(use); | |
| 980 use->set_req(MemNode::Memory, current); | |
| 981 _igvn._worklist.push(use); | |
| 982 --i; //deleted this edge; rescan position | |
| 983 } else if (!sched_up && use->is_Phi() && use->bottom_type() == Type::MEMORY) { | |
| 984 uint pos; //lip (lower insert point) must be the last one in the memory slice | |
| 985 _igvn.hash_delete(use); | |
| 986 for (pos=1; pos < use->req(); pos++) { | |
| 987 if (use->in(pos) == insert_pt) break; | |
| 988 } | |
| 989 use->set_req(pos, current); | |
| 990 _igvn._worklist.push(use); | |
| 991 --i; | |
| 992 } | |
| 993 } | |
| 994 | |
| 995 //connect current to insert_pt | |
| 996 current->set_req(MemNode::Memory, insert_pt); | |
| 997 _igvn._worklist.push(current); | |
| 998 } | |
| 999 | |
| 1000 //------------------------------co_locate_pack---------------------------------- | |
| 1001 // To schedule a store pack, we need to move any sandwiched memory ops either before | |
| 1002 // or after the pack, based upon dependence information: | |
| 1003 // (1) If any store in the pack depends on the sandwiched memory op, the | |
| 1004 // sandwiched memory op must be scheduled BEFORE the pack; | |
| 1005 // (2) If a sandwiched memory op depends on any store in the pack, the | |
| 1006 // sandwiched memory op must be scheduled AFTER the pack; | |
| 1007 // (3) If a sandwiched memory op (say, memA) depends on another sandwiched | |
| 1008 // memory op (say memB), memB must be scheduled before memA. So, if memA is | |
| 1009 // scheduled before the pack, memB must also be scheduled before the pack; | |
| 1010 // (4) If there is no dependence restriction for a sandwiched memory op, we simply | |
| 1011 // schedule this store AFTER the pack | |
| 1012 // (5) We know there is no dependence cycle, so there in no other case; | |
| 1013 // (6) Finally, all memory ops in another single pack should be moved in the same direction. | |
| 1014 // | |
| 952 | 1015 // To schedule a load pack, we use the memory state of either the first or the last load in |
| 1016 // the pack, based on the dependence constraint. | |
| 0 | 1017 void SuperWord::co_locate_pack(Node_List* pk) { |
| 1018 if (pk->at(0)->is_Store()) { | |
| 1019 MemNode* first = executed_first(pk)->as_Mem(); | |
| 1020 MemNode* last = executed_last(pk)->as_Mem(); | |
| 667 | 1021 Unique_Node_List schedule_before_pack; |
| 1022 Unique_Node_List memops; | |
| 1023 | |
| 0 | 1024 MemNode* current = last->in(MemNode::Memory)->as_Mem(); |
| 667 | 1025 MemNode* previous = last; |
| 0 | 1026 while (true) { |
| 1027 assert(in_bb(current), "stay in block"); | |
| 667 | 1028 memops.push(previous); |
| 1029 for (DUIterator i = current->outs(); current->has_out(i); i++) { | |
| 1030 Node* use = current->out(i); | |
| 1031 if (use->is_Mem() && use != previous) | |
| 1032 memops.push(use); | |
| 1033 } | |
| 1034 if(current == first) break; | |
| 1035 previous = current; | |
| 1036 current = current->in(MemNode::Memory)->as_Mem(); | |
| 1037 } | |
| 1038 | |
| 1039 // determine which memory operations should be scheduled before the pack | |
| 1040 for (uint i = 1; i < memops.size(); i++) { | |
| 1041 Node *s1 = memops.at(i); | |
| 1042 if (!in_pack(s1, pk) && !schedule_before_pack.member(s1)) { | |
| 1043 for (uint j = 0; j< i; j++) { | |
| 1044 Node *s2 = memops.at(j); | |
| 1045 if (!independent(s1, s2)) { | |
| 1046 if (in_pack(s2, pk) || schedule_before_pack.member(s2)) { | |
| 1047 schedule_before_pack.push(s1); //s1 must be scheduled before | |
| 1048 Node_List* mem_pk = my_pack(s1); | |
| 1049 if (mem_pk != NULL) { | |
| 1050 for (uint ii = 0; ii < mem_pk->size(); ii++) { | |
| 1051 Node* s = mem_pk->at(ii); // follow partner | |
| 1052 if (memops.member(s) && !schedule_before_pack.member(s)) | |
| 1053 schedule_before_pack.push(s); | |
| 1054 } | |
| 1055 } | |
| 1056 } | |
| 1057 } | |
| 1058 } | |
| 1059 } | |
| 1060 } | |
| 1061 | |
| 1062 MemNode* lower_insert_pt = last; | |
| 1063 Node* upper_insert_pt = first->in(MemNode::Memory); | |
| 1064 previous = last; //previous store in pk | |
| 1065 current = last->in(MemNode::Memory)->as_Mem(); | |
| 1066 | |
| 1067 //start scheduling from "last" to "first" | |
| 1068 while (true) { | |
| 1069 assert(in_bb(current), "stay in block"); | |
| 1070 assert(in_pack(previous, pk), "previous stays in pack"); | |
| 0 | 1071 Node* my_mem = current->in(MemNode::Memory); |
| 667 | 1072 |
| 0 | 1073 if (in_pack(current, pk)) { |
| 667 | 1074 // Forward users of my memory state (except "previous) to my input memory state |
| 0 | 1075 _igvn.hash_delete(current); |
| 1076 for (DUIterator i = current->outs(); current->has_out(i); i++) { | |
| 1077 Node* use = current->out(i); | |
| 667 | 1078 if (use->is_Mem() && use != previous) { |
| 0 | 1079 assert(use->in(MemNode::Memory) == current, "must be"); |
| 1080 _igvn.hash_delete(use); | |
| 667 | 1081 if (schedule_before_pack.member(use)) { |
| 1082 _igvn.hash_delete(upper_insert_pt); | |
| 1083 use->set_req(MemNode::Memory, upper_insert_pt); | |
| 1084 } else { | |
| 1085 _igvn.hash_delete(lower_insert_pt); | |
| 1086 use->set_req(MemNode::Memory, lower_insert_pt); | |
| 1087 } | |
| 0 | 1088 _igvn._worklist.push(use); |
| 1089 --i; // deleted this edge; rescan position | |
| 1090 } | |
| 1091 } | |
| 667 | 1092 previous = current; |
| 1093 } else { // !in_pack(current, pk) ==> a sandwiched store | |
| 1094 remove_and_insert(current, previous, lower_insert_pt, upper_insert_pt, schedule_before_pack); | |
| 0 | 1095 } |
| 667 | 1096 |
| 0 | 1097 if (current == first) break; |
| 1098 current = my_mem->as_Mem(); | |
| 667 | 1099 } // end while |
| 1100 } else if (pk->at(0)->is_Load()) { //load | |
| 952 | 1101 // all loads in the pack should have the same memory state. By default, |
| 1102 // we use the memory state of the last load. However, if any load could | |
| 1103 // not be moved down due to the dependence constraint, we use the memory | |
| 1104 // state of the first load. | |
| 1105 Node* last_mem = executed_last(pk)->in(MemNode::Memory); | |
| 1106 Node* first_mem = executed_first(pk)->in(MemNode::Memory); | |
| 1107 bool schedule_last = true; | |
| 1108 for (uint i = 0; i < pk->size(); i++) { | |
| 1109 Node* ld = pk->at(i); | |
| 1110 for (Node* current = last_mem; current != ld->in(MemNode::Memory); | |
| 1111 current=current->in(MemNode::Memory)) { | |
| 1112 assert(current != first_mem, "corrupted memory graph"); | |
| 1113 if(current->is_Mem() && !independent(current, ld)){ | |
| 1114 schedule_last = false; // a later store depends on this load | |
| 1115 break; | |
| 1116 } | |
| 1117 } | |
| 1118 } | |
| 1119 | |
| 1120 Node* mem_input = schedule_last ? last_mem : first_mem; | |
| 1121 _igvn.hash_delete(mem_input); | |
| 1122 // Give each load the same memory state | |
| 0 | 1123 for (uint i = 0; i < pk->size(); i++) { |
| 1124 LoadNode* ld = pk->at(i)->as_Load(); | |
| 1125 _igvn.hash_delete(ld); | |
| 952 | 1126 ld->set_req(MemNode::Memory, mem_input); |
| 0 | 1127 _igvn._worklist.push(ld); |
| 1128 } | |
| 1129 } | |
| 1130 } | |
| 1131 | |
| 1132 //------------------------------output--------------------------- | |
| 1133 // Convert packs into vector node operations | |
| 1134 void SuperWord::output() { | |
| 1135 if (_packset.length() == 0) return; | |
| 1136 | |
| 2445 | 1137 #ifndef PRODUCT |
| 1138 if (TraceLoopOpts) { | |
| 1139 tty->print("SuperWord "); | |
| 1140 lpt()->dump_head(); | |
| 1141 } | |
| 1142 #endif | |
| 1143 | |
| 0 | 1144 // MUST ENSURE main loop's initial value is properly aligned: |
| 1145 // (iv_initial_value + min_iv_offset) % vector_width_in_bytes() == 0 | |
| 1146 | |
| 1147 align_initial_loop_index(align_to_ref()); | |
| 1148 | |
| 1149 // Insert extract (unpack) operations for scalar uses | |
| 1150 for (int i = 0; i < _packset.length(); i++) { | |
| 1151 insert_extracts(_packset.at(i)); | |
| 1152 } | |
| 1153 | |
| 1154 for (int i = 0; i < _block.length(); i++) { | |
| 1155 Node* n = _block.at(i); | |
| 1156 Node_List* p = my_pack(n); | |
| 1157 if (p && n == executed_last(p)) { | |
| 1158 uint vlen = p->size(); | |
| 1159 Node* vn = NULL; | |
| 1160 Node* low_adr = p->at(0); | |
| 1161 Node* first = executed_first(p); | |
| 1162 if (n->is_Load()) { | |
| 1163 int opc = n->Opcode(); | |
| 1164 Node* ctl = n->in(MemNode::Control); | |
| 1165 Node* mem = first->in(MemNode::Memory); | |
| 1166 Node* adr = low_adr->in(MemNode::Address); | |
| 1167 const TypePtr* atyp = n->adr_type(); | |
| 1168 vn = VectorLoadNode::make(_phase->C, opc, ctl, mem, adr, atyp, vlen); | |
| 1169 | |
| 1170 } else if (n->is_Store()) { | |
| 1171 // Promote value to be stored to vector | |
| 3842 | 1172 Node* val = vector_opd(p, MemNode::ValueIn); |
| 0 | 1173 |
| 1174 int opc = n->Opcode(); | |
| 1175 Node* ctl = n->in(MemNode::Control); | |
| 1176 Node* mem = first->in(MemNode::Memory); | |
| 1177 Node* adr = low_adr->in(MemNode::Address); | |
| 1178 const TypePtr* atyp = n->adr_type(); | |
| 1179 vn = VectorStoreNode::make(_phase->C, opc, ctl, mem, adr, atyp, val, vlen); | |
| 1180 | |
| 1181 } else if (n->req() == 3) { | |
| 1182 // Promote operands to vector | |
| 1183 Node* in1 = vector_opd(p, 1); | |
| 1184 Node* in2 = vector_opd(p, 2); | |
| 1185 vn = VectorNode::make(_phase->C, n->Opcode(), in1, in2, vlen, velt_type(n)); | |
| 1186 | |
| 1187 } else { | |
| 1188 ShouldNotReachHere(); | |
| 1189 } | |
| 1190 | |
| 1191 _phase->_igvn.register_new_node_with_optimizer(vn); | |
| 1192 _phase->set_ctrl(vn, _phase->get_ctrl(p->at(0))); | |
| 1193 for (uint j = 0; j < p->size(); j++) { | |
| 1194 Node* pm = p->at(j); | |
|
1621
6027dddc26c6
6677629: PhaseIterGVN::subsume_node() should call hash_delete() and add_users_to_worklist()
kvn
parents:
1585
diff
changeset
|
1195 _igvn.replace_node(pm, vn); |
| 0 | 1196 } |
| 1197 _igvn._worklist.push(vn); | |
| 1198 } | |
| 1199 } | |
| 1200 } | |
| 1201 | |
| 1202 //------------------------------vector_opd--------------------------- | |
| 1203 // Create a vector operand for the nodes in pack p for operand: in(opd_idx) | |
| 3842 | 1204 Node* SuperWord::vector_opd(Node_List* p, int opd_idx) { |
| 0 | 1205 Node* p0 = p->at(0); |
| 1206 uint vlen = p->size(); | |
| 1207 Node* opd = p0->in(opd_idx); | |
| 1208 | |
| 1209 bool same_opd = true; | |
| 1210 for (uint i = 1; i < vlen; i++) { | |
| 1211 Node* pi = p->at(i); | |
| 1212 Node* in = pi->in(opd_idx); | |
| 1213 if (opd != in) { | |
| 1214 same_opd = false; | |
| 1215 break; | |
| 1216 } | |
| 1217 } | |
| 1218 | |
| 1219 if (same_opd) { | |
| 3842 | 1220 if (opd->is_Vector() || opd->is_VectorLoad()) { |
| 1221 return opd; // input is matching vector | |
| 0 | 1222 } |
| 3842 | 1223 assert(!opd->is_VectorStore(), "such vector is not expected here"); |
| 0 | 1224 // Convert scalar input to vector. Use p0's type because it's container |
| 1225 // maybe smaller than the operand's container. | |
| 1226 const Type* opd_t = velt_type(!in_bb(opd) ? p0 : opd); | |
| 1227 const Type* p0_t = velt_type(p0); | |
| 1228 if (p0_t->higher_equal(opd_t)) opd_t = p0_t; | |
| 1229 VectorNode* vn = VectorNode::scalar2vector(_phase->C, opd, vlen, opd_t); | |
| 1230 | |
| 1231 _phase->_igvn.register_new_node_with_optimizer(vn); | |
| 1232 _phase->set_ctrl(vn, _phase->get_ctrl(opd)); | |
| 1233 return vn; | |
| 1234 } | |
| 1235 | |
| 1236 // Insert pack operation | |
| 1237 const Type* opd_t = velt_type(!in_bb(opd) ? p0 : opd); | |
| 1238 PackNode* pk = PackNode::make(_phase->C, opd, opd_t); | |
| 1239 | |
| 1240 for (uint i = 1; i < vlen; i++) { | |
| 1241 Node* pi = p->at(i); | |
| 1242 Node* in = pi->in(opd_idx); | |
| 1243 assert(my_pack(in) == NULL, "Should already have been unpacked"); | |
| 1244 assert(opd_t == velt_type(!in_bb(in) ? pi : in), "all same type"); | |
| 1245 pk->add_opd(in); | |
| 1246 } | |
| 1247 _phase->_igvn.register_new_node_with_optimizer(pk); | |
| 1248 _phase->set_ctrl(pk, _phase->get_ctrl(opd)); | |
| 1249 return pk; | |
| 1250 } | |
| 1251 | |
| 1252 //------------------------------insert_extracts--------------------------- | |
| 1253 // If a use of pack p is not a vector use, then replace the | |
| 1254 // use with an extract operation. | |
| 1255 void SuperWord::insert_extracts(Node_List* p) { | |
| 1256 if (p->at(0)->is_Store()) return; | |
| 1257 assert(_n_idx_list.is_empty(), "empty (node,index) list"); | |
| 1258 | |
| 1259 // Inspect each use of each pack member. For each use that is | |
| 1260 // not a vector use, replace the use with an extract operation. | |
| 1261 | |
| 1262 for (uint i = 0; i < p->size(); i++) { | |
| 1263 Node* def = p->at(i); | |
| 1264 for (DUIterator_Fast jmax, j = def->fast_outs(jmax); j < jmax; j++) { | |
| 1265 Node* use = def->fast_out(j); | |
| 1266 for (uint k = 0; k < use->req(); k++) { | |
| 1267 Node* n = use->in(k); | |
| 1268 if (def == n) { | |
| 1269 if (!is_vector_use(use, k)) { | |
| 1270 _n_idx_list.push(use, k); | |
| 1271 } | |
| 1272 } | |
| 1273 } | |
| 1274 } | |
| 1275 } | |
| 1276 | |
| 1277 while (_n_idx_list.is_nonempty()) { | |
| 1278 Node* use = _n_idx_list.node(); | |
| 1279 int idx = _n_idx_list.index(); | |
| 1280 _n_idx_list.pop(); | |
| 1281 Node* def = use->in(idx); | |
| 1282 | |
| 1283 // Insert extract operation | |
| 1284 _igvn.hash_delete(def); | |
| 1285 _igvn.hash_delete(use); | |
| 1286 int def_pos = alignment(def) / data_size(def); | |
| 1287 const Type* def_t = velt_type(def); | |
| 1288 | |
| 1289 Node* ex = ExtractNode::make(_phase->C, def, def_pos, def_t); | |
| 1290 _phase->_igvn.register_new_node_with_optimizer(ex); | |
| 1291 _phase->set_ctrl(ex, _phase->get_ctrl(def)); | |
| 1292 use->set_req(idx, ex); | |
| 1293 _igvn._worklist.push(def); | |
| 1294 _igvn._worklist.push(use); | |
| 1295 | |
| 1296 bb_insert_after(ex, bb_idx(def)); | |
| 1297 set_velt_type(ex, def_t); | |
| 1298 } | |
| 1299 } | |
| 1300 | |
| 1301 //------------------------------is_vector_use--------------------------- | |
| 1302 // Is use->in(u_idx) a vector use? | |
| 1303 bool SuperWord::is_vector_use(Node* use, int u_idx) { | |
| 1304 Node_List* u_pk = my_pack(use); | |
| 1305 if (u_pk == NULL) return false; | |
| 1306 Node* def = use->in(u_idx); | |
| 1307 Node_List* d_pk = my_pack(def); | |
| 1308 if (d_pk == NULL) { | |
| 1309 // check for scalar promotion | |
| 1310 Node* n = u_pk->at(0)->in(u_idx); | |
| 1311 for (uint i = 1; i < u_pk->size(); i++) { | |
| 1312 if (u_pk->at(i)->in(u_idx) != n) return false; | |
| 1313 } | |
| 1314 return true; | |
| 1315 } | |
| 1316 if (u_pk->size() != d_pk->size()) | |
| 1317 return false; | |
| 1318 for (uint i = 0; i < u_pk->size(); i++) { | |
| 1319 Node* ui = u_pk->at(i); | |
| 1320 Node* di = d_pk->at(i); | |
| 1321 if (ui->in(u_idx) != di || alignment(ui) != alignment(di)) | |
| 1322 return false; | |
| 1323 } | |
| 1324 return true; | |
| 1325 } | |
| 1326 | |
| 1327 //------------------------------construct_bb--------------------------- | |
| 1328 // Construct reverse postorder list of block members | |
| 1329 void SuperWord::construct_bb() { | |
| 1330 Node* entry = bb(); | |
| 1331 | |
| 1332 assert(_stk.length() == 0, "stk is empty"); | |
| 1333 assert(_block.length() == 0, "block is empty"); | |
| 1334 assert(_data_entry.length() == 0, "data_entry is empty"); | |
| 1335 assert(_mem_slice_head.length() == 0, "mem_slice_head is empty"); | |
| 1336 assert(_mem_slice_tail.length() == 0, "mem_slice_tail is empty"); | |
| 1337 | |
| 1338 // Find non-control nodes with no inputs from within block, | |
| 1339 // create a temporary map from node _idx to bb_idx for use | |
| 1340 // by the visited and post_visited sets, | |
| 1341 // and count number of nodes in block. | |
| 1342 int bb_ct = 0; | |
| 1343 for (uint i = 0; i < lpt()->_body.size(); i++ ) { | |
| 1344 Node *n = lpt()->_body.at(i); | |
| 1345 set_bb_idx(n, i); // Create a temporary map | |
| 1346 if (in_bb(n)) { | |
| 1347 bb_ct++; | |
| 1348 if (!n->is_CFG()) { | |
| 1349 bool found = false; | |
| 1350 for (uint j = 0; j < n->req(); j++) { | |
| 1351 Node* def = n->in(j); | |
| 1352 if (def && in_bb(def)) { | |
| 1353 found = true; | |
| 1354 break; | |
| 1355 } | |
| 1356 } | |
| 1357 if (!found) { | |
| 1358 assert(n != entry, "can't be entry"); | |
| 1359 _data_entry.push(n); | |
| 1360 } | |
| 1361 } | |
| 1362 } | |
| 1363 } | |
| 1364 | |
| 1365 // Find memory slices (head and tail) | |
| 1366 for (DUIterator_Fast imax, i = lp()->fast_outs(imax); i < imax; i++) { | |
| 1367 Node *n = lp()->fast_out(i); | |
| 1368 if (in_bb(n) && (n->is_Phi() && n->bottom_type() == Type::MEMORY)) { | |
| 1369 Node* n_tail = n->in(LoopNode::LoopBackControl); | |
|
253
b0fe4deeb9fb
6726999: nsk/stress/jck12a/jck12a010 assert(n != null,"Bad immediate dominator info.")
kvn
parents:
235
diff
changeset
|
1370 if (n_tail != n->in(LoopNode::EntryControl)) { |
|
b0fe4deeb9fb
6726999: nsk/stress/jck12a/jck12a010 assert(n != null,"Bad immediate dominator info.")
kvn
parents:
235
diff
changeset
|
1371 _mem_slice_head.push(n); |
|
b0fe4deeb9fb
6726999: nsk/stress/jck12a/jck12a010 assert(n != null,"Bad immediate dominator info.")
kvn
parents:
235
diff
changeset
|
1372 _mem_slice_tail.push(n_tail); |
|
b0fe4deeb9fb
6726999: nsk/stress/jck12a/jck12a010 assert(n != null,"Bad immediate dominator info.")
kvn
parents:
235
diff
changeset
|
1373 } |
| 0 | 1374 } |
| 1375 } | |
| 1376 | |
| 1377 // Create an RPO list of nodes in block | |
| 1378 | |
| 1379 visited_clear(); | |
| 1380 post_visited_clear(); | |
| 1381 | |
| 1382 // Push all non-control nodes with no inputs from within block, then control entry | |
| 1383 for (int j = 0; j < _data_entry.length(); j++) { | |
| 1384 Node* n = _data_entry.at(j); | |
| 1385 visited_set(n); | |
| 1386 _stk.push(n); | |
| 1387 } | |
| 1388 visited_set(entry); | |
| 1389 _stk.push(entry); | |
| 1390 | |
| 1391 // Do a depth first walk over out edges | |
| 1392 int rpo_idx = bb_ct - 1; | |
| 1393 int size; | |
| 1394 while ((size = _stk.length()) > 0) { | |
| 1395 Node* n = _stk.top(); // Leave node on stack | |
| 1396 if (!visited_test_set(n)) { | |
| 1397 // forward arc in graph | |
| 1398 } else if (!post_visited_test(n)) { | |
| 1399 // cross or back arc | |
| 1400 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { | |
| 1401 Node *use = n->fast_out(i); | |
| 1402 if (in_bb(use) && !visited_test(use) && | |
| 1403 // Don't go around backedge | |
| 1404 (!use->is_Phi() || n == entry)) { | |
| 1405 _stk.push(use); | |
| 1406 } | |
| 1407 } | |
| 1408 if (_stk.length() == size) { | |
| 1409 // There were no additional uses, post visit node now | |
| 1410 _stk.pop(); // Remove node from stack | |
| 1411 assert(rpo_idx >= 0, ""); | |
| 1412 _block.at_put_grow(rpo_idx, n); | |
| 1413 rpo_idx--; | |
| 1414 post_visited_set(n); | |
| 1415 assert(rpo_idx >= 0 || _stk.is_empty(), ""); | |
| 1416 } | |
| 1417 } else { | |
| 1418 _stk.pop(); // Remove post-visited node from stack | |
| 1419 } | |
| 1420 } | |
| 1421 | |
| 1422 // Create real map of block indices for nodes | |
| 1423 for (int j = 0; j < _block.length(); j++) { | |
| 1424 Node* n = _block.at(j); | |
| 1425 set_bb_idx(n, j); | |
| 1426 } | |
| 1427 | |
| 1428 initialize_bb(); // Ensure extra info is allocated. | |
| 1429 | |
| 1430 #ifndef PRODUCT | |
| 1431 if (TraceSuperWord) { | |
| 1432 print_bb(); | |
| 1433 tty->print_cr("\ndata entry nodes: %s", _data_entry.length() > 0 ? "" : "NONE"); | |
| 1434 for (int m = 0; m < _data_entry.length(); m++) { | |
| 1435 tty->print("%3d ", m); | |
| 1436 _data_entry.at(m)->dump(); | |
| 1437 } | |
| 1438 tty->print_cr("\nmemory slices: %s", _mem_slice_head.length() > 0 ? "" : "NONE"); | |
| 1439 for (int m = 0; m < _mem_slice_head.length(); m++) { | |
| 1440 tty->print("%3d ", m); _mem_slice_head.at(m)->dump(); | |
| 1441 tty->print(" "); _mem_slice_tail.at(m)->dump(); | |
| 1442 } | |
| 1443 } | |
| 1444 #endif | |
| 1445 assert(rpo_idx == -1 && bb_ct == _block.length(), "all block members found"); | |
| 1446 } | |
| 1447 | |
| 1448 //------------------------------initialize_bb--------------------------- | |
| 1449 // Initialize per node info | |
| 1450 void SuperWord::initialize_bb() { | |
| 1451 Node* last = _block.at(_block.length() - 1); | |
| 1452 grow_node_info(bb_idx(last)); | |
| 1453 } | |
| 1454 | |
| 1455 //------------------------------bb_insert_after--------------------------- | |
| 1456 // Insert n into block after pos | |
| 1457 void SuperWord::bb_insert_after(Node* n, int pos) { | |
| 1458 int n_pos = pos + 1; | |
| 1459 // Make room | |
| 1460 for (int i = _block.length() - 1; i >= n_pos; i--) { | |
| 1461 _block.at_put_grow(i+1, _block.at(i)); | |
| 1462 } | |
| 1463 for (int j = _node_info.length() - 1; j >= n_pos; j--) { | |
| 1464 _node_info.at_put_grow(j+1, _node_info.at(j)); | |
| 1465 } | |
| 1466 // Set value | |
| 1467 _block.at_put_grow(n_pos, n); | |
| 1468 _node_info.at_put_grow(n_pos, SWNodeInfo::initial); | |
| 1469 // Adjust map from node->_idx to _block index | |
| 1470 for (int i = n_pos; i < _block.length(); i++) { | |
| 1471 set_bb_idx(_block.at(i), i); | |
| 1472 } | |
| 1473 } | |
| 1474 | |
| 1475 //------------------------------compute_max_depth--------------------------- | |
| 1476 // Compute max depth for expressions from beginning of block | |
| 1477 // Use to prune search paths during test for independence. | |
| 1478 void SuperWord::compute_max_depth() { | |
| 1479 int ct = 0; | |
| 1480 bool again; | |
| 1481 do { | |
| 1482 again = false; | |
| 1483 for (int i = 0; i < _block.length(); i++) { | |
| 1484 Node* n = _block.at(i); | |
| 1485 if (!n->is_Phi()) { | |
| 1486 int d_orig = depth(n); | |
| 1487 int d_in = 0; | |
| 1488 for (DepPreds preds(n, _dg); !preds.done(); preds.next()) { | |
| 1489 Node* pred = preds.current(); | |
| 1490 if (in_bb(pred)) { | |
| 1491 d_in = MAX2(d_in, depth(pred)); | |
| 1492 } | |
| 1493 } | |
| 1494 if (d_in + 1 != d_orig) { | |
| 1495 set_depth(n, d_in + 1); | |
| 1496 again = true; | |
| 1497 } | |
| 1498 } | |
| 1499 } | |
| 1500 ct++; | |
| 1501 } while (again); | |
| 1502 #ifndef PRODUCT | |
| 1503 if (TraceSuperWord && Verbose) | |
| 1504 tty->print_cr("compute_max_depth iterated: %d times", ct); | |
| 1505 #endif | |
| 1506 } | |
| 1507 | |
| 1508 //-------------------------compute_vector_element_type----------------------- | |
| 1509 // Compute necessary vector element type for expressions | |
| 1510 // This propagates backwards a narrower integer type when the | |
| 1511 // upper bits of the value are not needed. | |
| 1512 // Example: char a,b,c; a = b + c; | |
| 1513 // Normally the type of the add is integer, but for packed character | |
| 1514 // operations the type of the add needs to be char. | |
| 1515 void SuperWord::compute_vector_element_type() { | |
| 1516 #ifndef PRODUCT | |
| 1517 if (TraceSuperWord && Verbose) | |
| 1518 tty->print_cr("\ncompute_velt_type:"); | |
| 1519 #endif | |
| 1520 | |
| 1521 // Initial type | |
| 1522 for (int i = 0; i < _block.length(); i++) { | |
| 1523 Node* n = _block.at(i); | |
| 1524 const Type* t = n->is_Mem() ? Type::get_const_basic_type(n->as_Mem()->memory_type()) | |
| 1525 : _igvn.type(n); | |
| 1526 const Type* vt = container_type(t); | |
| 1527 set_velt_type(n, vt); | |
| 1528 } | |
| 1529 | |
| 1530 // Propagate narrowed type backwards through operations | |
| 1531 // that don't depend on higher order bits | |
| 1532 for (int i = _block.length() - 1; i >= 0; i--) { | |
| 1533 Node* n = _block.at(i); | |
| 1534 // Only integer types need be examined | |
| 1535 if (n->bottom_type()->isa_int()) { | |
| 1536 uint start, end; | |
| 1537 vector_opd_range(n, &start, &end); | |
| 1538 const Type* vt = velt_type(n); | |
| 1539 | |
| 1540 for (uint j = start; j < end; j++) { | |
| 1541 Node* in = n->in(j); | |
| 1542 // Don't propagate through a type conversion | |
| 1543 if (n->bottom_type() != in->bottom_type()) | |
| 1544 continue; | |
| 1545 switch(in->Opcode()) { | |
| 1546 case Op_AddI: case Op_AddL: | |
| 1547 case Op_SubI: case Op_SubL: | |
| 1548 case Op_MulI: case Op_MulL: | |
| 1549 case Op_AndI: case Op_AndL: | |
| 1550 case Op_OrI: case Op_OrL: | |
| 1551 case Op_XorI: case Op_XorL: | |
| 1552 case Op_LShiftI: case Op_LShiftL: | |
| 1553 case Op_CMoveI: case Op_CMoveL: | |
| 1554 if (in_bb(in)) { | |
| 1555 bool same_type = true; | |
| 1556 for (DUIterator_Fast kmax, k = in->fast_outs(kmax); k < kmax; k++) { | |
| 1557 Node *use = in->fast_out(k); | |
| 1558 if (!in_bb(use) || velt_type(use) != vt) { | |
| 1559 same_type = false; | |
| 1560 break; | |
| 1561 } | |
| 1562 } | |
| 1563 if (same_type) { | |
| 1564 set_velt_type(in, vt); | |
| 1565 } | |
| 1566 } | |
| 1567 } | |
| 1568 } | |
| 1569 } | |
| 1570 } | |
| 1571 #ifndef PRODUCT | |
| 1572 if (TraceSuperWord && Verbose) { | |
| 1573 for (int i = 0; i < _block.length(); i++) { | |
| 1574 Node* n = _block.at(i); | |
| 1575 velt_type(n)->dump(); | |
| 1576 tty->print("\t"); | |
| 1577 n->dump(); | |
| 1578 } | |
| 1579 } | |
| 1580 #endif | |
| 1581 } | |
| 1582 | |
| 1583 //------------------------------memory_alignment--------------------------- | |
| 1584 // Alignment within a vector memory reference | |
| 1585 int SuperWord::memory_alignment(MemNode* s, int iv_adjust_in_bytes) { | |
| 1586 SWPointer p(s, this); | |
| 1587 if (!p.valid()) { | |
| 1588 return bottom_align; | |
| 1589 } | |
| 1590 int offset = p.offset_in_bytes(); | |
| 1591 offset += iv_adjust_in_bytes; | |
| 1592 int off_rem = offset % vector_width_in_bytes(); | |
| 1593 int off_mod = off_rem >= 0 ? off_rem : off_rem + vector_width_in_bytes(); | |
| 1594 return off_mod; | |
| 1595 } | |
| 1596 | |
| 1597 //---------------------------container_type--------------------------- | |
| 1598 // Smallest type containing range of values | |
| 1599 const Type* SuperWord::container_type(const Type* t) { | |
|
221
1e026f8da827
6710487: More than half of JDI Regression tests hang with COOPs in -Xcomp mode
kvn
parents:
113
diff
changeset
|
1600 const Type* tp = t->make_ptr(); |
|
1e026f8da827
6710487: More than half of JDI Regression tests hang with COOPs in -Xcomp mode
kvn
parents:
113
diff
changeset
|
1601 if (tp && tp->isa_aryptr()) { |
|
1e026f8da827
6710487: More than half of JDI Regression tests hang with COOPs in -Xcomp mode
kvn
parents:
113
diff
changeset
|
1602 t = tp->is_aryptr()->elem(); |
| 0 | 1603 } |
| 1604 if (t->basic_type() == T_INT) { | |
| 1605 if (t->higher_equal(TypeInt::BOOL)) return TypeInt::BOOL; | |
| 1606 if (t->higher_equal(TypeInt::BYTE)) return TypeInt::BYTE; | |
| 1607 if (t->higher_equal(TypeInt::CHAR)) return TypeInt::CHAR; | |
| 1608 if (t->higher_equal(TypeInt::SHORT)) return TypeInt::SHORT; | |
| 1609 return TypeInt::INT; | |
| 1610 } | |
| 1611 return t; | |
| 1612 } | |
| 1613 | |
| 1614 //-------------------------vector_opd_range----------------------- | |
| 1615 // (Start, end] half-open range defining which operands are vector | |
| 1616 void SuperWord::vector_opd_range(Node* n, uint* start, uint* end) { | |
| 1617 switch (n->Opcode()) { | |
|
558
3b5ac9e7e6ea
6796746: rename LoadC (char) opcode class to LoadUS (unsigned short)
twisti
parents:
253
diff
changeset
|
1618 case Op_LoadB: case Op_LoadUS: |
| 0 | 1619 case Op_LoadI: case Op_LoadL: |
| 1620 case Op_LoadF: case Op_LoadD: | |
| 1621 case Op_LoadP: | |
| 1622 *start = 0; | |
| 1623 *end = 0; | |
| 1624 return; | |
| 1625 case Op_StoreB: case Op_StoreC: | |
| 1626 case Op_StoreI: case Op_StoreL: | |
| 1627 case Op_StoreF: case Op_StoreD: | |
| 1628 case Op_StoreP: | |
| 1629 *start = MemNode::ValueIn; | |
| 1630 *end = *start + 1; | |
| 1631 return; | |
| 1632 case Op_LShiftI: case Op_LShiftL: | |
| 1633 *start = 1; | |
| 1634 *end = 2; | |
| 1635 return; | |
| 1636 case Op_CMoveI: case Op_CMoveL: case Op_CMoveF: case Op_CMoveD: | |
| 1637 *start = 2; | |
| 1638 *end = n->req(); | |
| 1639 return; | |
| 1640 } | |
| 1641 *start = 1; | |
| 1642 *end = n->req(); // default is all operands | |
| 1643 } | |
| 1644 | |
| 1645 //------------------------------in_packset--------------------------- | |
| 1646 // Are s1 and s2 in a pack pair and ordered as s1,s2? | |
| 1647 bool SuperWord::in_packset(Node* s1, Node* s2) { | |
| 1648 for (int i = 0; i < _packset.length(); i++) { | |
| 1649 Node_List* p = _packset.at(i); | |
| 1650 assert(p->size() == 2, "must be"); | |
| 1651 if (p->at(0) == s1 && p->at(p->size()-1) == s2) { | |
| 1652 return true; | |
| 1653 } | |
| 1654 } | |
| 1655 return false; | |
| 1656 } | |
| 1657 | |
| 1658 //------------------------------in_pack--------------------------- | |
| 1659 // Is s in pack p? | |
| 1660 Node_List* SuperWord::in_pack(Node* s, Node_List* p) { | |
| 1661 for (uint i = 0; i < p->size(); i++) { | |
| 1662 if (p->at(i) == s) { | |
| 1663 return p; | |
| 1664 } | |
| 1665 } | |
| 1666 return NULL; | |
| 1667 } | |
| 1668 | |
| 1669 //------------------------------remove_pack_at--------------------------- | |
| 1670 // Remove the pack at position pos in the packset | |
| 1671 void SuperWord::remove_pack_at(int pos) { | |
| 1672 Node_List* p = _packset.at(pos); | |
| 1673 for (uint i = 0; i < p->size(); i++) { | |
| 1674 Node* s = p->at(i); | |
| 1675 set_my_pack(s, NULL); | |
| 1676 } | |
| 1677 _packset.remove_at(pos); | |
| 1678 } | |
| 1679 | |
| 1680 //------------------------------executed_first--------------------------- | |
| 1681 // Return the node executed first in pack p. Uses the RPO block list | |
| 1682 // to determine order. | |
| 1683 Node* SuperWord::executed_first(Node_List* p) { | |
| 1684 Node* n = p->at(0); | |
| 1685 int n_rpo = bb_idx(n); | |
| 1686 for (uint i = 1; i < p->size(); i++) { | |
| 1687 Node* s = p->at(i); | |
| 1688 int s_rpo = bb_idx(s); | |
| 1689 if (s_rpo < n_rpo) { | |
| 1690 n = s; | |
| 1691 n_rpo = s_rpo; | |
| 1692 } | |
| 1693 } | |
| 1694 return n; | |
| 1695 } | |
| 1696 | |
| 1697 //------------------------------executed_last--------------------------- | |
| 1698 // Return the node executed last in pack p. | |
| 1699 Node* SuperWord::executed_last(Node_List* p) { | |
| 1700 Node* n = p->at(0); | |
| 1701 int n_rpo = bb_idx(n); | |
| 1702 for (uint i = 1; i < p->size(); i++) { | |
| 1703 Node* s = p->at(i); | |
| 1704 int s_rpo = bb_idx(s); | |
| 1705 if (s_rpo > n_rpo) { | |
| 1706 n = s; | |
| 1707 n_rpo = s_rpo; | |
| 1708 } | |
| 1709 } | |
| 1710 return n; | |
| 1711 } | |
| 1712 | |
| 1713 //----------------------------align_initial_loop_index--------------------------- | |
| 1714 // Adjust pre-loop limit so that in main loop, a load/store reference | |
| 1715 // to align_to_ref will be a position zero in the vector. | |
| 1716 // (iv + k) mod vector_align == 0 | |
| 1717 void SuperWord::align_initial_loop_index(MemNode* align_to_ref) { | |
| 1718 CountedLoopNode *main_head = lp()->as_CountedLoop(); | |
| 1719 assert(main_head->is_main_loop(), ""); | |
| 1720 CountedLoopEndNode* pre_end = get_pre_loop_end(main_head); | |
| 1721 assert(pre_end != NULL, ""); | |
| 1722 Node *pre_opaq1 = pre_end->limit(); | |
| 1723 assert(pre_opaq1->Opcode() == Op_Opaque1, ""); | |
| 1724 Opaque1Node *pre_opaq = (Opaque1Node*)pre_opaq1; | |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1725 Node *lim0 = pre_opaq->in(1); |
| 0 | 1726 |
| 1727 // Where we put new limit calculations | |
| 1728 Node *pre_ctrl = pre_end->loopnode()->in(LoopNode::EntryControl); | |
| 1729 | |
| 1730 // Ensure the original loop limit is available from the | |
| 1731 // pre-loop Opaque1 node. | |
| 1732 Node *orig_limit = pre_opaq->original_loop_limit(); | |
| 1733 assert(orig_limit != NULL && _igvn.type(orig_limit) != Type::TOP, ""); | |
| 1734 | |
| 1735 SWPointer align_to_ref_p(align_to_ref, this); | |
| 1736 | |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1737 // Given: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1738 // lim0 == original pre loop limit |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1739 // V == v_align (power of 2) |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1740 // invar == extra invariant piece of the address expression |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1741 // e == k [ +/- invar ] |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1742 // |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1743 // When reassociating expressions involving '%' the basic rules are: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1744 // (a - b) % k == 0 => a % k == b % k |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1745 // and: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1746 // (a + b) % k == 0 => a % k == (k - b) % k |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1747 // |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1748 // For stride > 0 && scale > 0, |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1749 // Derive the new pre-loop limit "lim" such that the two constraints: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1750 // (1) lim = lim0 + N (where N is some positive integer < V) |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1751 // (2) (e + lim) % V == 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1752 // are true. |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1753 // |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1754 // Substituting (1) into (2), |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1755 // (e + lim0 + N) % V == 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1756 // solve for N: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1757 // N = (V - (e + lim0)) % V |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1758 // substitute back into (1), so that new limit |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1759 // lim = lim0 + (V - (e + lim0)) % V |
| 0 | 1760 // |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1761 // For stride > 0 && scale < 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1762 // Constraints: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1763 // lim = lim0 + N |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1764 // (e - lim) % V == 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1765 // Solving for lim: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1766 // (e - lim0 - N) % V == 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1767 // N = (e - lim0) % V |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1768 // lim = lim0 + (e - lim0) % V |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1769 // |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1770 // For stride < 0 && scale > 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1771 // Constraints: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1772 // lim = lim0 - N |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1773 // (e + lim) % V == 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1774 // Solving for lim: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1775 // (e + lim0 - N) % V == 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1776 // N = (e + lim0) % V |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1777 // lim = lim0 - (e + lim0) % V |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1778 // |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1779 // For stride < 0 && scale < 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1780 // Constraints: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1781 // lim = lim0 - N |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1782 // (e - lim) % V == 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1783 // Solving for lim: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1784 // (e - lim0 + N) % V == 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1785 // N = (V - (e - lim0)) % V |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1786 // lim = lim0 - (V - (e - lim0)) % V |
| 0 | 1787 |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1788 int stride = iv_stride(); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1789 int scale = align_to_ref_p.scale_in_bytes(); |
| 0 | 1790 int elt_size = align_to_ref_p.memory_size(); |
| 1791 int v_align = vector_width_in_bytes() / elt_size; | |
| 1792 int k = align_to_ref_p.offset_in_bytes() / elt_size; | |
| 1793 | |
| 1794 Node *kn = _igvn.intcon(k); | |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1795 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1796 Node *e = kn; |
| 0 | 1797 if (align_to_ref_p.invar() != NULL) { |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1798 // incorporate any extra invariant piece producing k +/- invar >>> log2(elt) |
| 0 | 1799 Node* log2_elt = _igvn.intcon(exact_log2(elt_size)); |
| 1800 Node* aref = new (_phase->C, 3) URShiftINode(align_to_ref_p.invar(), log2_elt); | |
| 1801 _phase->_igvn.register_new_node_with_optimizer(aref); | |
| 1802 _phase->set_ctrl(aref, pre_ctrl); | |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1803 if (align_to_ref_p.negate_invar()) { |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1804 e = new (_phase->C, 3) SubINode(e, aref); |
| 0 | 1805 } else { |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1806 e = new (_phase->C, 3) AddINode(e, aref); |
| 0 | 1807 } |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1808 _phase->_igvn.register_new_node_with_optimizer(e); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1809 _phase->set_ctrl(e, pre_ctrl); |
| 0 | 1810 } |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1811 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1812 // compute e +/- lim0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1813 if (scale < 0) { |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1814 e = new (_phase->C, 3) SubINode(e, lim0); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1815 } else { |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1816 e = new (_phase->C, 3) AddINode(e, lim0); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1817 } |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1818 _phase->_igvn.register_new_node_with_optimizer(e); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1819 _phase->set_ctrl(e, pre_ctrl); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1820 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1821 if (stride * scale > 0) { |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1822 // compute V - (e +/- lim0) |
| 0 | 1823 Node* va = _igvn.intcon(v_align); |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1824 e = new (_phase->C, 3) SubINode(va, e); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1825 _phase->_igvn.register_new_node_with_optimizer(e); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1826 _phase->set_ctrl(e, pre_ctrl); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1827 } |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1828 // compute N = (exp) % V |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1829 Node* va_msk = _igvn.intcon(v_align - 1); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1830 Node* N = new (_phase->C, 3) AndINode(e, va_msk); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1831 _phase->_igvn.register_new_node_with_optimizer(N); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1832 _phase->set_ctrl(N, pre_ctrl); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1833 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1834 // substitute back into (1), so that new limit |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1835 // lim = lim0 + N |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1836 Node* lim; |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1837 if (stride < 0) { |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1838 lim = new (_phase->C, 3) SubINode(lim0, N); |
| 0 | 1839 } else { |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1840 lim = new (_phase->C, 3) AddINode(lim0, N); |
| 0 | 1841 } |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1842 _phase->_igvn.register_new_node_with_optimizer(lim); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1843 _phase->set_ctrl(lim, pre_ctrl); |
| 0 | 1844 Node* constrained = |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1845 (stride > 0) ? (Node*) new (_phase->C,3) MinINode(lim, orig_limit) |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1846 : (Node*) new (_phase->C,3) MaxINode(lim, orig_limit); |
| 0 | 1847 _phase->_igvn.register_new_node_with_optimizer(constrained); |
| 1848 _phase->set_ctrl(constrained, pre_ctrl); | |
| 1849 _igvn.hash_delete(pre_opaq); | |
| 1850 pre_opaq->set_req(1, constrained); | |
| 1851 } | |
| 1852 | |
| 1853 //----------------------------get_pre_loop_end--------------------------- | |
| 1854 // Find pre loop end from main loop. Returns null if none. | |
| 1855 CountedLoopEndNode* SuperWord::get_pre_loop_end(CountedLoopNode *cl) { | |
| 1856 Node *ctrl = cl->in(LoopNode::EntryControl); | |
| 1857 if (!ctrl->is_IfTrue() && !ctrl->is_IfFalse()) return NULL; | |
| 1858 Node *iffm = ctrl->in(0); | |
| 1859 if (!iffm->is_If()) return NULL; | |
| 1860 Node *p_f = iffm->in(0); | |
| 1861 if (!p_f->is_IfFalse()) return NULL; | |
| 1862 if (!p_f->in(0)->is_CountedLoopEnd()) return NULL; | |
| 1863 CountedLoopEndNode *pre_end = p_f->in(0)->as_CountedLoopEnd(); | |
| 1864 if (!pre_end->loopnode()->is_pre_loop()) return NULL; | |
| 1865 return pre_end; | |
| 1866 } | |
| 1867 | |
| 1868 | |
| 1869 //------------------------------init--------------------------- | |
| 1870 void SuperWord::init() { | |
| 1871 _dg.init(); | |
| 1872 _packset.clear(); | |
| 1873 _disjoint_ptrs.clear(); | |
| 1874 _block.clear(); | |
| 1875 _data_entry.clear(); | |
| 1876 _mem_slice_head.clear(); | |
| 1877 _mem_slice_tail.clear(); | |
| 1878 _node_info.clear(); | |
| 1879 _align_to_ref = NULL; | |
| 1880 _lpt = NULL; | |
| 1881 _lp = NULL; | |
| 1882 _bb = NULL; | |
| 1883 _iv = NULL; | |
| 1884 } | |
| 1885 | |
| 1886 //------------------------------print_packset--------------------------- | |
| 1887 void SuperWord::print_packset() { | |
| 1888 #ifndef PRODUCT | |
| 1889 tty->print_cr("packset"); | |
| 1890 for (int i = 0; i < _packset.length(); i++) { | |
| 1891 tty->print_cr("Pack: %d", i); | |
| 1892 Node_List* p = _packset.at(i); | |
| 1893 print_pack(p); | |
| 1894 } | |
| 1895 #endif | |
| 1896 } | |
| 1897 | |
| 1898 //------------------------------print_pack--------------------------- | |
| 1899 void SuperWord::print_pack(Node_List* p) { | |
| 1900 for (uint i = 0; i < p->size(); i++) { | |
| 1901 print_stmt(p->at(i)); | |
| 1902 } | |
| 1903 } | |
| 1904 | |
| 1905 //------------------------------print_bb--------------------------- | |
| 1906 void SuperWord::print_bb() { | |
| 1907 #ifndef PRODUCT | |
| 1908 tty->print_cr("\nBlock"); | |
| 1909 for (int i = 0; i < _block.length(); i++) { | |
| 1910 Node* n = _block.at(i); | |
| 1911 tty->print("%d ", i); | |
| 1912 if (n) { | |
| 1913 n->dump(); | |
| 1914 } | |
| 1915 } | |
| 1916 #endif | |
| 1917 } | |
| 1918 | |
| 1919 //------------------------------print_stmt--------------------------- | |
| 1920 void SuperWord::print_stmt(Node* s) { | |
| 1921 #ifndef PRODUCT | |
| 1922 tty->print(" align: %d \t", alignment(s)); | |
| 1923 s->dump(); | |
| 1924 #endif | |
| 1925 } | |
| 1926 | |
| 1927 //------------------------------blank--------------------------- | |
| 1928 char* SuperWord::blank(uint depth) { | |
| 1929 static char blanks[101]; | |
| 1930 assert(depth < 101, "too deep"); | |
| 1931 for (uint i = 0; i < depth; i++) blanks[i] = ' '; | |
| 1932 blanks[depth] = '\0'; | |
| 1933 return blanks; | |
| 1934 } | |
| 1935 | |
| 1936 | |
| 1937 //==============================SWPointer=========================== | |
| 1938 | |
| 1939 //----------------------------SWPointer------------------------ | |
| 1940 SWPointer::SWPointer(MemNode* mem, SuperWord* slp) : | |
| 1941 _mem(mem), _slp(slp), _base(NULL), _adr(NULL), | |
| 1942 _scale(0), _offset(0), _invar(NULL), _negate_invar(false) { | |
| 1943 | |
| 1944 Node* adr = mem->in(MemNode::Address); | |
| 1945 if (!adr->is_AddP()) { | |
| 1946 assert(!valid(), "too complex"); | |
| 1947 return; | |
| 1948 } | |
| 1949 // Match AddP(base, AddP(ptr, k*iv [+ invariant]), constant) | |
| 1950 Node* base = adr->in(AddPNode::Base); | |
|
1058
73a726751507
6852078: HSX 14/16 in jdk 5.0: api/javax_management api/org_omg jck tests crashes or make tnameserv crash
cfang
parents:
987
diff
changeset
|
1951 //unsafe reference could not be aligned appropriately without runtime checking |
|
73a726751507
6852078: HSX 14/16 in jdk 5.0: api/javax_management api/org_omg jck tests crashes or make tnameserv crash
cfang
parents:
987
diff
changeset
|
1952 if (base == NULL || base->bottom_type() == Type::TOP) { |
|
73a726751507
6852078: HSX 14/16 in jdk 5.0: api/javax_management api/org_omg jck tests crashes or make tnameserv crash
cfang
parents:
987
diff
changeset
|
1953 assert(!valid(), "unsafe access"); |
|
73a726751507
6852078: HSX 14/16 in jdk 5.0: api/javax_management api/org_omg jck tests crashes or make tnameserv crash
cfang
parents:
987
diff
changeset
|
1954 return; |
|
73a726751507
6852078: HSX 14/16 in jdk 5.0: api/javax_management api/org_omg jck tests crashes or make tnameserv crash
cfang
parents:
987
diff
changeset
|
1955 } |
| 0 | 1956 for (int i = 0; i < 3; i++) { |
| 1957 if (!scaled_iv_plus_offset(adr->in(AddPNode::Offset))) { | |
| 1958 assert(!valid(), "too complex"); | |
| 1959 return; | |
| 1960 } | |
| 1961 adr = adr->in(AddPNode::Address); | |
| 1962 if (base == adr || !adr->is_AddP()) { | |
| 1963 break; // stop looking at addp's | |
| 1964 } | |
| 1965 } | |
| 1966 _base = base; | |
| 1967 _adr = adr; | |
| 1968 assert(valid(), "Usable"); | |
| 1969 } | |
| 1970 | |
| 1971 // Following is used to create a temporary object during | |
| 1972 // the pattern match of an address expression. | |
| 1973 SWPointer::SWPointer(SWPointer* p) : | |
| 1974 _mem(p->_mem), _slp(p->_slp), _base(NULL), _adr(NULL), | |
| 1975 _scale(0), _offset(0), _invar(NULL), _negate_invar(false) {} | |
| 1976 | |
| 1977 //------------------------scaled_iv_plus_offset-------------------- | |
| 1978 // Match: k*iv + offset | |
| 1979 // where: k is a constant that maybe zero, and | |
| 1980 // offset is (k2 [+/- invariant]) where k2 maybe zero and invariant is optional | |
| 1981 bool SWPointer::scaled_iv_plus_offset(Node* n) { | |
| 1982 if (scaled_iv(n)) { | |
| 1983 return true; | |
| 1984 } | |
| 1985 if (offset_plus_k(n)) { | |
| 1986 return true; | |
| 1987 } | |
| 1988 int opc = n->Opcode(); | |
| 1989 if (opc == Op_AddI) { | |
| 1990 if (scaled_iv(n->in(1)) && offset_plus_k(n->in(2))) { | |
| 1991 return true; | |
| 1992 } | |
| 1993 if (scaled_iv(n->in(2)) && offset_plus_k(n->in(1))) { | |
| 1994 return true; | |
| 1995 } | |
| 1996 } else if (opc == Op_SubI) { | |
| 1997 if (scaled_iv(n->in(1)) && offset_plus_k(n->in(2), true)) { | |
| 1998 return true; | |
| 1999 } | |
| 2000 if (scaled_iv(n->in(2)) && offset_plus_k(n->in(1))) { | |
| 2001 _scale *= -1; | |
| 2002 return true; | |
| 2003 } | |
| 2004 } | |
| 2005 return false; | |
| 2006 } | |
| 2007 | |
| 2008 //----------------------------scaled_iv------------------------ | |
| 2009 // Match: k*iv where k is a constant that's not zero | |
| 2010 bool SWPointer::scaled_iv(Node* n) { | |
| 2011 if (_scale != 0) { | |
| 2012 return false; // already found a scale | |
| 2013 } | |
| 2014 if (n == iv()) { | |
| 2015 _scale = 1; | |
| 2016 return true; | |
| 2017 } | |
| 2018 int opc = n->Opcode(); | |
| 2019 if (opc == Op_MulI) { | |
| 2020 if (n->in(1) == iv() && n->in(2)->is_Con()) { | |
| 2021 _scale = n->in(2)->get_int(); | |
| 2022 return true; | |
| 2023 } else if (n->in(2) == iv() && n->in(1)->is_Con()) { | |
| 2024 _scale = n->in(1)->get_int(); | |
| 2025 return true; | |
| 2026 } | |
| 2027 } else if (opc == Op_LShiftI) { | |
| 2028 if (n->in(1) == iv() && n->in(2)->is_Con()) { | |
| 2029 _scale = 1 << n->in(2)->get_int(); | |
| 2030 return true; | |
| 2031 } | |
| 2032 } else if (opc == Op_ConvI2L) { | |
| 2033 if (scaled_iv_plus_offset(n->in(1))) { | |
| 2034 return true; | |
| 2035 } | |
| 2036 } else if (opc == Op_LShiftL) { | |
| 2037 if (!has_iv() && _invar == NULL) { | |
| 2038 // Need to preserve the current _offset value, so | |
| 2039 // create a temporary object for this expression subtree. | |
| 2040 // Hacky, so should re-engineer the address pattern match. | |
| 2041 SWPointer tmp(this); | |
| 2042 if (tmp.scaled_iv_plus_offset(n->in(1))) { | |
| 2043 if (tmp._invar == NULL) { | |
| 2044 int mult = 1 << n->in(2)->get_int(); | |
| 2045 _scale = tmp._scale * mult; | |
| 2046 _offset += tmp._offset * mult; | |
| 2047 return true; | |
| 2048 } | |
| 2049 } | |
| 2050 } | |
| 2051 } | |
| 2052 return false; | |
| 2053 } | |
| 2054 | |
| 2055 //----------------------------offset_plus_k------------------------ | |
| 2056 // Match: offset is (k [+/- invariant]) | |
| 2057 // where k maybe zero and invariant is optional, but not both. | |
| 2058 bool SWPointer::offset_plus_k(Node* n, bool negate) { | |
| 2059 int opc = n->Opcode(); | |
| 2060 if (opc == Op_ConI) { | |
| 2061 _offset += negate ? -(n->get_int()) : n->get_int(); | |
| 2062 return true; | |
| 2063 } else if (opc == Op_ConL) { | |
| 2064 // Okay if value fits into an int | |
| 2065 const TypeLong* t = n->find_long_type(); | |
| 2066 if (t->higher_equal(TypeLong::INT)) { | |
| 2067 jlong loff = n->get_long(); | |
| 2068 jint off = (jint)loff; | |
| 2069 _offset += negate ? -off : loff; | |
| 2070 return true; | |
| 2071 } | |
| 2072 return false; | |
| 2073 } | |
| 2074 if (_invar != NULL) return false; // already have an invariant | |
| 2075 if (opc == Op_AddI) { | |
| 2076 if (n->in(2)->is_Con() && invariant(n->in(1))) { | |
| 2077 _negate_invar = negate; | |
| 2078 _invar = n->in(1); | |
| 2079 _offset += negate ? -(n->in(2)->get_int()) : n->in(2)->get_int(); | |
| 2080 return true; | |
| 2081 } else if (n->in(1)->is_Con() && invariant(n->in(2))) { | |
| 2082 _offset += negate ? -(n->in(1)->get_int()) : n->in(1)->get_int(); | |
| 2083 _negate_invar = negate; | |
| 2084 _invar = n->in(2); | |
| 2085 return true; | |
| 2086 } | |
| 2087 } | |
| 2088 if (opc == Op_SubI) { | |
| 2089 if (n->in(2)->is_Con() && invariant(n->in(1))) { | |
| 2090 _negate_invar = negate; | |
| 2091 _invar = n->in(1); | |
| 2092 _offset += !negate ? -(n->in(2)->get_int()) : n->in(2)->get_int(); | |
| 2093 return true; | |
| 2094 } else if (n->in(1)->is_Con() && invariant(n->in(2))) { | |
| 2095 _offset += negate ? -(n->in(1)->get_int()) : n->in(1)->get_int(); | |
| 2096 _negate_invar = !negate; | |
| 2097 _invar = n->in(2); | |
| 2098 return true; | |
| 2099 } | |
| 2100 } | |
| 2101 if (invariant(n)) { | |
| 2102 _negate_invar = negate; | |
| 2103 _invar = n; | |
| 2104 return true; | |
| 2105 } | |
| 2106 return false; | |
| 2107 } | |
| 2108 | |
| 2109 //----------------------------print------------------------ | |
| 2110 void SWPointer::print() { | |
| 2111 #ifndef PRODUCT | |
| 2112 tty->print("base: %d adr: %d scale: %d offset: %d invar: %c%d\n", | |
| 2113 _base != NULL ? _base->_idx : 0, | |
| 2114 _adr != NULL ? _adr->_idx : 0, | |
| 2115 _scale, _offset, | |
| 2116 _negate_invar?'-':'+', | |
| 2117 _invar != NULL ? _invar->_idx : 0); | |
| 2118 #endif | |
| 2119 } | |
| 2120 | |
| 2121 // ========================= OrderedPair ===================== | |
| 2122 | |
| 2123 const OrderedPair OrderedPair::initial; | |
| 2124 | |
| 2125 // ========================= SWNodeInfo ===================== | |
| 2126 | |
| 2127 const SWNodeInfo SWNodeInfo::initial; | |
| 2128 | |
| 2129 | |
| 2130 // ============================ DepGraph =========================== | |
| 2131 | |
| 2132 //------------------------------make_node--------------------------- | |
| 2133 // Make a new dependence graph node for an ideal node. | |
| 2134 DepMem* DepGraph::make_node(Node* node) { | |
| 2135 DepMem* m = new (_arena) DepMem(node); | |
| 2136 if (node != NULL) { | |
| 2137 assert(_map.at_grow(node->_idx) == NULL, "one init only"); | |
| 2138 _map.at_put_grow(node->_idx, m); | |
| 2139 } | |
| 2140 return m; | |
| 2141 } | |
| 2142 | |
| 2143 //------------------------------make_edge--------------------------- | |
| 2144 // Make a new dependence graph edge from dpred -> dsucc | |
| 2145 DepEdge* DepGraph::make_edge(DepMem* dpred, DepMem* dsucc) { | |
| 2146 DepEdge* e = new (_arena) DepEdge(dpred, dsucc, dsucc->in_head(), dpred->out_head()); | |
| 2147 dpred->set_out_head(e); | |
| 2148 dsucc->set_in_head(e); | |
| 2149 return e; | |
| 2150 } | |
| 2151 | |
| 2152 // ========================== DepMem ======================== | |
| 2153 | |
| 2154 //------------------------------in_cnt--------------------------- | |
| 2155 int DepMem::in_cnt() { | |
| 2156 int ct = 0; | |
| 2157 for (DepEdge* e = _in_head; e != NULL; e = e->next_in()) ct++; | |
| 2158 return ct; | |
| 2159 } | |
| 2160 | |
| 2161 //------------------------------out_cnt--------------------------- | |
| 2162 int DepMem::out_cnt() { | |
| 2163 int ct = 0; | |
| 2164 for (DepEdge* e = _out_head; e != NULL; e = e->next_out()) ct++; | |
| 2165 return ct; | |
| 2166 } | |
| 2167 | |
| 2168 //------------------------------print----------------------------- | |
| 2169 void DepMem::print() { | |
| 2170 #ifndef PRODUCT | |
| 2171 tty->print(" DepNode %d (", _node->_idx); | |
| 2172 for (DepEdge* p = _in_head; p != NULL; p = p->next_in()) { | |
| 2173 Node* pred = p->pred()->node(); | |
| 2174 tty->print(" %d", pred != NULL ? pred->_idx : 0); | |
| 2175 } | |
| 2176 tty->print(") ["); | |
| 2177 for (DepEdge* s = _out_head; s != NULL; s = s->next_out()) { | |
| 2178 Node* succ = s->succ()->node(); | |
| 2179 tty->print(" %d", succ != NULL ? succ->_idx : 0); | |
| 2180 } | |
| 2181 tty->print_cr(" ]"); | |
| 2182 #endif | |
| 2183 } | |
| 2184 | |
| 2185 // =========================== DepEdge ========================= | |
| 2186 | |
| 2187 //------------------------------DepPreds--------------------------- | |
| 2188 void DepEdge::print() { | |
| 2189 #ifndef PRODUCT | |
| 2190 tty->print_cr("DepEdge: %d [ %d ]", _pred->node()->_idx, _succ->node()->_idx); | |
| 2191 #endif | |
| 2192 } | |
| 2193 | |
| 2194 // =========================== DepPreds ========================= | |
| 2195 // Iterator over predecessor edges in the dependence graph. | |
| 2196 | |
| 2197 //------------------------------DepPreds--------------------------- | |
| 2198 DepPreds::DepPreds(Node* n, DepGraph& dg) { | |
| 2199 _n = n; | |
| 2200 _done = false; | |
| 2201 if (_n->is_Store() || _n->is_Load()) { | |
| 2202 _next_idx = MemNode::Address; | |
| 2203 _end_idx = n->req(); | |
| 2204 _dep_next = dg.dep(_n)->in_head(); | |
| 2205 } else if (_n->is_Mem()) { | |
| 2206 _next_idx = 0; | |
| 2207 _end_idx = 0; | |
| 2208 _dep_next = dg.dep(_n)->in_head(); | |
| 2209 } else { | |
| 2210 _next_idx = 1; | |
| 2211 _end_idx = _n->req(); | |
| 2212 _dep_next = NULL; | |
| 2213 } | |
| 2214 next(); | |
| 2215 } | |
| 2216 | |
| 2217 //------------------------------next--------------------------- | |
| 2218 void DepPreds::next() { | |
| 2219 if (_dep_next != NULL) { | |
| 2220 _current = _dep_next->pred()->node(); | |
| 2221 _dep_next = _dep_next->next_in(); | |
| 2222 } else if (_next_idx < _end_idx) { | |
| 2223 _current = _n->in(_next_idx++); | |
| 2224 } else { | |
| 2225 _done = true; | |
| 2226 } | |
| 2227 } | |
| 2228 | |
| 2229 // =========================== DepSuccs ========================= | |
| 2230 // Iterator over successor edges in the dependence graph. | |
| 2231 | |
| 2232 //------------------------------DepSuccs--------------------------- | |
| 2233 DepSuccs::DepSuccs(Node* n, DepGraph& dg) { | |
| 2234 _n = n; | |
| 2235 _done = false; | |
| 2236 if (_n->is_Load()) { | |
| 2237 _next_idx = 0; | |
| 2238 _end_idx = _n->outcnt(); | |
| 2239 _dep_next = dg.dep(_n)->out_head(); | |
| 2240 } else if (_n->is_Mem() || _n->is_Phi() && _n->bottom_type() == Type::MEMORY) { | |
| 2241 _next_idx = 0; | |
| 2242 _end_idx = 0; | |
| 2243 _dep_next = dg.dep(_n)->out_head(); | |
| 2244 } else { | |
| 2245 _next_idx = 0; | |
| 2246 _end_idx = _n->outcnt(); | |
| 2247 _dep_next = NULL; | |
| 2248 } | |
| 2249 next(); | |
| 2250 } | |
| 2251 | |
| 2252 //-------------------------------next--------------------------- | |
| 2253 void DepSuccs::next() { | |
| 2254 if (_dep_next != NULL) { | |
| 2255 _current = _dep_next->succ()->node(); | |
| 2256 _dep_next = _dep_next->next_out(); | |
| 2257 } else if (_next_idx < _end_idx) { | |
| 2258 _current = _n->raw_out(_next_idx++); | |
| 2259 } else { | |
| 2260 _done = true; | |
| 2261 } | |
| 2262 } |