Mercurial > hg > graal-compiler
annotate src/share/vm/opto/superword.cpp @ 1941:79d04223b8a5
Added caching for resolved types and resolved fields.
This is crucial, because the local load elimination will lead to wrong results, if field equality (of two RiField objects with the same object and the same RiType) is not given. The caching makes sure that the default equals implementation is sufficient.
| author | Thomas Wuerthinger <wuerthinger@ssw.jku.at> |
|---|---|
| date | Tue, 28 Dec 2010 18:33:26 +0100 |
| parents | 6027dddc26c6 |
| children | f95d63e2154a |
| rev | line source |
|---|---|
| 0 | 1 /* |
| 1585 | 2 * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. |
| 0 | 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| 4 * | |
| 5 * This code is free software; you can redistribute it and/or modify it | |
| 6 * under the terms of the GNU General Public License version 2 only, as | |
| 7 * published by the Free Software Foundation. | |
| 8 * | |
| 9 * This code is distributed in the hope that it will be useful, but WITHOUT | |
| 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
| 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
| 12 * version 2 for more details (a copy is included in the LICENSE file that | |
| 13 * accompanied this code). | |
| 14 * | |
| 15 * You should have received a copy of the GNU General Public License version | |
| 16 * 2 along with this work; if not, write to the Free Software Foundation, | |
| 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |
| 18 * | |
|
1552
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1058
diff
changeset
|
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
|
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1058
diff
changeset
|
20 * or visit www.oracle.com if you need additional information or have any |
|
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1058
diff
changeset
|
21 * questions. |
| 0 | 22 */ |
| 23 | |
| 24 #include "incls/_precompiled.incl" | |
| 25 #include "incls/_superword.cpp.incl" | |
| 26 | |
| 27 // | |
| 28 // S U P E R W O R D T R A N S F O R M | |
| 29 //============================================================================= | |
| 30 | |
| 31 //------------------------------SuperWord--------------------------- | |
| 32 SuperWord::SuperWord(PhaseIdealLoop* phase) : | |
| 33 _phase(phase), | |
| 34 _igvn(phase->_igvn), | |
| 35 _arena(phase->C->comp_arena()), | |
| 36 _packset(arena(), 8, 0, NULL), // packs for the current block | |
| 37 _bb_idx(arena(), (int)(1.10 * phase->C->unique()), 0, 0), // node idx to index in bb | |
| 38 _block(arena(), 8, 0, NULL), // nodes in current block | |
| 39 _data_entry(arena(), 8, 0, NULL), // nodes with all inputs from outside | |
| 40 _mem_slice_head(arena(), 8, 0, NULL), // memory slice heads | |
| 41 _mem_slice_tail(arena(), 8, 0, NULL), // memory slice tails | |
| 42 _node_info(arena(), 8, 0, SWNodeInfo::initial), // info needed per node | |
| 43 _align_to_ref(NULL), // memory reference to align vectors to | |
| 44 _disjoint_ptrs(arena(), 8, 0, OrderedPair::initial), // runtime disambiguated pointer pairs | |
| 45 _dg(_arena), // dependence graph | |
| 46 _visited(arena()), // visited node set | |
| 47 _post_visited(arena()), // post visited node set | |
| 48 _n_idx_list(arena(), 8), // scratch list of (node,index) pairs | |
| 49 _stk(arena(), 8, 0, NULL), // scratch stack of nodes | |
| 50 _nlist(arena(), 8, 0, NULL), // scratch list of nodes | |
| 51 _lpt(NULL), // loop tree node | |
| 52 _lp(NULL), // LoopNode | |
| 53 _bb(NULL), // basic block | |
| 54 _iv(NULL) // induction var | |
| 55 {} | |
| 56 | |
| 57 //------------------------------transform_loop--------------------------- | |
| 58 void SuperWord::transform_loop(IdealLoopTree* lpt) { | |
| 59 assert(lpt->_head->is_CountedLoop(), "must be"); | |
| 60 CountedLoopNode *cl = lpt->_head->as_CountedLoop(); | |
| 61 | |
| 62 if (!cl->is_main_loop() ) return; // skip normal, pre, and post loops | |
| 63 | |
| 64 // Check for no control flow in body (other than exit) | |
| 65 Node *cl_exit = cl->loopexit(); | |
| 66 if (cl_exit->in(0) != lpt->_head) return; | |
| 67 | |
|
105
a7d0f95410bd
6646020: assert(in_bb(n),"must be in block") in -Xcomp mode
never
parents:
72
diff
changeset
|
68 // Make sure the are no extra control users of the loop backedge |
|
a7d0f95410bd
6646020: assert(in_bb(n),"must be in block") in -Xcomp mode
never
parents:
72
diff
changeset
|
69 if (cl->back_control()->outcnt() != 1) { |
|
a7d0f95410bd
6646020: assert(in_bb(n),"must be in block") in -Xcomp mode
never
parents:
72
diff
changeset
|
70 return; |
|
a7d0f95410bd
6646020: assert(in_bb(n),"must be in block") in -Xcomp mode
never
parents:
72
diff
changeset
|
71 } |
|
a7d0f95410bd
6646020: assert(in_bb(n),"must be in block") in -Xcomp mode
never
parents:
72
diff
changeset
|
72 |
| 0 | 73 // Check for pre-loop ending with CountedLoopEnd(Bool(Cmp(x,Opaque1(limit)))) |
| 74 CountedLoopEndNode* pre_end = get_pre_loop_end(cl); | |
| 75 if (pre_end == NULL) return; | |
| 76 Node *pre_opaq1 = pre_end->limit(); | |
| 77 if (pre_opaq1->Opcode() != Op_Opaque1) return; | |
| 78 | |
| 79 // Do vectors exist on this architecture? | |
| 80 if (vector_width_in_bytes() == 0) return; | |
| 81 | |
| 82 init(); // initialize data structures | |
| 83 | |
| 84 set_lpt(lpt); | |
| 85 set_lp(cl); | |
| 86 | |
| 87 // For now, define one block which is the entire loop body | |
| 88 set_bb(cl); | |
| 89 | |
| 90 assert(_packset.length() == 0, "packset must be empty"); | |
| 91 SLP_extract(); | |
| 92 } | |
| 93 | |
| 94 //------------------------------SLP_extract--------------------------- | |
| 95 // Extract the superword level parallelism | |
| 96 // | |
| 97 // 1) A reverse post-order of nodes in the block is constructed. By scanning | |
| 98 // this list from first to last, all definitions are visited before their uses. | |
| 99 // | |
| 100 // 2) A point-to-point dependence graph is constructed between memory references. | |
| 101 // This simplies the upcoming "independence" checker. | |
| 102 // | |
| 103 // 3) The maximum depth in the node graph from the beginning of the block | |
| 104 // to each node is computed. This is used to prune the graph search | |
| 105 // in the independence checker. | |
| 106 // | |
| 107 // 4) For integer types, the necessary bit width is propagated backwards | |
| 108 // from stores to allow packed operations on byte, char, and short | |
| 109 // integers. This reverses the promotion to type "int" that javac | |
| 110 // did for operations like: char c1,c2,c3; c1 = c2 + c3. | |
| 111 // | |
| 112 // 5) One of the memory references is picked to be an aligned vector reference. | |
| 113 // The pre-loop trip count is adjusted to align this reference in the | |
| 114 // unrolled body. | |
| 115 // | |
| 116 // 6) The initial set of pack pairs is seeded with memory references. | |
| 117 // | |
| 118 // 7) The set of pack pairs is extended by following use->def and def->use links. | |
| 119 // | |
| 120 // 8) The pairs are combined into vector sized packs. | |
| 121 // | |
| 122 // 9) Reorder the memory slices to co-locate members of the memory packs. | |
| 123 // | |
| 124 // 10) Generate ideal vector nodes for the final set of packs and where necessary, | |
| 125 // inserting scalar promotion, vector creation from multiple scalars, and | |
| 126 // extraction of scalar values from vectors. | |
| 127 // | |
| 128 void SuperWord::SLP_extract() { | |
| 129 | |
| 130 // Ready the block | |
| 131 | |
| 132 construct_bb(); | |
| 133 | |
| 134 dependence_graph(); | |
| 135 | |
| 136 compute_max_depth(); | |
| 137 | |
| 138 compute_vector_element_type(); | |
| 139 | |
| 140 // Attempt vectorization | |
| 141 | |
| 142 find_adjacent_refs(); | |
| 143 | |
| 144 extend_packlist(); | |
| 145 | |
| 146 combine_packs(); | |
| 147 | |
| 148 construct_my_pack_map(); | |
| 149 | |
| 150 filter_packs(); | |
| 151 | |
| 152 schedule(); | |
| 153 | |
| 154 output(); | |
| 155 } | |
| 156 | |
| 157 //------------------------------find_adjacent_refs--------------------------- | |
| 158 // Find the adjacent memory references and create pack pairs for them. | |
| 159 // This is the initial set of packs that will then be extended by | |
| 160 // following use->def and def->use links. The align positions are | |
| 161 // assigned relative to the reference "align_to_ref" | |
| 162 void SuperWord::find_adjacent_refs() { | |
| 163 // Get list of memory operations | |
| 164 Node_List memops; | |
| 165 for (int i = 0; i < _block.length(); i++) { | |
| 166 Node* n = _block.at(i); | |
|
29
d5fc211aea19
6633953: type2aelembytes{T_ADDRESS} should be 8 bytes in 64 bit VM
kvn
parents:
0
diff
changeset
|
167 if (n->is_Mem() && in_bb(n) && |
|
d5fc211aea19
6633953: type2aelembytes{T_ADDRESS} should be 8 bytes in 64 bit VM
kvn
parents:
0
diff
changeset
|
168 is_java_primitive(n->as_Mem()->memory_type())) { |
| 0 | 169 int align = memory_alignment(n->as_Mem(), 0); |
| 170 if (align != bottom_align) { | |
| 171 memops.push(n); | |
| 172 } | |
| 173 } | |
| 174 } | |
| 175 if (memops.size() == 0) return; | |
| 176 | |
| 177 // Find a memory reference to align to. The pre-loop trip count | |
| 178 // is modified to align this reference to a vector-aligned address | |
| 179 find_align_to_ref(memops); | |
| 180 if (align_to_ref() == NULL) return; | |
| 181 | |
| 182 SWPointer align_to_ref_p(align_to_ref(), this); | |
| 183 int offset = align_to_ref_p.offset_in_bytes(); | |
| 184 int scale = align_to_ref_p.scale_in_bytes(); | |
| 185 int vw = vector_width_in_bytes(); | |
| 186 int stride_sign = (scale * iv_stride()) > 0 ? 1 : -1; | |
| 187 int iv_adjustment = (stride_sign * vw - (offset % vw)) % vw; | |
| 188 | |
| 189 #ifndef PRODUCT | |
| 190 if (TraceSuperWord) | |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
191 tty->print_cr("\noffset = %d iv_adjustment = %d elt_align = %d scale = %d iv_stride = %d", |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
192 offset, iv_adjustment, align_to_ref_p.memory_size(), align_to_ref_p.scale_in_bytes(), iv_stride()); |
| 0 | 193 #endif |
| 194 | |
| 195 // Set alignment relative to "align_to_ref" | |
| 196 for (int i = memops.size() - 1; i >= 0; i--) { | |
| 197 MemNode* s = memops.at(i)->as_Mem(); | |
| 198 SWPointer p2(s, this); | |
| 199 if (p2.comparable(align_to_ref_p)) { | |
| 200 int align = memory_alignment(s, iv_adjustment); | |
| 201 set_alignment(s, align); | |
| 202 } else { | |
| 203 memops.remove(i); | |
| 204 } | |
| 205 } | |
| 206 | |
| 207 // Create initial pack pairs of memory operations | |
| 208 for (uint i = 0; i < memops.size(); i++) { | |
| 209 Node* s1 = memops.at(i); | |
| 210 for (uint j = 0; j < memops.size(); j++) { | |
| 211 Node* s2 = memops.at(j); | |
| 212 if (s1 != s2 && are_adjacent_refs(s1, s2)) { | |
| 213 int align = alignment(s1); | |
| 214 if (stmts_can_pack(s1, s2, align)) { | |
| 215 Node_List* pair = new Node_List(); | |
| 216 pair->push(s1); | |
| 217 pair->push(s2); | |
| 218 _packset.append(pair); | |
| 219 } | |
| 220 } | |
| 221 } | |
| 222 } | |
| 223 | |
| 224 #ifndef PRODUCT | |
| 225 if (TraceSuperWord) { | |
| 226 tty->print_cr("\nAfter find_adjacent_refs"); | |
| 227 print_packset(); | |
| 228 } | |
| 229 #endif | |
| 230 } | |
| 231 | |
| 232 //------------------------------find_align_to_ref--------------------------- | |
| 233 // Find a memory reference to align the loop induction variable to. | |
| 234 // Looks first at stores then at loads, looking for a memory reference | |
| 235 // with the largest number of references similar to it. | |
| 236 void SuperWord::find_align_to_ref(Node_List &memops) { | |
| 237 GrowableArray<int> cmp_ct(arena(), memops.size(), memops.size(), 0); | |
| 238 | |
| 239 // Count number of comparable memory ops | |
| 240 for (uint i = 0; i < memops.size(); i++) { | |
| 241 MemNode* s1 = memops.at(i)->as_Mem(); | |
| 242 SWPointer p1(s1, this); | |
| 243 // Discard if pre loop can't align this reference | |
| 244 if (!ref_is_alignable(p1)) { | |
| 245 *cmp_ct.adr_at(i) = 0; | |
| 246 continue; | |
| 247 } | |
| 248 for (uint j = i+1; j < memops.size(); j++) { | |
| 249 MemNode* s2 = memops.at(j)->as_Mem(); | |
| 250 if (isomorphic(s1, s2)) { | |
| 251 SWPointer p2(s2, this); | |
| 252 if (p1.comparable(p2)) { | |
| 253 (*cmp_ct.adr_at(i))++; | |
| 254 (*cmp_ct.adr_at(j))++; | |
| 255 } | |
| 256 } | |
| 257 } | |
| 258 } | |
| 259 | |
| 260 // Find Store (or Load) with the greatest number of "comparable" references | |
| 261 int max_ct = 0; | |
| 262 int max_idx = -1; | |
| 263 int min_size = max_jint; | |
| 264 int min_iv_offset = max_jint; | |
| 265 for (uint j = 0; j < memops.size(); j++) { | |
| 266 MemNode* s = memops.at(j)->as_Mem(); | |
| 267 if (s->is_Store()) { | |
| 268 SWPointer p(s, this); | |
| 269 if (cmp_ct.at(j) > max_ct || | |
| 270 cmp_ct.at(j) == max_ct && (data_size(s) < min_size || | |
| 271 data_size(s) == min_size && | |
| 272 p.offset_in_bytes() < min_iv_offset)) { | |
| 273 max_ct = cmp_ct.at(j); | |
| 274 max_idx = j; | |
| 275 min_size = data_size(s); | |
| 276 min_iv_offset = p.offset_in_bytes(); | |
| 277 } | |
| 278 } | |
| 279 } | |
| 280 // If no stores, look at loads | |
| 281 if (max_ct == 0) { | |
| 282 for (uint j = 0; j < memops.size(); j++) { | |
| 283 MemNode* s = memops.at(j)->as_Mem(); | |
| 284 if (s->is_Load()) { | |
| 285 SWPointer p(s, this); | |
| 286 if (cmp_ct.at(j) > max_ct || | |
| 287 cmp_ct.at(j) == max_ct && (data_size(s) < min_size || | |
| 288 data_size(s) == min_size && | |
| 289 p.offset_in_bytes() < min_iv_offset)) { | |
| 290 max_ct = cmp_ct.at(j); | |
| 291 max_idx = j; | |
| 292 min_size = data_size(s); | |
| 293 min_iv_offset = p.offset_in_bytes(); | |
| 294 } | |
| 295 } | |
| 296 } | |
| 297 } | |
| 298 | |
| 299 if (max_ct > 0) | |
| 300 set_align_to_ref(memops.at(max_idx)->as_Mem()); | |
| 301 | |
| 302 #ifndef PRODUCT | |
| 303 if (TraceSuperWord && Verbose) { | |
| 304 tty->print_cr("\nVector memops after find_align_to_refs"); | |
| 305 for (uint i = 0; i < memops.size(); i++) { | |
| 306 MemNode* s = memops.at(i)->as_Mem(); | |
| 307 s->dump(); | |
| 308 } | |
| 309 } | |
| 310 #endif | |
| 311 } | |
| 312 | |
| 313 //------------------------------ref_is_alignable--------------------------- | |
| 314 // Can the preloop align the reference to position zero in the vector? | |
| 315 bool SuperWord::ref_is_alignable(SWPointer& p) { | |
| 316 if (!p.has_iv()) { | |
| 317 return true; // no induction variable | |
| 318 } | |
| 319 CountedLoopEndNode* pre_end = get_pre_loop_end(lp()->as_CountedLoop()); | |
| 320 assert(pre_end->stride_is_con(), "pre loop stride is constant"); | |
| 321 int preloop_stride = pre_end->stride_con(); | |
| 322 | |
| 323 int span = preloop_stride * p.scale_in_bytes(); | |
| 324 | |
| 325 // Stride one accesses are alignable. | |
| 326 if (ABS(span) == p.memory_size()) | |
| 327 return true; | |
| 328 | |
| 329 // If initial offset from start of object is computable, | |
| 330 // compute alignment within the vector. | |
| 331 int vw = vector_width_in_bytes(); | |
| 332 if (vw % span == 0) { | |
| 333 Node* init_nd = pre_end->init_trip(); | |
| 334 if (init_nd->is_Con() && p.invar() == NULL) { | |
| 335 int init = init_nd->bottom_type()->is_int()->get_con(); | |
| 336 | |
| 337 int init_offset = init * p.scale_in_bytes() + p.offset_in_bytes(); | |
| 338 assert(init_offset >= 0, "positive offset from object start"); | |
| 339 | |
| 340 if (span > 0) { | |
| 341 return (vw - (init_offset % vw)) % span == 0; | |
| 342 } else { | |
| 343 assert(span < 0, "nonzero stride * scale"); | |
| 344 return (init_offset % vw) % -span == 0; | |
| 345 } | |
| 346 } | |
| 347 } | |
| 348 return false; | |
| 349 } | |
| 350 | |
| 351 //---------------------------dependence_graph--------------------------- | |
| 352 // Construct dependency graph. | |
| 353 // Add dependence edges to load/store nodes for memory dependence | |
| 354 // A.out()->DependNode.in(1) and DependNode.out()->B.prec(x) | |
| 355 void SuperWord::dependence_graph() { | |
| 356 // First, assign a dependence node to each memory node | |
| 357 for (int i = 0; i < _block.length(); i++ ) { | |
| 358 Node *n = _block.at(i); | |
| 359 if (n->is_Mem() || n->is_Phi() && n->bottom_type() == Type::MEMORY) { | |
| 360 _dg.make_node(n); | |
| 361 } | |
| 362 } | |
| 363 | |
| 364 // For each memory slice, create the dependences | |
| 365 for (int i = 0; i < _mem_slice_head.length(); i++) { | |
| 366 Node* n = _mem_slice_head.at(i); | |
| 367 Node* n_tail = _mem_slice_tail.at(i); | |
| 368 | |
| 369 // Get slice in predecessor order (last is first) | |
| 370 mem_slice_preds(n_tail, n, _nlist); | |
| 371 | |
| 372 // Make the slice dependent on the root | |
| 373 DepMem* slice = _dg.dep(n); | |
| 374 _dg.make_edge(_dg.root(), slice); | |
| 375 | |
| 376 // Create a sink for the slice | |
| 377 DepMem* slice_sink = _dg.make_node(NULL); | |
| 378 _dg.make_edge(slice_sink, _dg.tail()); | |
| 379 | |
| 380 // Now visit each pair of memory ops, creating the edges | |
| 381 for (int j = _nlist.length() - 1; j >= 0 ; j--) { | |
| 382 Node* s1 = _nlist.at(j); | |
| 383 | |
| 384 // If no dependency yet, use slice | |
| 385 if (_dg.dep(s1)->in_cnt() == 0) { | |
| 386 _dg.make_edge(slice, s1); | |
| 387 } | |
| 388 SWPointer p1(s1->as_Mem(), this); | |
| 389 bool sink_dependent = true; | |
| 390 for (int k = j - 1; k >= 0; k--) { | |
| 391 Node* s2 = _nlist.at(k); | |
| 392 if (s1->is_Load() && s2->is_Load()) | |
| 393 continue; | |
| 394 SWPointer p2(s2->as_Mem(), this); | |
| 395 | |
| 396 int cmp = p1.cmp(p2); | |
| 397 if (SuperWordRTDepCheck && | |
| 398 p1.base() != p2.base() && p1.valid() && p2.valid()) { | |
| 399 // Create a runtime check to disambiguate | |
| 400 OrderedPair pp(p1.base(), p2.base()); | |
| 401 _disjoint_ptrs.append_if_missing(pp); | |
| 402 } else if (!SWPointer::not_equal(cmp)) { | |
| 403 // Possibly same address | |
| 404 _dg.make_edge(s1, s2); | |
| 405 sink_dependent = false; | |
| 406 } | |
| 407 } | |
| 408 if (sink_dependent) { | |
| 409 _dg.make_edge(s1, slice_sink); | |
| 410 } | |
| 411 } | |
| 412 #ifndef PRODUCT | |
| 413 if (TraceSuperWord) { | |
| 414 tty->print_cr("\nDependence graph for slice: %d", n->_idx); | |
| 415 for (int q = 0; q < _nlist.length(); q++) { | |
| 416 _dg.print(_nlist.at(q)); | |
| 417 } | |
| 418 tty->cr(); | |
| 419 } | |
| 420 #endif | |
| 421 _nlist.clear(); | |
| 422 } | |
| 423 | |
| 424 #ifndef PRODUCT | |
| 425 if (TraceSuperWord) { | |
| 426 tty->print_cr("\ndisjoint_ptrs: %s", _disjoint_ptrs.length() > 0 ? "" : "NONE"); | |
| 427 for (int r = 0; r < _disjoint_ptrs.length(); r++) { | |
| 428 _disjoint_ptrs.at(r).print(); | |
| 429 tty->cr(); | |
| 430 } | |
| 431 tty->cr(); | |
| 432 } | |
| 433 #endif | |
| 434 } | |
| 435 | |
| 436 //---------------------------mem_slice_preds--------------------------- | |
| 437 // Return a memory slice (node list) in predecessor order starting at "start" | |
| 438 void SuperWord::mem_slice_preds(Node* start, Node* stop, GrowableArray<Node*> &preds) { | |
| 439 assert(preds.length() == 0, "start empty"); | |
| 440 Node* n = start; | |
| 441 Node* prev = NULL; | |
| 442 while (true) { | |
| 443 assert(in_bb(n), "must be in block"); | |
| 444 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { | |
| 445 Node* out = n->fast_out(i); | |
| 446 if (out->is_Load()) { | |
| 447 if (in_bb(out)) { | |
| 448 preds.push(out); | |
| 449 } | |
| 450 } else { | |
| 451 // FIXME | |
| 452 if (out->is_MergeMem() && !in_bb(out)) { | |
| 453 // Either unrolling is causing a memory edge not to disappear, | |
| 454 // or need to run igvn.optimize() again before SLP | |
| 455 } else if (out->is_Phi() && out->bottom_type() == Type::MEMORY && !in_bb(out)) { | |
| 456 // Ditto. Not sure what else to check further. | |
| 667 | 457 } else if (out->Opcode() == Op_StoreCM && out->in(MemNode::OopStore) == n) { |
| 0 | 458 // StoreCM has an input edge used as a precedence edge. |
| 459 // Maybe an issue when oop stores are vectorized. | |
| 460 } else { | |
| 461 assert(out == prev || prev == NULL, "no branches off of store slice"); | |
| 462 } | |
| 463 } | |
| 464 } | |
| 465 if (n == stop) break; | |
| 466 preds.push(n); | |
| 467 prev = n; | |
| 468 n = n->in(MemNode::Memory); | |
| 469 } | |
| 470 } | |
| 471 | |
| 472 //------------------------------stmts_can_pack--------------------------- | |
| 605 | 473 // Can s1 and s2 be in a pack with s1 immediately preceding s2 and |
| 0 | 474 // s1 aligned at "align" |
| 475 bool SuperWord::stmts_can_pack(Node* s1, Node* s2, int align) { | |
|
987
00977607da34
6879921: CTW failure jdk6_18/hotspot/src/share/vm/utilities/globalDefinitions.cpp:268
cfang
parents:
985
diff
changeset
|
476 |
|
00977607da34
6879921: CTW failure jdk6_18/hotspot/src/share/vm/utilities/globalDefinitions.cpp:268
cfang
parents:
985
diff
changeset
|
477 // Do not use superword for non-primitives |
|
00977607da34
6879921: CTW failure jdk6_18/hotspot/src/share/vm/utilities/globalDefinitions.cpp:268
cfang
parents:
985
diff
changeset
|
478 if((s1->is_Mem() && !is_java_primitive(s1->as_Mem()->memory_type())) || |
|
00977607da34
6879921: CTW failure jdk6_18/hotspot/src/share/vm/utilities/globalDefinitions.cpp:268
cfang
parents:
985
diff
changeset
|
479 (s2->is_Mem() && !is_java_primitive(s2->as_Mem()->memory_type()))) |
|
00977607da34
6879921: CTW failure jdk6_18/hotspot/src/share/vm/utilities/globalDefinitions.cpp:268
cfang
parents:
985
diff
changeset
|
480 return false; |
|
00977607da34
6879921: CTW failure jdk6_18/hotspot/src/share/vm/utilities/globalDefinitions.cpp:268
cfang
parents:
985
diff
changeset
|
481 |
| 0 | 482 if (isomorphic(s1, s2)) { |
| 483 if (independent(s1, s2)) { | |
| 484 if (!exists_at(s1, 0) && !exists_at(s2, 1)) { | |
| 485 if (!s1->is_Mem() || are_adjacent_refs(s1, s2)) { | |
| 486 int s1_align = alignment(s1); | |
| 487 int s2_align = alignment(s2); | |
| 488 if (s1_align == top_align || s1_align == align) { | |
| 489 if (s2_align == top_align || s2_align == align + data_size(s1)) { | |
| 490 return true; | |
| 491 } | |
| 492 } | |
| 493 } | |
| 494 } | |
| 495 } | |
| 496 } | |
| 497 return false; | |
| 498 } | |
| 499 | |
| 500 //------------------------------exists_at--------------------------- | |
| 501 // Does s exist in a pack at position pos? | |
| 502 bool SuperWord::exists_at(Node* s, uint pos) { | |
| 503 for (int i = 0; i < _packset.length(); i++) { | |
| 504 Node_List* p = _packset.at(i); | |
| 505 if (p->at(pos) == s) { | |
| 506 return true; | |
| 507 } | |
| 508 } | |
| 509 return false; | |
| 510 } | |
| 511 | |
| 512 //------------------------------are_adjacent_refs--------------------------- | |
| 513 // Is s1 immediately before s2 in memory? | |
| 514 bool SuperWord::are_adjacent_refs(Node* s1, Node* s2) { | |
| 515 if (!s1->is_Mem() || !s2->is_Mem()) return false; | |
| 516 if (!in_bb(s1) || !in_bb(s2)) return false; | |
| 1585 | 517 |
| 518 // Do not use superword for non-primitives | |
| 519 if (!is_java_primitive(s1->as_Mem()->memory_type()) || | |
| 520 !is_java_primitive(s2->as_Mem()->memory_type())) { | |
| 521 return false; | |
| 522 } | |
| 523 | |
| 0 | 524 // FIXME - co_locate_pack fails on Stores in different mem-slices, so |
| 525 // only pack memops that are in the same alias set until that's fixed. | |
| 526 if (_phase->C->get_alias_index(s1->as_Mem()->adr_type()) != | |
| 527 _phase->C->get_alias_index(s2->as_Mem()->adr_type())) | |
| 528 return false; | |
| 529 SWPointer p1(s1->as_Mem(), this); | |
| 530 SWPointer p2(s2->as_Mem(), this); | |
| 531 if (p1.base() != p2.base() || !p1.comparable(p2)) return false; | |
| 532 int diff = p2.offset_in_bytes() - p1.offset_in_bytes(); | |
| 533 return diff == data_size(s1); | |
| 534 } | |
| 535 | |
| 536 //------------------------------isomorphic--------------------------- | |
| 537 // Are s1 and s2 similar? | |
| 538 bool SuperWord::isomorphic(Node* s1, Node* s2) { | |
| 539 if (s1->Opcode() != s2->Opcode()) return false; | |
| 540 if (s1->req() != s2->req()) return false; | |
| 541 if (s1->in(0) != s2->in(0)) return false; | |
| 542 if (velt_type(s1) != velt_type(s2)) return false; | |
| 543 return true; | |
| 544 } | |
| 545 | |
| 546 //------------------------------independent--------------------------- | |
| 547 // Is there no data path from s1 to s2 or s2 to s1? | |
| 548 bool SuperWord::independent(Node* s1, Node* s2) { | |
| 549 // assert(s1->Opcode() == s2->Opcode(), "check isomorphic first"); | |
| 550 int d1 = depth(s1); | |
| 551 int d2 = depth(s2); | |
| 552 if (d1 == d2) return s1 != s2; | |
| 553 Node* deep = d1 > d2 ? s1 : s2; | |
| 554 Node* shallow = d1 > d2 ? s2 : s1; | |
| 555 | |
| 556 visited_clear(); | |
| 557 | |
| 558 return independent_path(shallow, deep); | |
| 559 } | |
| 560 | |
| 561 //------------------------------independent_path------------------------------ | |
| 562 // Helper for independent | |
| 563 bool SuperWord::independent_path(Node* shallow, Node* deep, uint dp) { | |
| 564 if (dp >= 1000) return false; // stop deep recursion | |
| 565 visited_set(deep); | |
| 566 int shal_depth = depth(shallow); | |
| 567 assert(shal_depth <= depth(deep), "must be"); | |
| 568 for (DepPreds preds(deep, _dg); !preds.done(); preds.next()) { | |
| 569 Node* pred = preds.current(); | |
| 570 if (in_bb(pred) && !visited_test(pred)) { | |
| 571 if (shallow == pred) { | |
| 572 return false; | |
| 573 } | |
| 574 if (shal_depth < depth(pred) && !independent_path(shallow, pred, dp+1)) { | |
| 575 return false; | |
| 576 } | |
| 577 } | |
| 578 } | |
| 579 return true; | |
| 580 } | |
| 581 | |
| 582 //------------------------------set_alignment--------------------------- | |
| 583 void SuperWord::set_alignment(Node* s1, Node* s2, int align) { | |
| 584 set_alignment(s1, align); | |
| 585 set_alignment(s2, align + data_size(s1)); | |
| 586 } | |
| 587 | |
| 588 //------------------------------data_size--------------------------- | |
| 589 int SuperWord::data_size(Node* s) { | |
| 590 const Type* t = velt_type(s); | |
| 591 BasicType bt = t->array_element_basic_type(); | |
|
29
d5fc211aea19
6633953: type2aelembytes{T_ADDRESS} should be 8 bytes in 64 bit VM
kvn
parents:
0
diff
changeset
|
592 int bsize = type2aelembytes(bt); |
| 0 | 593 assert(bsize != 0, "valid size"); |
| 594 return bsize; | |
| 595 } | |
| 596 | |
| 597 //------------------------------extend_packlist--------------------------- | |
| 598 // Extend packset by following use->def and def->use links from pack members. | |
| 599 void SuperWord::extend_packlist() { | |
| 600 bool changed; | |
| 601 do { | |
| 602 changed = false; | |
| 603 for (int i = 0; i < _packset.length(); i++) { | |
| 604 Node_List* p = _packset.at(i); | |
| 605 changed |= follow_use_defs(p); | |
| 606 changed |= follow_def_uses(p); | |
| 607 } | |
| 608 } while (changed); | |
| 609 | |
| 610 #ifndef PRODUCT | |
| 611 if (TraceSuperWord) { | |
| 612 tty->print_cr("\nAfter extend_packlist"); | |
| 613 print_packset(); | |
| 614 } | |
| 615 #endif | |
| 616 } | |
| 617 | |
| 618 //------------------------------follow_use_defs--------------------------- | |
| 619 // Extend the packset by visiting operand definitions of nodes in pack p | |
| 620 bool SuperWord::follow_use_defs(Node_List* p) { | |
| 621 Node* s1 = p->at(0); | |
| 622 Node* s2 = p->at(1); | |
| 623 assert(p->size() == 2, "just checking"); | |
| 624 assert(s1->req() == s2->req(), "just checking"); | |
| 625 assert(alignment(s1) + data_size(s1) == alignment(s2), "just checking"); | |
| 626 | |
| 627 if (s1->is_Load()) return false; | |
| 628 | |
| 629 int align = alignment(s1); | |
| 630 bool changed = false; | |
| 631 int start = s1->is_Store() ? MemNode::ValueIn : 1; | |
| 632 int end = s1->is_Store() ? MemNode::ValueIn+1 : s1->req(); | |
| 633 for (int j = start; j < end; j++) { | |
| 634 Node* t1 = s1->in(j); | |
| 635 Node* t2 = s2->in(j); | |
| 636 if (!in_bb(t1) || !in_bb(t2)) | |
| 637 continue; | |
| 638 if (stmts_can_pack(t1, t2, align)) { | |
| 639 if (est_savings(t1, t2) >= 0) { | |
| 640 Node_List* pair = new Node_List(); | |
| 641 pair->push(t1); | |
| 642 pair->push(t2); | |
| 643 _packset.append(pair); | |
| 644 set_alignment(t1, t2, align); | |
| 645 changed = true; | |
| 646 } | |
| 647 } | |
| 648 } | |
| 649 return changed; | |
| 650 } | |
| 651 | |
| 652 //------------------------------follow_def_uses--------------------------- | |
| 653 // Extend the packset by visiting uses of nodes in pack p | |
| 654 bool SuperWord::follow_def_uses(Node_List* p) { | |
| 655 bool changed = false; | |
| 656 Node* s1 = p->at(0); | |
| 657 Node* s2 = p->at(1); | |
| 658 assert(p->size() == 2, "just checking"); | |
| 659 assert(s1->req() == s2->req(), "just checking"); | |
| 660 assert(alignment(s1) + data_size(s1) == alignment(s2), "just checking"); | |
| 661 | |
| 662 if (s1->is_Store()) return false; | |
| 663 | |
| 664 int align = alignment(s1); | |
| 665 int savings = -1; | |
| 666 Node* u1 = NULL; | |
| 667 Node* u2 = NULL; | |
| 668 for (DUIterator_Fast imax, i = s1->fast_outs(imax); i < imax; i++) { | |
| 669 Node* t1 = s1->fast_out(i); | |
| 670 if (!in_bb(t1)) continue; | |
| 671 for (DUIterator_Fast jmax, j = s2->fast_outs(jmax); j < jmax; j++) { | |
| 672 Node* t2 = s2->fast_out(j); | |
| 673 if (!in_bb(t2)) continue; | |
| 674 if (!opnd_positions_match(s1, t1, s2, t2)) | |
| 675 continue; | |
| 676 if (stmts_can_pack(t1, t2, align)) { | |
| 677 int my_savings = est_savings(t1, t2); | |
| 678 if (my_savings > savings) { | |
| 679 savings = my_savings; | |
| 680 u1 = t1; | |
| 681 u2 = t2; | |
| 682 } | |
| 683 } | |
| 684 } | |
| 685 } | |
| 686 if (savings >= 0) { | |
| 687 Node_List* pair = new Node_List(); | |
| 688 pair->push(u1); | |
| 689 pair->push(u2); | |
| 690 _packset.append(pair); | |
| 691 set_alignment(u1, u2, align); | |
| 692 changed = true; | |
| 693 } | |
| 694 return changed; | |
| 695 } | |
| 696 | |
| 697 //---------------------------opnd_positions_match------------------------- | |
| 698 // Is the use of d1 in u1 at the same operand position as d2 in u2? | |
| 699 bool SuperWord::opnd_positions_match(Node* d1, Node* u1, Node* d2, Node* u2) { | |
| 700 uint ct = u1->req(); | |
| 701 if (ct != u2->req()) return false; | |
| 702 uint i1 = 0; | |
| 703 uint i2 = 0; | |
| 704 do { | |
| 705 for (i1++; i1 < ct; i1++) if (u1->in(i1) == d1) break; | |
| 706 for (i2++; i2 < ct; i2++) if (u2->in(i2) == d2) break; | |
| 707 if (i1 != i2) { | |
| 708 return false; | |
| 709 } | |
| 710 } while (i1 < ct); | |
| 711 return true; | |
| 712 } | |
| 713 | |
| 714 //------------------------------est_savings--------------------------- | |
| 715 // Estimate the savings from executing s1 and s2 as a pack | |
| 716 int SuperWord::est_savings(Node* s1, Node* s2) { | |
| 717 int save = 2 - 1; // 2 operations per instruction in packed form | |
| 718 | |
| 719 // inputs | |
| 720 for (uint i = 1; i < s1->req(); i++) { | |
| 721 Node* x1 = s1->in(i); | |
| 722 Node* x2 = s2->in(i); | |
| 723 if (x1 != x2) { | |
| 724 if (are_adjacent_refs(x1, x2)) { | |
| 725 save += adjacent_profit(x1, x2); | |
| 726 } else if (!in_packset(x1, x2)) { | |
| 727 save -= pack_cost(2); | |
| 728 } else { | |
| 729 save += unpack_cost(2); | |
| 730 } | |
| 731 } | |
| 732 } | |
| 733 | |
| 734 // uses of result | |
| 735 uint ct = 0; | |
| 736 for (DUIterator_Fast imax, i = s1->fast_outs(imax); i < imax; i++) { | |
| 737 Node* s1_use = s1->fast_out(i); | |
| 738 for (int j = 0; j < _packset.length(); j++) { | |
| 739 Node_List* p = _packset.at(j); | |
| 740 if (p->at(0) == s1_use) { | |
| 741 for (DUIterator_Fast kmax, k = s2->fast_outs(kmax); k < kmax; k++) { | |
| 742 Node* s2_use = s2->fast_out(k); | |
| 743 if (p->at(p->size()-1) == s2_use) { | |
| 744 ct++; | |
| 745 if (are_adjacent_refs(s1_use, s2_use)) { | |
| 746 save += adjacent_profit(s1_use, s2_use); | |
| 747 } | |
| 748 } | |
| 749 } | |
| 750 } | |
| 751 } | |
| 752 } | |
| 753 | |
| 754 if (ct < s1->outcnt()) save += unpack_cost(1); | |
| 755 if (ct < s2->outcnt()) save += unpack_cost(1); | |
| 756 | |
| 757 return save; | |
| 758 } | |
| 759 | |
| 760 //------------------------------costs--------------------------- | |
| 761 int SuperWord::adjacent_profit(Node* s1, Node* s2) { return 2; } | |
| 762 int SuperWord::pack_cost(int ct) { return ct; } | |
| 763 int SuperWord::unpack_cost(int ct) { return ct; } | |
| 764 | |
| 765 //------------------------------combine_packs--------------------------- | |
| 766 // Combine packs A and B with A.last == B.first into A.first..,A.last,B.second,..B.last | |
| 767 void SuperWord::combine_packs() { | |
| 768 bool changed; | |
| 769 do { | |
| 770 changed = false; | |
| 771 for (int i = 0; i < _packset.length(); i++) { | |
| 772 Node_List* p1 = _packset.at(i); | |
| 773 if (p1 == NULL) continue; | |
| 774 for (int j = 0; j < _packset.length(); j++) { | |
| 775 Node_List* p2 = _packset.at(j); | |
| 776 if (p2 == NULL) continue; | |
| 777 if (p1->at(p1->size()-1) == p2->at(0)) { | |
| 778 for (uint k = 1; k < p2->size(); k++) { | |
| 779 p1->push(p2->at(k)); | |
| 780 } | |
| 781 _packset.at_put(j, NULL); | |
| 782 changed = true; | |
| 783 } | |
| 784 } | |
| 785 } | |
| 786 } while (changed); | |
| 787 | |
| 788 for (int i = _packset.length() - 1; i >= 0; i--) { | |
| 789 Node_List* p1 = _packset.at(i); | |
| 790 if (p1 == NULL) { | |
| 791 _packset.remove_at(i); | |
| 792 } | |
| 793 } | |
| 794 | |
| 795 #ifndef PRODUCT | |
| 796 if (TraceSuperWord) { | |
| 797 tty->print_cr("\nAfter combine_packs"); | |
| 798 print_packset(); | |
| 799 } | |
| 800 #endif | |
| 801 } | |
| 802 | |
| 803 //-----------------------------construct_my_pack_map-------------------------- | |
| 804 // Construct the map from nodes to packs. Only valid after the | |
| 805 // point where a node is only in one pack (after combine_packs). | |
| 806 void SuperWord::construct_my_pack_map() { | |
| 807 Node_List* rslt = NULL; | |
| 808 for (int i = 0; i < _packset.length(); i++) { | |
| 809 Node_List* p = _packset.at(i); | |
| 810 for (uint j = 0; j < p->size(); j++) { | |
| 811 Node* s = p->at(j); | |
| 812 assert(my_pack(s) == NULL, "only in one pack"); | |
| 813 set_my_pack(s, p); | |
| 814 } | |
| 815 } | |
| 816 } | |
| 817 | |
| 818 //------------------------------filter_packs--------------------------- | |
| 819 // Remove packs that are not implemented or not profitable. | |
| 820 void SuperWord::filter_packs() { | |
| 821 | |
| 822 // Remove packs that are not implemented | |
| 823 for (int i = _packset.length() - 1; i >= 0; i--) { | |
| 824 Node_List* pk = _packset.at(i); | |
| 825 bool impl = implemented(pk); | |
| 826 if (!impl) { | |
| 827 #ifndef PRODUCT | |
| 828 if (TraceSuperWord && Verbose) { | |
| 829 tty->print_cr("Unimplemented"); | |
| 830 pk->at(0)->dump(); | |
| 831 } | |
| 832 #endif | |
| 833 remove_pack_at(i); | |
| 834 } | |
| 835 } | |
| 836 | |
| 837 // Remove packs that are not profitable | |
| 838 bool changed; | |
| 839 do { | |
| 840 changed = false; | |
| 841 for (int i = _packset.length() - 1; i >= 0; i--) { | |
| 842 Node_List* pk = _packset.at(i); | |
| 843 bool prof = profitable(pk); | |
| 844 if (!prof) { | |
| 845 #ifndef PRODUCT | |
| 846 if (TraceSuperWord && Verbose) { | |
| 847 tty->print_cr("Unprofitable"); | |
| 848 pk->at(0)->dump(); | |
| 849 } | |
| 850 #endif | |
| 851 remove_pack_at(i); | |
| 852 changed = true; | |
| 853 } | |
| 854 } | |
| 855 } while (changed); | |
| 856 | |
| 857 #ifndef PRODUCT | |
| 858 if (TraceSuperWord) { | |
| 859 tty->print_cr("\nAfter filter_packs"); | |
| 860 print_packset(); | |
| 861 tty->cr(); | |
| 862 } | |
| 863 #endif | |
| 864 } | |
| 865 | |
| 866 //------------------------------implemented--------------------------- | |
| 867 // Can code be generated for pack p? | |
| 868 bool SuperWord::implemented(Node_List* p) { | |
| 869 Node* p0 = p->at(0); | |
| 870 int vopc = VectorNode::opcode(p0->Opcode(), p->size(), velt_type(p0)); | |
| 871 return vopc > 0 && Matcher::has_match_rule(vopc); | |
| 872 } | |
| 873 | |
| 874 //------------------------------profitable--------------------------- | |
| 875 // For pack p, are all operands and all uses (with in the block) vector? | |
| 876 bool SuperWord::profitable(Node_List* p) { | |
| 877 Node* p0 = p->at(0); | |
| 878 uint start, end; | |
| 879 vector_opd_range(p0, &start, &end); | |
| 880 | |
| 881 // Return false if some input is not vector and inside block | |
| 882 for (uint i = start; i < end; i++) { | |
| 883 if (!is_vector_use(p0, i)) { | |
| 884 // For now, return false if not scalar promotion case (inputs are the same.) | |
| 605 | 885 // Later, implement PackNode and allow differing, non-vector inputs |
| 0 | 886 // (maybe just the ones from outside the block.) |
| 887 Node* p0_def = p0->in(i); | |
| 888 for (uint j = 1; j < p->size(); j++) { | |
| 889 Node* use = p->at(j); | |
| 890 Node* def = use->in(i); | |
| 891 if (p0_def != def) | |
| 892 return false; | |
| 893 } | |
| 894 } | |
| 895 } | |
| 896 if (!p0->is_Store()) { | |
| 897 // For now, return false if not all uses are vector. | |
| 898 // Later, implement ExtractNode and allow non-vector uses (maybe | |
| 899 // just the ones outside the block.) | |
| 900 for (uint i = 0; i < p->size(); i++) { | |
| 901 Node* def = p->at(i); | |
| 902 for (DUIterator_Fast jmax, j = def->fast_outs(jmax); j < jmax; j++) { | |
| 903 Node* use = def->fast_out(j); | |
| 904 for (uint k = 0; k < use->req(); k++) { | |
| 905 Node* n = use->in(k); | |
| 906 if (def == n) { | |
| 907 if (!is_vector_use(use, k)) { | |
| 908 return false; | |
| 909 } | |
| 910 } | |
| 911 } | |
| 912 } | |
| 913 } | |
| 914 } | |
| 915 return true; | |
| 916 } | |
| 917 | |
| 918 //------------------------------schedule--------------------------- | |
| 919 // Adjust the memory graph for the packed operations | |
| 920 void SuperWord::schedule() { | |
| 921 | |
| 922 // Co-locate in the memory graph the members of each memory pack | |
| 923 for (int i = 0; i < _packset.length(); i++) { | |
| 924 co_locate_pack(_packset.at(i)); | |
| 925 } | |
| 926 } | |
| 927 | |
| 667 | 928 //-------------------------------remove_and_insert------------------- |
| 929 //remove "current" from its current position in the memory graph and insert | |
| 930 //it after the appropriate insertion point (lip or uip) | |
| 931 void SuperWord::remove_and_insert(MemNode *current, MemNode *prev, MemNode *lip, | |
| 932 Node *uip, Unique_Node_List &sched_before) { | |
| 933 Node* my_mem = current->in(MemNode::Memory); | |
| 934 _igvn.hash_delete(current); | |
| 935 _igvn.hash_delete(my_mem); | |
| 936 | |
| 937 //remove current_store from its current position in the memmory graph | |
| 938 for (DUIterator i = current->outs(); current->has_out(i); i++) { | |
| 939 Node* use = current->out(i); | |
| 940 if (use->is_Mem()) { | |
| 941 assert(use->in(MemNode::Memory) == current, "must be"); | |
| 942 _igvn.hash_delete(use); | |
| 943 if (use == prev) { // connect prev to my_mem | |
| 944 use->set_req(MemNode::Memory, my_mem); | |
| 945 } else if (sched_before.member(use)) { | |
| 946 _igvn.hash_delete(uip); | |
| 947 use->set_req(MemNode::Memory, uip); | |
| 948 } else { | |
| 949 _igvn.hash_delete(lip); | |
| 950 use->set_req(MemNode::Memory, lip); | |
| 951 } | |
| 952 _igvn._worklist.push(use); | |
| 953 --i; //deleted this edge; rescan position | |
| 954 } | |
| 955 } | |
| 956 | |
| 957 bool sched_up = sched_before.member(current); | |
| 958 Node *insert_pt = sched_up ? uip : lip; | |
| 959 _igvn.hash_delete(insert_pt); | |
| 960 | |
| 961 // all uses of insert_pt's memory state should use current's instead | |
| 962 for (DUIterator i = insert_pt->outs(); insert_pt->has_out(i); i++) { | |
| 963 Node* use = insert_pt->out(i); | |
| 964 if (use->is_Mem()) { | |
| 965 assert(use->in(MemNode::Memory) == insert_pt, "must be"); | |
| 966 _igvn.hash_delete(use); | |
| 967 use->set_req(MemNode::Memory, current); | |
| 968 _igvn._worklist.push(use); | |
| 969 --i; //deleted this edge; rescan position | |
| 970 } else if (!sched_up && use->is_Phi() && use->bottom_type() == Type::MEMORY) { | |
| 971 uint pos; //lip (lower insert point) must be the last one in the memory slice | |
| 972 _igvn.hash_delete(use); | |
| 973 for (pos=1; pos < use->req(); pos++) { | |
| 974 if (use->in(pos) == insert_pt) break; | |
| 975 } | |
| 976 use->set_req(pos, current); | |
| 977 _igvn._worklist.push(use); | |
| 978 --i; | |
| 979 } | |
| 980 } | |
| 981 | |
| 982 //connect current to insert_pt | |
| 983 current->set_req(MemNode::Memory, insert_pt); | |
| 984 _igvn._worklist.push(current); | |
| 985 } | |
| 986 | |
| 987 //------------------------------co_locate_pack---------------------------------- | |
| 988 // To schedule a store pack, we need to move any sandwiched memory ops either before | |
| 989 // or after the pack, based upon dependence information: | |
| 990 // (1) If any store in the pack depends on the sandwiched memory op, the | |
| 991 // sandwiched memory op must be scheduled BEFORE the pack; | |
| 992 // (2) If a sandwiched memory op depends on any store in the pack, the | |
| 993 // sandwiched memory op must be scheduled AFTER the pack; | |
| 994 // (3) If a sandwiched memory op (say, memA) depends on another sandwiched | |
| 995 // memory op (say memB), memB must be scheduled before memA. So, if memA is | |
| 996 // scheduled before the pack, memB must also be scheduled before the pack; | |
| 997 // (4) If there is no dependence restriction for a sandwiched memory op, we simply | |
| 998 // schedule this store AFTER the pack | |
| 999 // (5) We know there is no dependence cycle, so there in no other case; | |
| 1000 // (6) Finally, all memory ops in another single pack should be moved in the same direction. | |
| 1001 // | |
| 952 | 1002 // To schedule a load pack, we use the memory state of either the first or the last load in |
| 1003 // the pack, based on the dependence constraint. | |
| 0 | 1004 void SuperWord::co_locate_pack(Node_List* pk) { |
| 1005 if (pk->at(0)->is_Store()) { | |
| 1006 MemNode* first = executed_first(pk)->as_Mem(); | |
| 1007 MemNode* last = executed_last(pk)->as_Mem(); | |
| 667 | 1008 Unique_Node_List schedule_before_pack; |
| 1009 Unique_Node_List memops; | |
| 1010 | |
| 0 | 1011 MemNode* current = last->in(MemNode::Memory)->as_Mem(); |
| 667 | 1012 MemNode* previous = last; |
| 0 | 1013 while (true) { |
| 1014 assert(in_bb(current), "stay in block"); | |
| 667 | 1015 memops.push(previous); |
| 1016 for (DUIterator i = current->outs(); current->has_out(i); i++) { | |
| 1017 Node* use = current->out(i); | |
| 1018 if (use->is_Mem() && use != previous) | |
| 1019 memops.push(use); | |
| 1020 } | |
| 1021 if(current == first) break; | |
| 1022 previous = current; | |
| 1023 current = current->in(MemNode::Memory)->as_Mem(); | |
| 1024 } | |
| 1025 | |
| 1026 // determine which memory operations should be scheduled before the pack | |
| 1027 for (uint i = 1; i < memops.size(); i++) { | |
| 1028 Node *s1 = memops.at(i); | |
| 1029 if (!in_pack(s1, pk) && !schedule_before_pack.member(s1)) { | |
| 1030 for (uint j = 0; j< i; j++) { | |
| 1031 Node *s2 = memops.at(j); | |
| 1032 if (!independent(s1, s2)) { | |
| 1033 if (in_pack(s2, pk) || schedule_before_pack.member(s2)) { | |
| 1034 schedule_before_pack.push(s1); //s1 must be scheduled before | |
| 1035 Node_List* mem_pk = my_pack(s1); | |
| 1036 if (mem_pk != NULL) { | |
| 1037 for (uint ii = 0; ii < mem_pk->size(); ii++) { | |
| 1038 Node* s = mem_pk->at(ii); // follow partner | |
| 1039 if (memops.member(s) && !schedule_before_pack.member(s)) | |
| 1040 schedule_before_pack.push(s); | |
| 1041 } | |
| 1042 } | |
| 1043 } | |
| 1044 } | |
| 1045 } | |
| 1046 } | |
| 1047 } | |
| 1048 | |
| 1049 MemNode* lower_insert_pt = last; | |
| 1050 Node* upper_insert_pt = first->in(MemNode::Memory); | |
| 1051 previous = last; //previous store in pk | |
| 1052 current = last->in(MemNode::Memory)->as_Mem(); | |
| 1053 | |
| 1054 //start scheduling from "last" to "first" | |
| 1055 while (true) { | |
| 1056 assert(in_bb(current), "stay in block"); | |
| 1057 assert(in_pack(previous, pk), "previous stays in pack"); | |
| 0 | 1058 Node* my_mem = current->in(MemNode::Memory); |
| 667 | 1059 |
| 0 | 1060 if (in_pack(current, pk)) { |
| 667 | 1061 // Forward users of my memory state (except "previous) to my input memory state |
| 0 | 1062 _igvn.hash_delete(current); |
| 1063 for (DUIterator i = current->outs(); current->has_out(i); i++) { | |
| 1064 Node* use = current->out(i); | |
| 667 | 1065 if (use->is_Mem() && use != previous) { |
| 0 | 1066 assert(use->in(MemNode::Memory) == current, "must be"); |
| 1067 _igvn.hash_delete(use); | |
| 667 | 1068 if (schedule_before_pack.member(use)) { |
| 1069 _igvn.hash_delete(upper_insert_pt); | |
| 1070 use->set_req(MemNode::Memory, upper_insert_pt); | |
| 1071 } else { | |
| 1072 _igvn.hash_delete(lower_insert_pt); | |
| 1073 use->set_req(MemNode::Memory, lower_insert_pt); | |
| 1074 } | |
| 0 | 1075 _igvn._worklist.push(use); |
| 1076 --i; // deleted this edge; rescan position | |
| 1077 } | |
| 1078 } | |
| 667 | 1079 previous = current; |
| 1080 } else { // !in_pack(current, pk) ==> a sandwiched store | |
| 1081 remove_and_insert(current, previous, lower_insert_pt, upper_insert_pt, schedule_before_pack); | |
| 0 | 1082 } |
| 667 | 1083 |
| 0 | 1084 if (current == first) break; |
| 1085 current = my_mem->as_Mem(); | |
| 667 | 1086 } // end while |
| 1087 } else if (pk->at(0)->is_Load()) { //load | |
| 952 | 1088 // all loads in the pack should have the same memory state. By default, |
| 1089 // we use the memory state of the last load. However, if any load could | |
| 1090 // not be moved down due to the dependence constraint, we use the memory | |
| 1091 // state of the first load. | |
| 1092 Node* last_mem = executed_last(pk)->in(MemNode::Memory); | |
| 1093 Node* first_mem = executed_first(pk)->in(MemNode::Memory); | |
| 1094 bool schedule_last = true; | |
| 1095 for (uint i = 0; i < pk->size(); i++) { | |
| 1096 Node* ld = pk->at(i); | |
| 1097 for (Node* current = last_mem; current != ld->in(MemNode::Memory); | |
| 1098 current=current->in(MemNode::Memory)) { | |
| 1099 assert(current != first_mem, "corrupted memory graph"); | |
| 1100 if(current->is_Mem() && !independent(current, ld)){ | |
| 1101 schedule_last = false; // a later store depends on this load | |
| 1102 break; | |
| 1103 } | |
| 1104 } | |
| 1105 } | |
| 1106 | |
| 1107 Node* mem_input = schedule_last ? last_mem : first_mem; | |
| 1108 _igvn.hash_delete(mem_input); | |
| 1109 // Give each load the same memory state | |
| 0 | 1110 for (uint i = 0; i < pk->size(); i++) { |
| 1111 LoadNode* ld = pk->at(i)->as_Load(); | |
| 1112 _igvn.hash_delete(ld); | |
| 952 | 1113 ld->set_req(MemNode::Memory, mem_input); |
| 0 | 1114 _igvn._worklist.push(ld); |
| 1115 } | |
| 1116 } | |
| 1117 } | |
| 1118 | |
| 1119 //------------------------------output--------------------------- | |
| 1120 // Convert packs into vector node operations | |
| 1121 void SuperWord::output() { | |
| 1122 if (_packset.length() == 0) return; | |
| 1123 | |
| 1124 // MUST ENSURE main loop's initial value is properly aligned: | |
| 1125 // (iv_initial_value + min_iv_offset) % vector_width_in_bytes() == 0 | |
| 1126 | |
| 1127 align_initial_loop_index(align_to_ref()); | |
| 1128 | |
| 1129 // Insert extract (unpack) operations for scalar uses | |
| 1130 for (int i = 0; i < _packset.length(); i++) { | |
| 1131 insert_extracts(_packset.at(i)); | |
| 1132 } | |
| 1133 | |
| 1134 for (int i = 0; i < _block.length(); i++) { | |
| 1135 Node* n = _block.at(i); | |
| 1136 Node_List* p = my_pack(n); | |
| 1137 if (p && n == executed_last(p)) { | |
| 1138 uint vlen = p->size(); | |
| 1139 Node* vn = NULL; | |
| 1140 Node* low_adr = p->at(0); | |
| 1141 Node* first = executed_first(p); | |
| 1142 if (n->is_Load()) { | |
| 1143 int opc = n->Opcode(); | |
| 1144 Node* ctl = n->in(MemNode::Control); | |
| 1145 Node* mem = first->in(MemNode::Memory); | |
| 1146 Node* adr = low_adr->in(MemNode::Address); | |
| 1147 const TypePtr* atyp = n->adr_type(); | |
| 1148 vn = VectorLoadNode::make(_phase->C, opc, ctl, mem, adr, atyp, vlen); | |
| 1149 | |
| 1150 } else if (n->is_Store()) { | |
| 1151 // Promote value to be stored to vector | |
| 1152 VectorNode* val = vector_opd(p, MemNode::ValueIn); | |
| 1153 | |
| 1154 int opc = n->Opcode(); | |
| 1155 Node* ctl = n->in(MemNode::Control); | |
| 1156 Node* mem = first->in(MemNode::Memory); | |
| 1157 Node* adr = low_adr->in(MemNode::Address); | |
| 1158 const TypePtr* atyp = n->adr_type(); | |
| 1159 vn = VectorStoreNode::make(_phase->C, opc, ctl, mem, adr, atyp, val, vlen); | |
| 1160 | |
| 1161 } else if (n->req() == 3) { | |
| 1162 // Promote operands to vector | |
| 1163 Node* in1 = vector_opd(p, 1); | |
| 1164 Node* in2 = vector_opd(p, 2); | |
| 1165 vn = VectorNode::make(_phase->C, n->Opcode(), in1, in2, vlen, velt_type(n)); | |
| 1166 | |
| 1167 } else { | |
| 1168 ShouldNotReachHere(); | |
| 1169 } | |
| 1170 | |
| 1171 _phase->_igvn.register_new_node_with_optimizer(vn); | |
| 1172 _phase->set_ctrl(vn, _phase->get_ctrl(p->at(0))); | |
| 1173 for (uint j = 0; j < p->size(); j++) { | |
| 1174 Node* pm = p->at(j); | |
|
1621
6027dddc26c6
6677629: PhaseIterGVN::subsume_node() should call hash_delete() and add_users_to_worklist()
kvn
parents:
1585
diff
changeset
|
1175 _igvn.replace_node(pm, vn); |
| 0 | 1176 } |
| 1177 _igvn._worklist.push(vn); | |
| 1178 } | |
| 1179 } | |
| 1180 } | |
| 1181 | |
| 1182 //------------------------------vector_opd--------------------------- | |
| 1183 // Create a vector operand for the nodes in pack p for operand: in(opd_idx) | |
| 1184 VectorNode* SuperWord::vector_opd(Node_List* p, int opd_idx) { | |
| 1185 Node* p0 = p->at(0); | |
| 1186 uint vlen = p->size(); | |
| 1187 Node* opd = p0->in(opd_idx); | |
| 1188 | |
| 1189 bool same_opd = true; | |
| 1190 for (uint i = 1; i < vlen; i++) { | |
| 1191 Node* pi = p->at(i); | |
| 1192 Node* in = pi->in(opd_idx); | |
| 1193 if (opd != in) { | |
| 1194 same_opd = false; | |
| 1195 break; | |
| 1196 } | |
| 1197 } | |
| 1198 | |
| 1199 if (same_opd) { | |
| 1200 if (opd->is_Vector()) { | |
| 1201 return (VectorNode*)opd; // input is matching vector | |
| 1202 } | |
| 1203 // Convert scalar input to vector. Use p0's type because it's container | |
| 1204 // maybe smaller than the operand's container. | |
| 1205 const Type* opd_t = velt_type(!in_bb(opd) ? p0 : opd); | |
| 1206 const Type* p0_t = velt_type(p0); | |
| 1207 if (p0_t->higher_equal(opd_t)) opd_t = p0_t; | |
| 1208 VectorNode* vn = VectorNode::scalar2vector(_phase->C, opd, vlen, opd_t); | |
| 1209 | |
| 1210 _phase->_igvn.register_new_node_with_optimizer(vn); | |
| 1211 _phase->set_ctrl(vn, _phase->get_ctrl(opd)); | |
| 1212 return vn; | |
| 1213 } | |
| 1214 | |
| 1215 // Insert pack operation | |
| 1216 const Type* opd_t = velt_type(!in_bb(opd) ? p0 : opd); | |
| 1217 PackNode* pk = PackNode::make(_phase->C, opd, opd_t); | |
| 1218 | |
| 1219 for (uint i = 1; i < vlen; i++) { | |
| 1220 Node* pi = p->at(i); | |
| 1221 Node* in = pi->in(opd_idx); | |
| 1222 assert(my_pack(in) == NULL, "Should already have been unpacked"); | |
| 1223 assert(opd_t == velt_type(!in_bb(in) ? pi : in), "all same type"); | |
| 1224 pk->add_opd(in); | |
| 1225 } | |
| 1226 _phase->_igvn.register_new_node_with_optimizer(pk); | |
| 1227 _phase->set_ctrl(pk, _phase->get_ctrl(opd)); | |
| 1228 return pk; | |
| 1229 } | |
| 1230 | |
| 1231 //------------------------------insert_extracts--------------------------- | |
| 1232 // If a use of pack p is not a vector use, then replace the | |
| 1233 // use with an extract operation. | |
| 1234 void SuperWord::insert_extracts(Node_List* p) { | |
| 1235 if (p->at(0)->is_Store()) return; | |
| 1236 assert(_n_idx_list.is_empty(), "empty (node,index) list"); | |
| 1237 | |
| 1238 // Inspect each use of each pack member. For each use that is | |
| 1239 // not a vector use, replace the use with an extract operation. | |
| 1240 | |
| 1241 for (uint i = 0; i < p->size(); i++) { | |
| 1242 Node* def = p->at(i); | |
| 1243 for (DUIterator_Fast jmax, j = def->fast_outs(jmax); j < jmax; j++) { | |
| 1244 Node* use = def->fast_out(j); | |
| 1245 for (uint k = 0; k < use->req(); k++) { | |
| 1246 Node* n = use->in(k); | |
| 1247 if (def == n) { | |
| 1248 if (!is_vector_use(use, k)) { | |
| 1249 _n_idx_list.push(use, k); | |
| 1250 } | |
| 1251 } | |
| 1252 } | |
| 1253 } | |
| 1254 } | |
| 1255 | |
| 1256 while (_n_idx_list.is_nonempty()) { | |
| 1257 Node* use = _n_idx_list.node(); | |
| 1258 int idx = _n_idx_list.index(); | |
| 1259 _n_idx_list.pop(); | |
| 1260 Node* def = use->in(idx); | |
| 1261 | |
| 1262 // Insert extract operation | |
| 1263 _igvn.hash_delete(def); | |
| 1264 _igvn.hash_delete(use); | |
| 1265 int def_pos = alignment(def) / data_size(def); | |
| 1266 const Type* def_t = velt_type(def); | |
| 1267 | |
| 1268 Node* ex = ExtractNode::make(_phase->C, def, def_pos, def_t); | |
| 1269 _phase->_igvn.register_new_node_with_optimizer(ex); | |
| 1270 _phase->set_ctrl(ex, _phase->get_ctrl(def)); | |
| 1271 use->set_req(idx, ex); | |
| 1272 _igvn._worklist.push(def); | |
| 1273 _igvn._worklist.push(use); | |
| 1274 | |
| 1275 bb_insert_after(ex, bb_idx(def)); | |
| 1276 set_velt_type(ex, def_t); | |
| 1277 } | |
| 1278 } | |
| 1279 | |
| 1280 //------------------------------is_vector_use--------------------------- | |
| 1281 // Is use->in(u_idx) a vector use? | |
| 1282 bool SuperWord::is_vector_use(Node* use, int u_idx) { | |
| 1283 Node_List* u_pk = my_pack(use); | |
| 1284 if (u_pk == NULL) return false; | |
| 1285 Node* def = use->in(u_idx); | |
| 1286 Node_List* d_pk = my_pack(def); | |
| 1287 if (d_pk == NULL) { | |
| 1288 // check for scalar promotion | |
| 1289 Node* n = u_pk->at(0)->in(u_idx); | |
| 1290 for (uint i = 1; i < u_pk->size(); i++) { | |
| 1291 if (u_pk->at(i)->in(u_idx) != n) return false; | |
| 1292 } | |
| 1293 return true; | |
| 1294 } | |
| 1295 if (u_pk->size() != d_pk->size()) | |
| 1296 return false; | |
| 1297 for (uint i = 0; i < u_pk->size(); i++) { | |
| 1298 Node* ui = u_pk->at(i); | |
| 1299 Node* di = d_pk->at(i); | |
| 1300 if (ui->in(u_idx) != di || alignment(ui) != alignment(di)) | |
| 1301 return false; | |
| 1302 } | |
| 1303 return true; | |
| 1304 } | |
| 1305 | |
| 1306 //------------------------------construct_bb--------------------------- | |
| 1307 // Construct reverse postorder list of block members | |
| 1308 void SuperWord::construct_bb() { | |
| 1309 Node* entry = bb(); | |
| 1310 | |
| 1311 assert(_stk.length() == 0, "stk is empty"); | |
| 1312 assert(_block.length() == 0, "block is empty"); | |
| 1313 assert(_data_entry.length() == 0, "data_entry is empty"); | |
| 1314 assert(_mem_slice_head.length() == 0, "mem_slice_head is empty"); | |
| 1315 assert(_mem_slice_tail.length() == 0, "mem_slice_tail is empty"); | |
| 1316 | |
| 1317 // Find non-control nodes with no inputs from within block, | |
| 1318 // create a temporary map from node _idx to bb_idx for use | |
| 1319 // by the visited and post_visited sets, | |
| 1320 // and count number of nodes in block. | |
| 1321 int bb_ct = 0; | |
| 1322 for (uint i = 0; i < lpt()->_body.size(); i++ ) { | |
| 1323 Node *n = lpt()->_body.at(i); | |
| 1324 set_bb_idx(n, i); // Create a temporary map | |
| 1325 if (in_bb(n)) { | |
| 1326 bb_ct++; | |
| 1327 if (!n->is_CFG()) { | |
| 1328 bool found = false; | |
| 1329 for (uint j = 0; j < n->req(); j++) { | |
| 1330 Node* def = n->in(j); | |
| 1331 if (def && in_bb(def)) { | |
| 1332 found = true; | |
| 1333 break; | |
| 1334 } | |
| 1335 } | |
| 1336 if (!found) { | |
| 1337 assert(n != entry, "can't be entry"); | |
| 1338 _data_entry.push(n); | |
| 1339 } | |
| 1340 } | |
| 1341 } | |
| 1342 } | |
| 1343 | |
| 1344 // Find memory slices (head and tail) | |
| 1345 for (DUIterator_Fast imax, i = lp()->fast_outs(imax); i < imax; i++) { | |
| 1346 Node *n = lp()->fast_out(i); | |
| 1347 if (in_bb(n) && (n->is_Phi() && n->bottom_type() == Type::MEMORY)) { | |
| 1348 Node* n_tail = n->in(LoopNode::LoopBackControl); | |
|
253
b0fe4deeb9fb
6726999: nsk/stress/jck12a/jck12a010 assert(n != null,"Bad immediate dominator info.")
kvn
parents:
235
diff
changeset
|
1349 if (n_tail != n->in(LoopNode::EntryControl)) { |
|
b0fe4deeb9fb
6726999: nsk/stress/jck12a/jck12a010 assert(n != null,"Bad immediate dominator info.")
kvn
parents:
235
diff
changeset
|
1350 _mem_slice_head.push(n); |
|
b0fe4deeb9fb
6726999: nsk/stress/jck12a/jck12a010 assert(n != null,"Bad immediate dominator info.")
kvn
parents:
235
diff
changeset
|
1351 _mem_slice_tail.push(n_tail); |
|
b0fe4deeb9fb
6726999: nsk/stress/jck12a/jck12a010 assert(n != null,"Bad immediate dominator info.")
kvn
parents:
235
diff
changeset
|
1352 } |
| 0 | 1353 } |
| 1354 } | |
| 1355 | |
| 1356 // Create an RPO list of nodes in block | |
| 1357 | |
| 1358 visited_clear(); | |
| 1359 post_visited_clear(); | |
| 1360 | |
| 1361 // Push all non-control nodes with no inputs from within block, then control entry | |
| 1362 for (int j = 0; j < _data_entry.length(); j++) { | |
| 1363 Node* n = _data_entry.at(j); | |
| 1364 visited_set(n); | |
| 1365 _stk.push(n); | |
| 1366 } | |
| 1367 visited_set(entry); | |
| 1368 _stk.push(entry); | |
| 1369 | |
| 1370 // Do a depth first walk over out edges | |
| 1371 int rpo_idx = bb_ct - 1; | |
| 1372 int size; | |
| 1373 while ((size = _stk.length()) > 0) { | |
| 1374 Node* n = _stk.top(); // Leave node on stack | |
| 1375 if (!visited_test_set(n)) { | |
| 1376 // forward arc in graph | |
| 1377 } else if (!post_visited_test(n)) { | |
| 1378 // cross or back arc | |
| 1379 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { | |
| 1380 Node *use = n->fast_out(i); | |
| 1381 if (in_bb(use) && !visited_test(use) && | |
| 1382 // Don't go around backedge | |
| 1383 (!use->is_Phi() || n == entry)) { | |
| 1384 _stk.push(use); | |
| 1385 } | |
| 1386 } | |
| 1387 if (_stk.length() == size) { | |
| 1388 // There were no additional uses, post visit node now | |
| 1389 _stk.pop(); // Remove node from stack | |
| 1390 assert(rpo_idx >= 0, ""); | |
| 1391 _block.at_put_grow(rpo_idx, n); | |
| 1392 rpo_idx--; | |
| 1393 post_visited_set(n); | |
| 1394 assert(rpo_idx >= 0 || _stk.is_empty(), ""); | |
| 1395 } | |
| 1396 } else { | |
| 1397 _stk.pop(); // Remove post-visited node from stack | |
| 1398 } | |
| 1399 } | |
| 1400 | |
| 1401 // Create real map of block indices for nodes | |
| 1402 for (int j = 0; j < _block.length(); j++) { | |
| 1403 Node* n = _block.at(j); | |
| 1404 set_bb_idx(n, j); | |
| 1405 } | |
| 1406 | |
| 1407 initialize_bb(); // Ensure extra info is allocated. | |
| 1408 | |
| 1409 #ifndef PRODUCT | |
| 1410 if (TraceSuperWord) { | |
| 1411 print_bb(); | |
| 1412 tty->print_cr("\ndata entry nodes: %s", _data_entry.length() > 0 ? "" : "NONE"); | |
| 1413 for (int m = 0; m < _data_entry.length(); m++) { | |
| 1414 tty->print("%3d ", m); | |
| 1415 _data_entry.at(m)->dump(); | |
| 1416 } | |
| 1417 tty->print_cr("\nmemory slices: %s", _mem_slice_head.length() > 0 ? "" : "NONE"); | |
| 1418 for (int m = 0; m < _mem_slice_head.length(); m++) { | |
| 1419 tty->print("%3d ", m); _mem_slice_head.at(m)->dump(); | |
| 1420 tty->print(" "); _mem_slice_tail.at(m)->dump(); | |
| 1421 } | |
| 1422 } | |
| 1423 #endif | |
| 1424 assert(rpo_idx == -1 && bb_ct == _block.length(), "all block members found"); | |
| 1425 } | |
| 1426 | |
| 1427 //------------------------------initialize_bb--------------------------- | |
| 1428 // Initialize per node info | |
| 1429 void SuperWord::initialize_bb() { | |
| 1430 Node* last = _block.at(_block.length() - 1); | |
| 1431 grow_node_info(bb_idx(last)); | |
| 1432 } | |
| 1433 | |
| 1434 //------------------------------bb_insert_after--------------------------- | |
| 1435 // Insert n into block after pos | |
| 1436 void SuperWord::bb_insert_after(Node* n, int pos) { | |
| 1437 int n_pos = pos + 1; | |
| 1438 // Make room | |
| 1439 for (int i = _block.length() - 1; i >= n_pos; i--) { | |
| 1440 _block.at_put_grow(i+1, _block.at(i)); | |
| 1441 } | |
| 1442 for (int j = _node_info.length() - 1; j >= n_pos; j--) { | |
| 1443 _node_info.at_put_grow(j+1, _node_info.at(j)); | |
| 1444 } | |
| 1445 // Set value | |
| 1446 _block.at_put_grow(n_pos, n); | |
| 1447 _node_info.at_put_grow(n_pos, SWNodeInfo::initial); | |
| 1448 // Adjust map from node->_idx to _block index | |
| 1449 for (int i = n_pos; i < _block.length(); i++) { | |
| 1450 set_bb_idx(_block.at(i), i); | |
| 1451 } | |
| 1452 } | |
| 1453 | |
| 1454 //------------------------------compute_max_depth--------------------------- | |
| 1455 // Compute max depth for expressions from beginning of block | |
| 1456 // Use to prune search paths during test for independence. | |
| 1457 void SuperWord::compute_max_depth() { | |
| 1458 int ct = 0; | |
| 1459 bool again; | |
| 1460 do { | |
| 1461 again = false; | |
| 1462 for (int i = 0; i < _block.length(); i++) { | |
| 1463 Node* n = _block.at(i); | |
| 1464 if (!n->is_Phi()) { | |
| 1465 int d_orig = depth(n); | |
| 1466 int d_in = 0; | |
| 1467 for (DepPreds preds(n, _dg); !preds.done(); preds.next()) { | |
| 1468 Node* pred = preds.current(); | |
| 1469 if (in_bb(pred)) { | |
| 1470 d_in = MAX2(d_in, depth(pred)); | |
| 1471 } | |
| 1472 } | |
| 1473 if (d_in + 1 != d_orig) { | |
| 1474 set_depth(n, d_in + 1); | |
| 1475 again = true; | |
| 1476 } | |
| 1477 } | |
| 1478 } | |
| 1479 ct++; | |
| 1480 } while (again); | |
| 1481 #ifndef PRODUCT | |
| 1482 if (TraceSuperWord && Verbose) | |
| 1483 tty->print_cr("compute_max_depth iterated: %d times", ct); | |
| 1484 #endif | |
| 1485 } | |
| 1486 | |
| 1487 //-------------------------compute_vector_element_type----------------------- | |
| 1488 // Compute necessary vector element type for expressions | |
| 1489 // This propagates backwards a narrower integer type when the | |
| 1490 // upper bits of the value are not needed. | |
| 1491 // Example: char a,b,c; a = b + c; | |
| 1492 // Normally the type of the add is integer, but for packed character | |
| 1493 // operations the type of the add needs to be char. | |
| 1494 void SuperWord::compute_vector_element_type() { | |
| 1495 #ifndef PRODUCT | |
| 1496 if (TraceSuperWord && Verbose) | |
| 1497 tty->print_cr("\ncompute_velt_type:"); | |
| 1498 #endif | |
| 1499 | |
| 1500 // Initial type | |
| 1501 for (int i = 0; i < _block.length(); i++) { | |
| 1502 Node* n = _block.at(i); | |
| 1503 const Type* t = n->is_Mem() ? Type::get_const_basic_type(n->as_Mem()->memory_type()) | |
| 1504 : _igvn.type(n); | |
| 1505 const Type* vt = container_type(t); | |
| 1506 set_velt_type(n, vt); | |
| 1507 } | |
| 1508 | |
| 1509 // Propagate narrowed type backwards through operations | |
| 1510 // that don't depend on higher order bits | |
| 1511 for (int i = _block.length() - 1; i >= 0; i--) { | |
| 1512 Node* n = _block.at(i); | |
| 1513 // Only integer types need be examined | |
| 1514 if (n->bottom_type()->isa_int()) { | |
| 1515 uint start, end; | |
| 1516 vector_opd_range(n, &start, &end); | |
| 1517 const Type* vt = velt_type(n); | |
| 1518 | |
| 1519 for (uint j = start; j < end; j++) { | |
| 1520 Node* in = n->in(j); | |
| 1521 // Don't propagate through a type conversion | |
| 1522 if (n->bottom_type() != in->bottom_type()) | |
| 1523 continue; | |
| 1524 switch(in->Opcode()) { | |
| 1525 case Op_AddI: case Op_AddL: | |
| 1526 case Op_SubI: case Op_SubL: | |
| 1527 case Op_MulI: case Op_MulL: | |
| 1528 case Op_AndI: case Op_AndL: | |
| 1529 case Op_OrI: case Op_OrL: | |
| 1530 case Op_XorI: case Op_XorL: | |
| 1531 case Op_LShiftI: case Op_LShiftL: | |
| 1532 case Op_CMoveI: case Op_CMoveL: | |
| 1533 if (in_bb(in)) { | |
| 1534 bool same_type = true; | |
| 1535 for (DUIterator_Fast kmax, k = in->fast_outs(kmax); k < kmax; k++) { | |
| 1536 Node *use = in->fast_out(k); | |
| 1537 if (!in_bb(use) || velt_type(use) != vt) { | |
| 1538 same_type = false; | |
| 1539 break; | |
| 1540 } | |
| 1541 } | |
| 1542 if (same_type) { | |
| 1543 set_velt_type(in, vt); | |
| 1544 } | |
| 1545 } | |
| 1546 } | |
| 1547 } | |
| 1548 } | |
| 1549 } | |
| 1550 #ifndef PRODUCT | |
| 1551 if (TraceSuperWord && Verbose) { | |
| 1552 for (int i = 0; i < _block.length(); i++) { | |
| 1553 Node* n = _block.at(i); | |
| 1554 velt_type(n)->dump(); | |
| 1555 tty->print("\t"); | |
| 1556 n->dump(); | |
| 1557 } | |
| 1558 } | |
| 1559 #endif | |
| 1560 } | |
| 1561 | |
| 1562 //------------------------------memory_alignment--------------------------- | |
| 1563 // Alignment within a vector memory reference | |
| 1564 int SuperWord::memory_alignment(MemNode* s, int iv_adjust_in_bytes) { | |
| 1565 SWPointer p(s, this); | |
| 1566 if (!p.valid()) { | |
| 1567 return bottom_align; | |
| 1568 } | |
| 1569 int offset = p.offset_in_bytes(); | |
| 1570 offset += iv_adjust_in_bytes; | |
| 1571 int off_rem = offset % vector_width_in_bytes(); | |
| 1572 int off_mod = off_rem >= 0 ? off_rem : off_rem + vector_width_in_bytes(); | |
| 1573 return off_mod; | |
| 1574 } | |
| 1575 | |
| 1576 //---------------------------container_type--------------------------- | |
| 1577 // Smallest type containing range of values | |
| 1578 const Type* SuperWord::container_type(const Type* t) { | |
|
221
1e026f8da827
6710487: More than half of JDI Regression tests hang with COOPs in -Xcomp mode
kvn
parents:
113
diff
changeset
|
1579 const Type* tp = t->make_ptr(); |
|
1e026f8da827
6710487: More than half of JDI Regression tests hang with COOPs in -Xcomp mode
kvn
parents:
113
diff
changeset
|
1580 if (tp && tp->isa_aryptr()) { |
|
1e026f8da827
6710487: More than half of JDI Regression tests hang with COOPs in -Xcomp mode
kvn
parents:
113
diff
changeset
|
1581 t = tp->is_aryptr()->elem(); |
| 0 | 1582 } |
| 1583 if (t->basic_type() == T_INT) { | |
| 1584 if (t->higher_equal(TypeInt::BOOL)) return TypeInt::BOOL; | |
| 1585 if (t->higher_equal(TypeInt::BYTE)) return TypeInt::BYTE; | |
| 1586 if (t->higher_equal(TypeInt::CHAR)) return TypeInt::CHAR; | |
| 1587 if (t->higher_equal(TypeInt::SHORT)) return TypeInt::SHORT; | |
| 1588 return TypeInt::INT; | |
| 1589 } | |
| 1590 return t; | |
| 1591 } | |
| 1592 | |
| 1593 //-------------------------vector_opd_range----------------------- | |
| 1594 // (Start, end] half-open range defining which operands are vector | |
| 1595 void SuperWord::vector_opd_range(Node* n, uint* start, uint* end) { | |
| 1596 switch (n->Opcode()) { | |
|
558
3b5ac9e7e6ea
6796746: rename LoadC (char) opcode class to LoadUS (unsigned short)
twisti
parents:
253
diff
changeset
|
1597 case Op_LoadB: case Op_LoadUS: |
| 0 | 1598 case Op_LoadI: case Op_LoadL: |
| 1599 case Op_LoadF: case Op_LoadD: | |
| 1600 case Op_LoadP: | |
| 1601 *start = 0; | |
| 1602 *end = 0; | |
| 1603 return; | |
| 1604 case Op_StoreB: case Op_StoreC: | |
| 1605 case Op_StoreI: case Op_StoreL: | |
| 1606 case Op_StoreF: case Op_StoreD: | |
| 1607 case Op_StoreP: | |
| 1608 *start = MemNode::ValueIn; | |
| 1609 *end = *start + 1; | |
| 1610 return; | |
| 1611 case Op_LShiftI: case Op_LShiftL: | |
| 1612 *start = 1; | |
| 1613 *end = 2; | |
| 1614 return; | |
| 1615 case Op_CMoveI: case Op_CMoveL: case Op_CMoveF: case Op_CMoveD: | |
| 1616 *start = 2; | |
| 1617 *end = n->req(); | |
| 1618 return; | |
| 1619 } | |
| 1620 *start = 1; | |
| 1621 *end = n->req(); // default is all operands | |
| 1622 } | |
| 1623 | |
| 1624 //------------------------------in_packset--------------------------- | |
| 1625 // Are s1 and s2 in a pack pair and ordered as s1,s2? | |
| 1626 bool SuperWord::in_packset(Node* s1, Node* s2) { | |
| 1627 for (int i = 0; i < _packset.length(); i++) { | |
| 1628 Node_List* p = _packset.at(i); | |
| 1629 assert(p->size() == 2, "must be"); | |
| 1630 if (p->at(0) == s1 && p->at(p->size()-1) == s2) { | |
| 1631 return true; | |
| 1632 } | |
| 1633 } | |
| 1634 return false; | |
| 1635 } | |
| 1636 | |
| 1637 //------------------------------in_pack--------------------------- | |
| 1638 // Is s in pack p? | |
| 1639 Node_List* SuperWord::in_pack(Node* s, Node_List* p) { | |
| 1640 for (uint i = 0; i < p->size(); i++) { | |
| 1641 if (p->at(i) == s) { | |
| 1642 return p; | |
| 1643 } | |
| 1644 } | |
| 1645 return NULL; | |
| 1646 } | |
| 1647 | |
| 1648 //------------------------------remove_pack_at--------------------------- | |
| 1649 // Remove the pack at position pos in the packset | |
| 1650 void SuperWord::remove_pack_at(int pos) { | |
| 1651 Node_List* p = _packset.at(pos); | |
| 1652 for (uint i = 0; i < p->size(); i++) { | |
| 1653 Node* s = p->at(i); | |
| 1654 set_my_pack(s, NULL); | |
| 1655 } | |
| 1656 _packset.remove_at(pos); | |
| 1657 } | |
| 1658 | |
| 1659 //------------------------------executed_first--------------------------- | |
| 1660 // Return the node executed first in pack p. Uses the RPO block list | |
| 1661 // to determine order. | |
| 1662 Node* SuperWord::executed_first(Node_List* p) { | |
| 1663 Node* n = p->at(0); | |
| 1664 int n_rpo = bb_idx(n); | |
| 1665 for (uint i = 1; i < p->size(); i++) { | |
| 1666 Node* s = p->at(i); | |
| 1667 int s_rpo = bb_idx(s); | |
| 1668 if (s_rpo < n_rpo) { | |
| 1669 n = s; | |
| 1670 n_rpo = s_rpo; | |
| 1671 } | |
| 1672 } | |
| 1673 return n; | |
| 1674 } | |
| 1675 | |
| 1676 //------------------------------executed_last--------------------------- | |
| 1677 // Return the node executed last in pack p. | |
| 1678 Node* SuperWord::executed_last(Node_List* p) { | |
| 1679 Node* n = p->at(0); | |
| 1680 int n_rpo = bb_idx(n); | |
| 1681 for (uint i = 1; i < p->size(); i++) { | |
| 1682 Node* s = p->at(i); | |
| 1683 int s_rpo = bb_idx(s); | |
| 1684 if (s_rpo > n_rpo) { | |
| 1685 n = s; | |
| 1686 n_rpo = s_rpo; | |
| 1687 } | |
| 1688 } | |
| 1689 return n; | |
| 1690 } | |
| 1691 | |
| 1692 //----------------------------align_initial_loop_index--------------------------- | |
| 1693 // Adjust pre-loop limit so that in main loop, a load/store reference | |
| 1694 // to align_to_ref will be a position zero in the vector. | |
| 1695 // (iv + k) mod vector_align == 0 | |
| 1696 void SuperWord::align_initial_loop_index(MemNode* align_to_ref) { | |
| 1697 CountedLoopNode *main_head = lp()->as_CountedLoop(); | |
| 1698 assert(main_head->is_main_loop(), ""); | |
| 1699 CountedLoopEndNode* pre_end = get_pre_loop_end(main_head); | |
| 1700 assert(pre_end != NULL, ""); | |
| 1701 Node *pre_opaq1 = pre_end->limit(); | |
| 1702 assert(pre_opaq1->Opcode() == Op_Opaque1, ""); | |
| 1703 Opaque1Node *pre_opaq = (Opaque1Node*)pre_opaq1; | |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1704 Node *lim0 = pre_opaq->in(1); |
| 0 | 1705 |
| 1706 // Where we put new limit calculations | |
| 1707 Node *pre_ctrl = pre_end->loopnode()->in(LoopNode::EntryControl); | |
| 1708 | |
| 1709 // Ensure the original loop limit is available from the | |
| 1710 // pre-loop Opaque1 node. | |
| 1711 Node *orig_limit = pre_opaq->original_loop_limit(); | |
| 1712 assert(orig_limit != NULL && _igvn.type(orig_limit) != Type::TOP, ""); | |
| 1713 | |
| 1714 SWPointer align_to_ref_p(align_to_ref, this); | |
| 1715 | |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1716 // Given: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1717 // lim0 == original pre loop limit |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1718 // V == v_align (power of 2) |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1719 // invar == extra invariant piece of the address expression |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1720 // e == k [ +/- invar ] |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1721 // |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1722 // When reassociating expressions involving '%' the basic rules are: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1723 // (a - b) % k == 0 => a % k == b % k |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1724 // and: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1725 // (a + b) % k == 0 => a % k == (k - b) % k |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1726 // |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1727 // For stride > 0 && scale > 0, |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1728 // Derive the new pre-loop limit "lim" such that the two constraints: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1729 // (1) lim = lim0 + N (where N is some positive integer < V) |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1730 // (2) (e + lim) % V == 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1731 // are true. |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1732 // |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1733 // Substituting (1) into (2), |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1734 // (e + lim0 + N) % V == 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1735 // solve for N: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1736 // N = (V - (e + lim0)) % V |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1737 // substitute back into (1), so that new limit |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1738 // lim = lim0 + (V - (e + lim0)) % V |
| 0 | 1739 // |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1740 // For stride > 0 && scale < 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1741 // Constraints: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1742 // lim = lim0 + N |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1743 // (e - lim) % V == 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1744 // Solving for lim: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1745 // (e - lim0 - N) % V == 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1746 // N = (e - lim0) % V |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1747 // lim = lim0 + (e - lim0) % V |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1748 // |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1749 // For stride < 0 && scale > 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1750 // Constraints: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1751 // lim = lim0 - N |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1752 // (e + lim) % V == 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1753 // Solving for lim: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1754 // (e + lim0 - N) % V == 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1755 // N = (e + lim0) % V |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1756 // lim = lim0 - (e + lim0) % V |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1757 // |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1758 // For stride < 0 && scale < 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1759 // Constraints: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1760 // lim = lim0 - N |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1761 // (e - lim) % V == 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1762 // Solving for lim: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1763 // (e - lim0 + N) % V == 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1764 // N = (V - (e - lim0)) % V |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1765 // lim = lim0 - (V - (e - lim0)) % V |
| 0 | 1766 |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1767 int stride = iv_stride(); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1768 int scale = align_to_ref_p.scale_in_bytes(); |
| 0 | 1769 int elt_size = align_to_ref_p.memory_size(); |
| 1770 int v_align = vector_width_in_bytes() / elt_size; | |
| 1771 int k = align_to_ref_p.offset_in_bytes() / elt_size; | |
| 1772 | |
| 1773 Node *kn = _igvn.intcon(k); | |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1774 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1775 Node *e = kn; |
| 0 | 1776 if (align_to_ref_p.invar() != NULL) { |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1777 // incorporate any extra invariant piece producing k +/- invar >>> log2(elt) |
| 0 | 1778 Node* log2_elt = _igvn.intcon(exact_log2(elt_size)); |
| 1779 Node* aref = new (_phase->C, 3) URShiftINode(align_to_ref_p.invar(), log2_elt); | |
| 1780 _phase->_igvn.register_new_node_with_optimizer(aref); | |
| 1781 _phase->set_ctrl(aref, pre_ctrl); | |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1782 if (align_to_ref_p.negate_invar()) { |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1783 e = new (_phase->C, 3) SubINode(e, aref); |
| 0 | 1784 } else { |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1785 e = new (_phase->C, 3) AddINode(e, aref); |
| 0 | 1786 } |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1787 _phase->_igvn.register_new_node_with_optimizer(e); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1788 _phase->set_ctrl(e, pre_ctrl); |
| 0 | 1789 } |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1790 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1791 // compute e +/- lim0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1792 if (scale < 0) { |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1793 e = new (_phase->C, 3) SubINode(e, lim0); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1794 } else { |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1795 e = new (_phase->C, 3) AddINode(e, lim0); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1796 } |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1797 _phase->_igvn.register_new_node_with_optimizer(e); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1798 _phase->set_ctrl(e, pre_ctrl); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1799 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1800 if (stride * scale > 0) { |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1801 // compute V - (e +/- lim0) |
| 0 | 1802 Node* va = _igvn.intcon(v_align); |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1803 e = new (_phase->C, 3) SubINode(va, e); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1804 _phase->_igvn.register_new_node_with_optimizer(e); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1805 _phase->set_ctrl(e, pre_ctrl); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1806 } |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1807 // compute N = (exp) % V |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1808 Node* va_msk = _igvn.intcon(v_align - 1); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1809 Node* N = new (_phase->C, 3) AndINode(e, va_msk); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1810 _phase->_igvn.register_new_node_with_optimizer(N); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1811 _phase->set_ctrl(N, pre_ctrl); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1812 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1813 // substitute back into (1), so that new limit |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1814 // lim = lim0 + N |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1815 Node* lim; |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1816 if (stride < 0) { |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1817 lim = new (_phase->C, 3) SubINode(lim0, N); |
| 0 | 1818 } else { |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1819 lim = new (_phase->C, 3) AddINode(lim0, N); |
| 0 | 1820 } |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1821 _phase->_igvn.register_new_node_with_optimizer(lim); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1822 _phase->set_ctrl(lim, pre_ctrl); |
| 0 | 1823 Node* constrained = |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1824 (stride > 0) ? (Node*) new (_phase->C,3) MinINode(lim, orig_limit) |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1825 : (Node*) new (_phase->C,3) MaxINode(lim, orig_limit); |
| 0 | 1826 _phase->_igvn.register_new_node_with_optimizer(constrained); |
| 1827 _phase->set_ctrl(constrained, pre_ctrl); | |
| 1828 _igvn.hash_delete(pre_opaq); | |
| 1829 pre_opaq->set_req(1, constrained); | |
| 1830 } | |
| 1831 | |
| 1832 //----------------------------get_pre_loop_end--------------------------- | |
| 1833 // Find pre loop end from main loop. Returns null if none. | |
| 1834 CountedLoopEndNode* SuperWord::get_pre_loop_end(CountedLoopNode *cl) { | |
| 1835 Node *ctrl = cl->in(LoopNode::EntryControl); | |
| 1836 if (!ctrl->is_IfTrue() && !ctrl->is_IfFalse()) return NULL; | |
| 1837 Node *iffm = ctrl->in(0); | |
| 1838 if (!iffm->is_If()) return NULL; | |
| 1839 Node *p_f = iffm->in(0); | |
| 1840 if (!p_f->is_IfFalse()) return NULL; | |
| 1841 if (!p_f->in(0)->is_CountedLoopEnd()) return NULL; | |
| 1842 CountedLoopEndNode *pre_end = p_f->in(0)->as_CountedLoopEnd(); | |
| 1843 if (!pre_end->loopnode()->is_pre_loop()) return NULL; | |
| 1844 return pre_end; | |
| 1845 } | |
| 1846 | |
| 1847 | |
| 1848 //------------------------------init--------------------------- | |
| 1849 void SuperWord::init() { | |
| 1850 _dg.init(); | |
| 1851 _packset.clear(); | |
| 1852 _disjoint_ptrs.clear(); | |
| 1853 _block.clear(); | |
| 1854 _data_entry.clear(); | |
| 1855 _mem_slice_head.clear(); | |
| 1856 _mem_slice_tail.clear(); | |
| 1857 _node_info.clear(); | |
| 1858 _align_to_ref = NULL; | |
| 1859 _lpt = NULL; | |
| 1860 _lp = NULL; | |
| 1861 _bb = NULL; | |
| 1862 _iv = NULL; | |
| 1863 } | |
| 1864 | |
| 1865 //------------------------------print_packset--------------------------- | |
| 1866 void SuperWord::print_packset() { | |
| 1867 #ifndef PRODUCT | |
| 1868 tty->print_cr("packset"); | |
| 1869 for (int i = 0; i < _packset.length(); i++) { | |
| 1870 tty->print_cr("Pack: %d", i); | |
| 1871 Node_List* p = _packset.at(i); | |
| 1872 print_pack(p); | |
| 1873 } | |
| 1874 #endif | |
| 1875 } | |
| 1876 | |
| 1877 //------------------------------print_pack--------------------------- | |
| 1878 void SuperWord::print_pack(Node_List* p) { | |
| 1879 for (uint i = 0; i < p->size(); i++) { | |
| 1880 print_stmt(p->at(i)); | |
| 1881 } | |
| 1882 } | |
| 1883 | |
| 1884 //------------------------------print_bb--------------------------- | |
| 1885 void SuperWord::print_bb() { | |
| 1886 #ifndef PRODUCT | |
| 1887 tty->print_cr("\nBlock"); | |
| 1888 for (int i = 0; i < _block.length(); i++) { | |
| 1889 Node* n = _block.at(i); | |
| 1890 tty->print("%d ", i); | |
| 1891 if (n) { | |
| 1892 n->dump(); | |
| 1893 } | |
| 1894 } | |
| 1895 #endif | |
| 1896 } | |
| 1897 | |
| 1898 //------------------------------print_stmt--------------------------- | |
| 1899 void SuperWord::print_stmt(Node* s) { | |
| 1900 #ifndef PRODUCT | |
| 1901 tty->print(" align: %d \t", alignment(s)); | |
| 1902 s->dump(); | |
| 1903 #endif | |
| 1904 } | |
| 1905 | |
| 1906 //------------------------------blank--------------------------- | |
| 1907 char* SuperWord::blank(uint depth) { | |
| 1908 static char blanks[101]; | |
| 1909 assert(depth < 101, "too deep"); | |
| 1910 for (uint i = 0; i < depth; i++) blanks[i] = ' '; | |
| 1911 blanks[depth] = '\0'; | |
| 1912 return blanks; | |
| 1913 } | |
| 1914 | |
| 1915 | |
| 1916 //==============================SWPointer=========================== | |
| 1917 | |
| 1918 //----------------------------SWPointer------------------------ | |
| 1919 SWPointer::SWPointer(MemNode* mem, SuperWord* slp) : | |
| 1920 _mem(mem), _slp(slp), _base(NULL), _adr(NULL), | |
| 1921 _scale(0), _offset(0), _invar(NULL), _negate_invar(false) { | |
| 1922 | |
| 1923 Node* adr = mem->in(MemNode::Address); | |
| 1924 if (!adr->is_AddP()) { | |
| 1925 assert(!valid(), "too complex"); | |
| 1926 return; | |
| 1927 } | |
| 1928 // Match AddP(base, AddP(ptr, k*iv [+ invariant]), constant) | |
| 1929 Node* base = adr->in(AddPNode::Base); | |
|
1058
73a726751507
6852078: HSX 14/16 in jdk 5.0: api/javax_management api/org_omg jck tests crashes or make tnameserv crash
cfang
parents:
987
diff
changeset
|
1930 //unsafe reference could not be aligned appropriately without runtime checking |
|
73a726751507
6852078: HSX 14/16 in jdk 5.0: api/javax_management api/org_omg jck tests crashes or make tnameserv crash
cfang
parents:
987
diff
changeset
|
1931 if (base == NULL || base->bottom_type() == Type::TOP) { |
|
73a726751507
6852078: HSX 14/16 in jdk 5.0: api/javax_management api/org_omg jck tests crashes or make tnameserv crash
cfang
parents:
987
diff
changeset
|
1932 assert(!valid(), "unsafe access"); |
|
73a726751507
6852078: HSX 14/16 in jdk 5.0: api/javax_management api/org_omg jck tests crashes or make tnameserv crash
cfang
parents:
987
diff
changeset
|
1933 return; |
|
73a726751507
6852078: HSX 14/16 in jdk 5.0: api/javax_management api/org_omg jck tests crashes or make tnameserv crash
cfang
parents:
987
diff
changeset
|
1934 } |
| 0 | 1935 for (int i = 0; i < 3; i++) { |
| 1936 if (!scaled_iv_plus_offset(adr->in(AddPNode::Offset))) { | |
| 1937 assert(!valid(), "too complex"); | |
| 1938 return; | |
| 1939 } | |
| 1940 adr = adr->in(AddPNode::Address); | |
| 1941 if (base == adr || !adr->is_AddP()) { | |
| 1942 break; // stop looking at addp's | |
| 1943 } | |
| 1944 } | |
| 1945 _base = base; | |
| 1946 _adr = adr; | |
| 1947 assert(valid(), "Usable"); | |
| 1948 } | |
| 1949 | |
| 1950 // Following is used to create a temporary object during | |
| 1951 // the pattern match of an address expression. | |
| 1952 SWPointer::SWPointer(SWPointer* p) : | |
| 1953 _mem(p->_mem), _slp(p->_slp), _base(NULL), _adr(NULL), | |
| 1954 _scale(0), _offset(0), _invar(NULL), _negate_invar(false) {} | |
| 1955 | |
| 1956 //------------------------scaled_iv_plus_offset-------------------- | |
| 1957 // Match: k*iv + offset | |
| 1958 // where: k is a constant that maybe zero, and | |
| 1959 // offset is (k2 [+/- invariant]) where k2 maybe zero and invariant is optional | |
| 1960 bool SWPointer::scaled_iv_plus_offset(Node* n) { | |
| 1961 if (scaled_iv(n)) { | |
| 1962 return true; | |
| 1963 } | |
| 1964 if (offset_plus_k(n)) { | |
| 1965 return true; | |
| 1966 } | |
| 1967 int opc = n->Opcode(); | |
| 1968 if (opc == Op_AddI) { | |
| 1969 if (scaled_iv(n->in(1)) && offset_plus_k(n->in(2))) { | |
| 1970 return true; | |
| 1971 } | |
| 1972 if (scaled_iv(n->in(2)) && offset_plus_k(n->in(1))) { | |
| 1973 return true; | |
| 1974 } | |
| 1975 } else if (opc == Op_SubI) { | |
| 1976 if (scaled_iv(n->in(1)) && offset_plus_k(n->in(2), true)) { | |
| 1977 return true; | |
| 1978 } | |
| 1979 if (scaled_iv(n->in(2)) && offset_plus_k(n->in(1))) { | |
| 1980 _scale *= -1; | |
| 1981 return true; | |
| 1982 } | |
| 1983 } | |
| 1984 return false; | |
| 1985 } | |
| 1986 | |
| 1987 //----------------------------scaled_iv------------------------ | |
| 1988 // Match: k*iv where k is a constant that's not zero | |
| 1989 bool SWPointer::scaled_iv(Node* n) { | |
| 1990 if (_scale != 0) { | |
| 1991 return false; // already found a scale | |
| 1992 } | |
| 1993 if (n == iv()) { | |
| 1994 _scale = 1; | |
| 1995 return true; | |
| 1996 } | |
| 1997 int opc = n->Opcode(); | |
| 1998 if (opc == Op_MulI) { | |
| 1999 if (n->in(1) == iv() && n->in(2)->is_Con()) { | |
| 2000 _scale = n->in(2)->get_int(); | |
| 2001 return true; | |
| 2002 } else if (n->in(2) == iv() && n->in(1)->is_Con()) { | |
| 2003 _scale = n->in(1)->get_int(); | |
| 2004 return true; | |
| 2005 } | |
| 2006 } else if (opc == Op_LShiftI) { | |
| 2007 if (n->in(1) == iv() && n->in(2)->is_Con()) { | |
| 2008 _scale = 1 << n->in(2)->get_int(); | |
| 2009 return true; | |
| 2010 } | |
| 2011 } else if (opc == Op_ConvI2L) { | |
| 2012 if (scaled_iv_plus_offset(n->in(1))) { | |
| 2013 return true; | |
| 2014 } | |
| 2015 } else if (opc == Op_LShiftL) { | |
| 2016 if (!has_iv() && _invar == NULL) { | |
| 2017 // Need to preserve the current _offset value, so | |
| 2018 // create a temporary object for this expression subtree. | |
| 2019 // Hacky, so should re-engineer the address pattern match. | |
| 2020 SWPointer tmp(this); | |
| 2021 if (tmp.scaled_iv_plus_offset(n->in(1))) { | |
| 2022 if (tmp._invar == NULL) { | |
| 2023 int mult = 1 << n->in(2)->get_int(); | |
| 2024 _scale = tmp._scale * mult; | |
| 2025 _offset += tmp._offset * mult; | |
| 2026 return true; | |
| 2027 } | |
| 2028 } | |
| 2029 } | |
| 2030 } | |
| 2031 return false; | |
| 2032 } | |
| 2033 | |
| 2034 //----------------------------offset_plus_k------------------------ | |
| 2035 // Match: offset is (k [+/- invariant]) | |
| 2036 // where k maybe zero and invariant is optional, but not both. | |
| 2037 bool SWPointer::offset_plus_k(Node* n, bool negate) { | |
| 2038 int opc = n->Opcode(); | |
| 2039 if (opc == Op_ConI) { | |
| 2040 _offset += negate ? -(n->get_int()) : n->get_int(); | |
| 2041 return true; | |
| 2042 } else if (opc == Op_ConL) { | |
| 2043 // Okay if value fits into an int | |
| 2044 const TypeLong* t = n->find_long_type(); | |
| 2045 if (t->higher_equal(TypeLong::INT)) { | |
| 2046 jlong loff = n->get_long(); | |
| 2047 jint off = (jint)loff; | |
| 2048 _offset += negate ? -off : loff; | |
| 2049 return true; | |
| 2050 } | |
| 2051 return false; | |
| 2052 } | |
| 2053 if (_invar != NULL) return false; // already have an invariant | |
| 2054 if (opc == Op_AddI) { | |
| 2055 if (n->in(2)->is_Con() && invariant(n->in(1))) { | |
| 2056 _negate_invar = negate; | |
| 2057 _invar = n->in(1); | |
| 2058 _offset += negate ? -(n->in(2)->get_int()) : n->in(2)->get_int(); | |
| 2059 return true; | |
| 2060 } else if (n->in(1)->is_Con() && invariant(n->in(2))) { | |
| 2061 _offset += negate ? -(n->in(1)->get_int()) : n->in(1)->get_int(); | |
| 2062 _negate_invar = negate; | |
| 2063 _invar = n->in(2); | |
| 2064 return true; | |
| 2065 } | |
| 2066 } | |
| 2067 if (opc == Op_SubI) { | |
| 2068 if (n->in(2)->is_Con() && invariant(n->in(1))) { | |
| 2069 _negate_invar = negate; | |
| 2070 _invar = n->in(1); | |
| 2071 _offset += !negate ? -(n->in(2)->get_int()) : n->in(2)->get_int(); | |
| 2072 return true; | |
| 2073 } else if (n->in(1)->is_Con() && invariant(n->in(2))) { | |
| 2074 _offset += negate ? -(n->in(1)->get_int()) : n->in(1)->get_int(); | |
| 2075 _negate_invar = !negate; | |
| 2076 _invar = n->in(2); | |
| 2077 return true; | |
| 2078 } | |
| 2079 } | |
| 2080 if (invariant(n)) { | |
| 2081 _negate_invar = negate; | |
| 2082 _invar = n; | |
| 2083 return true; | |
| 2084 } | |
| 2085 return false; | |
| 2086 } | |
| 2087 | |
| 2088 //----------------------------print------------------------ | |
| 2089 void SWPointer::print() { | |
| 2090 #ifndef PRODUCT | |
| 2091 tty->print("base: %d adr: %d scale: %d offset: %d invar: %c%d\n", | |
| 2092 _base != NULL ? _base->_idx : 0, | |
| 2093 _adr != NULL ? _adr->_idx : 0, | |
| 2094 _scale, _offset, | |
| 2095 _negate_invar?'-':'+', | |
| 2096 _invar != NULL ? _invar->_idx : 0); | |
| 2097 #endif | |
| 2098 } | |
| 2099 | |
| 2100 // ========================= OrderedPair ===================== | |
| 2101 | |
| 2102 const OrderedPair OrderedPair::initial; | |
| 2103 | |
| 2104 // ========================= SWNodeInfo ===================== | |
| 2105 | |
| 2106 const SWNodeInfo SWNodeInfo::initial; | |
| 2107 | |
| 2108 | |
| 2109 // ============================ DepGraph =========================== | |
| 2110 | |
| 2111 //------------------------------make_node--------------------------- | |
| 2112 // Make a new dependence graph node for an ideal node. | |
| 2113 DepMem* DepGraph::make_node(Node* node) { | |
| 2114 DepMem* m = new (_arena) DepMem(node); | |
| 2115 if (node != NULL) { | |
| 2116 assert(_map.at_grow(node->_idx) == NULL, "one init only"); | |
| 2117 _map.at_put_grow(node->_idx, m); | |
| 2118 } | |
| 2119 return m; | |
| 2120 } | |
| 2121 | |
| 2122 //------------------------------make_edge--------------------------- | |
| 2123 // Make a new dependence graph edge from dpred -> dsucc | |
| 2124 DepEdge* DepGraph::make_edge(DepMem* dpred, DepMem* dsucc) { | |
| 2125 DepEdge* e = new (_arena) DepEdge(dpred, dsucc, dsucc->in_head(), dpred->out_head()); | |
| 2126 dpred->set_out_head(e); | |
| 2127 dsucc->set_in_head(e); | |
| 2128 return e; | |
| 2129 } | |
| 2130 | |
| 2131 // ========================== DepMem ======================== | |
| 2132 | |
| 2133 //------------------------------in_cnt--------------------------- | |
| 2134 int DepMem::in_cnt() { | |
| 2135 int ct = 0; | |
| 2136 for (DepEdge* e = _in_head; e != NULL; e = e->next_in()) ct++; | |
| 2137 return ct; | |
| 2138 } | |
| 2139 | |
| 2140 //------------------------------out_cnt--------------------------- | |
| 2141 int DepMem::out_cnt() { | |
| 2142 int ct = 0; | |
| 2143 for (DepEdge* e = _out_head; e != NULL; e = e->next_out()) ct++; | |
| 2144 return ct; | |
| 2145 } | |
| 2146 | |
| 2147 //------------------------------print----------------------------- | |
| 2148 void DepMem::print() { | |
| 2149 #ifndef PRODUCT | |
| 2150 tty->print(" DepNode %d (", _node->_idx); | |
| 2151 for (DepEdge* p = _in_head; p != NULL; p = p->next_in()) { | |
| 2152 Node* pred = p->pred()->node(); | |
| 2153 tty->print(" %d", pred != NULL ? pred->_idx : 0); | |
| 2154 } | |
| 2155 tty->print(") ["); | |
| 2156 for (DepEdge* s = _out_head; s != NULL; s = s->next_out()) { | |
| 2157 Node* succ = s->succ()->node(); | |
| 2158 tty->print(" %d", succ != NULL ? succ->_idx : 0); | |
| 2159 } | |
| 2160 tty->print_cr(" ]"); | |
| 2161 #endif | |
| 2162 } | |
| 2163 | |
| 2164 // =========================== DepEdge ========================= | |
| 2165 | |
| 2166 //------------------------------DepPreds--------------------------- | |
| 2167 void DepEdge::print() { | |
| 2168 #ifndef PRODUCT | |
| 2169 tty->print_cr("DepEdge: %d [ %d ]", _pred->node()->_idx, _succ->node()->_idx); | |
| 2170 #endif | |
| 2171 } | |
| 2172 | |
| 2173 // =========================== DepPreds ========================= | |
| 2174 // Iterator over predecessor edges in the dependence graph. | |
| 2175 | |
| 2176 //------------------------------DepPreds--------------------------- | |
| 2177 DepPreds::DepPreds(Node* n, DepGraph& dg) { | |
| 2178 _n = n; | |
| 2179 _done = false; | |
| 2180 if (_n->is_Store() || _n->is_Load()) { | |
| 2181 _next_idx = MemNode::Address; | |
| 2182 _end_idx = n->req(); | |
| 2183 _dep_next = dg.dep(_n)->in_head(); | |
| 2184 } else if (_n->is_Mem()) { | |
| 2185 _next_idx = 0; | |
| 2186 _end_idx = 0; | |
| 2187 _dep_next = dg.dep(_n)->in_head(); | |
| 2188 } else { | |
| 2189 _next_idx = 1; | |
| 2190 _end_idx = _n->req(); | |
| 2191 _dep_next = NULL; | |
| 2192 } | |
| 2193 next(); | |
| 2194 } | |
| 2195 | |
| 2196 //------------------------------next--------------------------- | |
| 2197 void DepPreds::next() { | |
| 2198 if (_dep_next != NULL) { | |
| 2199 _current = _dep_next->pred()->node(); | |
| 2200 _dep_next = _dep_next->next_in(); | |
| 2201 } else if (_next_idx < _end_idx) { | |
| 2202 _current = _n->in(_next_idx++); | |
| 2203 } else { | |
| 2204 _done = true; | |
| 2205 } | |
| 2206 } | |
| 2207 | |
| 2208 // =========================== DepSuccs ========================= | |
| 2209 // Iterator over successor edges in the dependence graph. | |
| 2210 | |
| 2211 //------------------------------DepSuccs--------------------------- | |
| 2212 DepSuccs::DepSuccs(Node* n, DepGraph& dg) { | |
| 2213 _n = n; | |
| 2214 _done = false; | |
| 2215 if (_n->is_Load()) { | |
| 2216 _next_idx = 0; | |
| 2217 _end_idx = _n->outcnt(); | |
| 2218 _dep_next = dg.dep(_n)->out_head(); | |
| 2219 } else if (_n->is_Mem() || _n->is_Phi() && _n->bottom_type() == Type::MEMORY) { | |
| 2220 _next_idx = 0; | |
| 2221 _end_idx = 0; | |
| 2222 _dep_next = dg.dep(_n)->out_head(); | |
| 2223 } else { | |
| 2224 _next_idx = 0; | |
| 2225 _end_idx = _n->outcnt(); | |
| 2226 _dep_next = NULL; | |
| 2227 } | |
| 2228 next(); | |
| 2229 } | |
| 2230 | |
| 2231 //-------------------------------next--------------------------- | |
| 2232 void DepSuccs::next() { | |
| 2233 if (_dep_next != NULL) { | |
| 2234 _current = _dep_next->succ()->node(); | |
| 2235 _dep_next = _dep_next->next_out(); | |
| 2236 } else if (_next_idx < _end_idx) { | |
| 2237 _current = _n->raw_out(_next_idx++); | |
| 2238 } else { | |
| 2239 _done = true; | |
| 2240 } | |
| 2241 } |