Mercurial > hg > truffle
annotate src/share/vm/opto/superword.cpp @ 1994:6cd6d394f280
7001033: assert(gch->gc_cause() == GCCause::_scavenge_alot || !gch->incremental_collection_failed())
7002546: regression on SpecJbb2005 on 7b118 comparing to 7b117 on small heaps
Summary: Relaxed assertion checking related to incremental_collection_failed flag to allow for ExplicitGCInvokesConcurrent behaviour where we do not want a failing scavenge to bail to a stop-world collection. Parameterized incremental_collection_will_fail() so we can selectively use, or not use, as appropriate, the statistical prediction at specific use sites. This essentially reverts the scavenge bail-out logic to what it was prior to some recent changes that had inadvertently started using the statistical prediction which can be noisy in the presence of bursty loads. Added some associated verbose non-product debugging messages.
Reviewed-by: johnc, tonyp
| author | ysr |
|---|---|
| date | Tue, 07 Dec 2010 21:55:53 -0800 |
| parents | f95d63e2154a |
| children | 08eb13460b3a |
| rev | line source |
|---|---|
| 0 | 1 /* |
| 1585 | 2 * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved. |
| 0 | 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| 4 * | |
| 5 * This code is free software; you can redistribute it and/or modify it | |
| 6 * under the terms of the GNU General Public License version 2 only, as | |
| 7 * published by the Free Software Foundation. | |
| 8 * | |
| 9 * This code is distributed in the hope that it will be useful, but WITHOUT | |
| 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
| 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
| 12 * version 2 for more details (a copy is included in the LICENSE file that | |
| 13 * accompanied this code). | |
| 14 * | |
| 15 * You should have received a copy of the GNU General Public License version | |
| 16 * 2 along with this work; if not, write to the Free Software Foundation, | |
| 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |
| 18 * | |
|
1552
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1058
diff
changeset
|
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
|
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1058
diff
changeset
|
20 * or visit www.oracle.com if you need additional information or have any |
|
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1058
diff
changeset
|
21 * questions. |
| 0 | 22 */ |
| 23 | |
| 1972 | 24 #include "precompiled.hpp" |
| 25 #include "compiler/compileLog.hpp" | |
| 26 #include "libadt/vectset.hpp" | |
| 27 #include "memory/allocation.inline.hpp" | |
| 28 #include "opto/addnode.hpp" | |
| 29 #include "opto/callnode.hpp" | |
| 30 #include "opto/divnode.hpp" | |
| 31 #include "opto/matcher.hpp" | |
| 32 #include "opto/memnode.hpp" | |
| 33 #include "opto/mulnode.hpp" | |
| 34 #include "opto/opcodes.hpp" | |
| 35 #include "opto/superword.hpp" | |
| 36 #include "opto/vectornode.hpp" | |
| 0 | 37 |
| 38 // | |
| 39 // S U P E R W O R D T R A N S F O R M | |
| 40 //============================================================================= | |
| 41 | |
| 42 //------------------------------SuperWord--------------------------- | |
| 43 SuperWord::SuperWord(PhaseIdealLoop* phase) : | |
| 44 _phase(phase), | |
| 45 _igvn(phase->_igvn), | |
| 46 _arena(phase->C->comp_arena()), | |
| 47 _packset(arena(), 8, 0, NULL), // packs for the current block | |
| 48 _bb_idx(arena(), (int)(1.10 * phase->C->unique()), 0, 0), // node idx to index in bb | |
| 49 _block(arena(), 8, 0, NULL), // nodes in current block | |
| 50 _data_entry(arena(), 8, 0, NULL), // nodes with all inputs from outside | |
| 51 _mem_slice_head(arena(), 8, 0, NULL), // memory slice heads | |
| 52 _mem_slice_tail(arena(), 8, 0, NULL), // memory slice tails | |
| 53 _node_info(arena(), 8, 0, SWNodeInfo::initial), // info needed per node | |
| 54 _align_to_ref(NULL), // memory reference to align vectors to | |
| 55 _disjoint_ptrs(arena(), 8, 0, OrderedPair::initial), // runtime disambiguated pointer pairs | |
| 56 _dg(_arena), // dependence graph | |
| 57 _visited(arena()), // visited node set | |
| 58 _post_visited(arena()), // post visited node set | |
| 59 _n_idx_list(arena(), 8), // scratch list of (node,index) pairs | |
| 60 _stk(arena(), 8, 0, NULL), // scratch stack of nodes | |
| 61 _nlist(arena(), 8, 0, NULL), // scratch list of nodes | |
| 62 _lpt(NULL), // loop tree node | |
| 63 _lp(NULL), // LoopNode | |
| 64 _bb(NULL), // basic block | |
| 65 _iv(NULL) // induction var | |
| 66 {} | |
| 67 | |
| 68 //------------------------------transform_loop--------------------------- | |
| 69 void SuperWord::transform_loop(IdealLoopTree* lpt) { | |
| 70 assert(lpt->_head->is_CountedLoop(), "must be"); | |
| 71 CountedLoopNode *cl = lpt->_head->as_CountedLoop(); | |
| 72 | |
| 73 if (!cl->is_main_loop() ) return; // skip normal, pre, and post loops | |
| 74 | |
| 75 // Check for no control flow in body (other than exit) | |
| 76 Node *cl_exit = cl->loopexit(); | |
| 77 if (cl_exit->in(0) != lpt->_head) return; | |
| 78 | |
|
105
a7d0f95410bd
6646020: assert(in_bb(n),"must be in block") in -Xcomp mode
never
parents:
72
diff
changeset
|
79 // Make sure the are no extra control users of the loop backedge |
|
a7d0f95410bd
6646020: assert(in_bb(n),"must be in block") in -Xcomp mode
never
parents:
72
diff
changeset
|
80 if (cl->back_control()->outcnt() != 1) { |
|
a7d0f95410bd
6646020: assert(in_bb(n),"must be in block") in -Xcomp mode
never
parents:
72
diff
changeset
|
81 return; |
|
a7d0f95410bd
6646020: assert(in_bb(n),"must be in block") in -Xcomp mode
never
parents:
72
diff
changeset
|
82 } |
|
a7d0f95410bd
6646020: assert(in_bb(n),"must be in block") in -Xcomp mode
never
parents:
72
diff
changeset
|
83 |
| 0 | 84 // Check for pre-loop ending with CountedLoopEnd(Bool(Cmp(x,Opaque1(limit)))) |
| 85 CountedLoopEndNode* pre_end = get_pre_loop_end(cl); | |
| 86 if (pre_end == NULL) return; | |
| 87 Node *pre_opaq1 = pre_end->limit(); | |
| 88 if (pre_opaq1->Opcode() != Op_Opaque1) return; | |
| 89 | |
| 90 // Do vectors exist on this architecture? | |
| 91 if (vector_width_in_bytes() == 0) return; | |
| 92 | |
| 93 init(); // initialize data structures | |
| 94 | |
| 95 set_lpt(lpt); | |
| 96 set_lp(cl); | |
| 97 | |
| 98 // For now, define one block which is the entire loop body | |
| 99 set_bb(cl); | |
| 100 | |
| 101 assert(_packset.length() == 0, "packset must be empty"); | |
| 102 SLP_extract(); | |
| 103 } | |
| 104 | |
| 105 //------------------------------SLP_extract--------------------------- | |
| 106 // Extract the superword level parallelism | |
| 107 // | |
| 108 // 1) A reverse post-order of nodes in the block is constructed. By scanning | |
| 109 // this list from first to last, all definitions are visited before their uses. | |
| 110 // | |
| 111 // 2) A point-to-point dependence graph is constructed between memory references. | |
| 112 // This simplies the upcoming "independence" checker. | |
| 113 // | |
| 114 // 3) The maximum depth in the node graph from the beginning of the block | |
| 115 // to each node is computed. This is used to prune the graph search | |
| 116 // in the independence checker. | |
| 117 // | |
| 118 // 4) For integer types, the necessary bit width is propagated backwards | |
| 119 // from stores to allow packed operations on byte, char, and short | |
| 120 // integers. This reverses the promotion to type "int" that javac | |
| 121 // did for operations like: char c1,c2,c3; c1 = c2 + c3. | |
| 122 // | |
| 123 // 5) One of the memory references is picked to be an aligned vector reference. | |
| 124 // The pre-loop trip count is adjusted to align this reference in the | |
| 125 // unrolled body. | |
| 126 // | |
| 127 // 6) The initial set of pack pairs is seeded with memory references. | |
| 128 // | |
| 129 // 7) The set of pack pairs is extended by following use->def and def->use links. | |
| 130 // | |
| 131 // 8) The pairs are combined into vector sized packs. | |
| 132 // | |
| 133 // 9) Reorder the memory slices to co-locate members of the memory packs. | |
| 134 // | |
| 135 // 10) Generate ideal vector nodes for the final set of packs and where necessary, | |
| 136 // inserting scalar promotion, vector creation from multiple scalars, and | |
| 137 // extraction of scalar values from vectors. | |
| 138 // | |
| 139 void SuperWord::SLP_extract() { | |
| 140 | |
| 141 // Ready the block | |
| 142 | |
| 143 construct_bb(); | |
| 144 | |
| 145 dependence_graph(); | |
| 146 | |
| 147 compute_max_depth(); | |
| 148 | |
| 149 compute_vector_element_type(); | |
| 150 | |
| 151 // Attempt vectorization | |
| 152 | |
| 153 find_adjacent_refs(); | |
| 154 | |
| 155 extend_packlist(); | |
| 156 | |
| 157 combine_packs(); | |
| 158 | |
| 159 construct_my_pack_map(); | |
| 160 | |
| 161 filter_packs(); | |
| 162 | |
| 163 schedule(); | |
| 164 | |
| 165 output(); | |
| 166 } | |
| 167 | |
| 168 //------------------------------find_adjacent_refs--------------------------- | |
| 169 // Find the adjacent memory references and create pack pairs for them. | |
| 170 // This is the initial set of packs that will then be extended by | |
| 171 // following use->def and def->use links. The align positions are | |
| 172 // assigned relative to the reference "align_to_ref" | |
| 173 void SuperWord::find_adjacent_refs() { | |
| 174 // Get list of memory operations | |
| 175 Node_List memops; | |
| 176 for (int i = 0; i < _block.length(); i++) { | |
| 177 Node* n = _block.at(i); | |
|
29
d5fc211aea19
6633953: type2aelembytes{T_ADDRESS} should be 8 bytes in 64 bit VM
kvn
parents:
0
diff
changeset
|
178 if (n->is_Mem() && in_bb(n) && |
|
d5fc211aea19
6633953: type2aelembytes{T_ADDRESS} should be 8 bytes in 64 bit VM
kvn
parents:
0
diff
changeset
|
179 is_java_primitive(n->as_Mem()->memory_type())) { |
| 0 | 180 int align = memory_alignment(n->as_Mem(), 0); |
| 181 if (align != bottom_align) { | |
| 182 memops.push(n); | |
| 183 } | |
| 184 } | |
| 185 } | |
| 186 if (memops.size() == 0) return; | |
| 187 | |
| 188 // Find a memory reference to align to. The pre-loop trip count | |
| 189 // is modified to align this reference to a vector-aligned address | |
| 190 find_align_to_ref(memops); | |
| 191 if (align_to_ref() == NULL) return; | |
| 192 | |
| 193 SWPointer align_to_ref_p(align_to_ref(), this); | |
| 194 int offset = align_to_ref_p.offset_in_bytes(); | |
| 195 int scale = align_to_ref_p.scale_in_bytes(); | |
| 196 int vw = vector_width_in_bytes(); | |
| 197 int stride_sign = (scale * iv_stride()) > 0 ? 1 : -1; | |
| 198 int iv_adjustment = (stride_sign * vw - (offset % vw)) % vw; | |
| 199 | |
| 200 #ifndef PRODUCT | |
| 201 if (TraceSuperWord) | |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
202 tty->print_cr("\noffset = %d iv_adjustment = %d elt_align = %d scale = %d iv_stride = %d", |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
203 offset, iv_adjustment, align_to_ref_p.memory_size(), align_to_ref_p.scale_in_bytes(), iv_stride()); |
| 0 | 204 #endif |
| 205 | |
| 206 // Set alignment relative to "align_to_ref" | |
| 207 for (int i = memops.size() - 1; i >= 0; i--) { | |
| 208 MemNode* s = memops.at(i)->as_Mem(); | |
| 209 SWPointer p2(s, this); | |
| 210 if (p2.comparable(align_to_ref_p)) { | |
| 211 int align = memory_alignment(s, iv_adjustment); | |
| 212 set_alignment(s, align); | |
| 213 } else { | |
| 214 memops.remove(i); | |
| 215 } | |
| 216 } | |
| 217 | |
| 218 // Create initial pack pairs of memory operations | |
| 219 for (uint i = 0; i < memops.size(); i++) { | |
| 220 Node* s1 = memops.at(i); | |
| 221 for (uint j = 0; j < memops.size(); j++) { | |
| 222 Node* s2 = memops.at(j); | |
| 223 if (s1 != s2 && are_adjacent_refs(s1, s2)) { | |
| 224 int align = alignment(s1); | |
| 225 if (stmts_can_pack(s1, s2, align)) { | |
| 226 Node_List* pair = new Node_List(); | |
| 227 pair->push(s1); | |
| 228 pair->push(s2); | |
| 229 _packset.append(pair); | |
| 230 } | |
| 231 } | |
| 232 } | |
| 233 } | |
| 234 | |
| 235 #ifndef PRODUCT | |
| 236 if (TraceSuperWord) { | |
| 237 tty->print_cr("\nAfter find_adjacent_refs"); | |
| 238 print_packset(); | |
| 239 } | |
| 240 #endif | |
| 241 } | |
| 242 | |
| 243 //------------------------------find_align_to_ref--------------------------- | |
| 244 // Find a memory reference to align the loop induction variable to. | |
| 245 // Looks first at stores then at loads, looking for a memory reference | |
| 246 // with the largest number of references similar to it. | |
| 247 void SuperWord::find_align_to_ref(Node_List &memops) { | |
| 248 GrowableArray<int> cmp_ct(arena(), memops.size(), memops.size(), 0); | |
| 249 | |
| 250 // Count number of comparable memory ops | |
| 251 for (uint i = 0; i < memops.size(); i++) { | |
| 252 MemNode* s1 = memops.at(i)->as_Mem(); | |
| 253 SWPointer p1(s1, this); | |
| 254 // Discard if pre loop can't align this reference | |
| 255 if (!ref_is_alignable(p1)) { | |
| 256 *cmp_ct.adr_at(i) = 0; | |
| 257 continue; | |
| 258 } | |
| 259 for (uint j = i+1; j < memops.size(); j++) { | |
| 260 MemNode* s2 = memops.at(j)->as_Mem(); | |
| 261 if (isomorphic(s1, s2)) { | |
| 262 SWPointer p2(s2, this); | |
| 263 if (p1.comparable(p2)) { | |
| 264 (*cmp_ct.adr_at(i))++; | |
| 265 (*cmp_ct.adr_at(j))++; | |
| 266 } | |
| 267 } | |
| 268 } | |
| 269 } | |
| 270 | |
| 271 // Find Store (or Load) with the greatest number of "comparable" references | |
| 272 int max_ct = 0; | |
| 273 int max_idx = -1; | |
| 274 int min_size = max_jint; | |
| 275 int min_iv_offset = max_jint; | |
| 276 for (uint j = 0; j < memops.size(); j++) { | |
| 277 MemNode* s = memops.at(j)->as_Mem(); | |
| 278 if (s->is_Store()) { | |
| 279 SWPointer p(s, this); | |
| 280 if (cmp_ct.at(j) > max_ct || | |
| 281 cmp_ct.at(j) == max_ct && (data_size(s) < min_size || | |
| 282 data_size(s) == min_size && | |
| 283 p.offset_in_bytes() < min_iv_offset)) { | |
| 284 max_ct = cmp_ct.at(j); | |
| 285 max_idx = j; | |
| 286 min_size = data_size(s); | |
| 287 min_iv_offset = p.offset_in_bytes(); | |
| 288 } | |
| 289 } | |
| 290 } | |
| 291 // If no stores, look at loads | |
| 292 if (max_ct == 0) { | |
| 293 for (uint j = 0; j < memops.size(); j++) { | |
| 294 MemNode* s = memops.at(j)->as_Mem(); | |
| 295 if (s->is_Load()) { | |
| 296 SWPointer p(s, this); | |
| 297 if (cmp_ct.at(j) > max_ct || | |
| 298 cmp_ct.at(j) == max_ct && (data_size(s) < min_size || | |
| 299 data_size(s) == min_size && | |
| 300 p.offset_in_bytes() < min_iv_offset)) { | |
| 301 max_ct = cmp_ct.at(j); | |
| 302 max_idx = j; | |
| 303 min_size = data_size(s); | |
| 304 min_iv_offset = p.offset_in_bytes(); | |
| 305 } | |
| 306 } | |
| 307 } | |
| 308 } | |
| 309 | |
| 310 if (max_ct > 0) | |
| 311 set_align_to_ref(memops.at(max_idx)->as_Mem()); | |
| 312 | |
| 313 #ifndef PRODUCT | |
| 314 if (TraceSuperWord && Verbose) { | |
| 315 tty->print_cr("\nVector memops after find_align_to_refs"); | |
| 316 for (uint i = 0; i < memops.size(); i++) { | |
| 317 MemNode* s = memops.at(i)->as_Mem(); | |
| 318 s->dump(); | |
| 319 } | |
| 320 } | |
| 321 #endif | |
| 322 } | |
| 323 | |
| 324 //------------------------------ref_is_alignable--------------------------- | |
| 325 // Can the preloop align the reference to position zero in the vector? | |
| 326 bool SuperWord::ref_is_alignable(SWPointer& p) { | |
| 327 if (!p.has_iv()) { | |
| 328 return true; // no induction variable | |
| 329 } | |
| 330 CountedLoopEndNode* pre_end = get_pre_loop_end(lp()->as_CountedLoop()); | |
| 331 assert(pre_end->stride_is_con(), "pre loop stride is constant"); | |
| 332 int preloop_stride = pre_end->stride_con(); | |
| 333 | |
| 334 int span = preloop_stride * p.scale_in_bytes(); | |
| 335 | |
| 336 // Stride one accesses are alignable. | |
| 337 if (ABS(span) == p.memory_size()) | |
| 338 return true; | |
| 339 | |
| 340 // If initial offset from start of object is computable, | |
| 341 // compute alignment within the vector. | |
| 342 int vw = vector_width_in_bytes(); | |
| 343 if (vw % span == 0) { | |
| 344 Node* init_nd = pre_end->init_trip(); | |
| 345 if (init_nd->is_Con() && p.invar() == NULL) { | |
| 346 int init = init_nd->bottom_type()->is_int()->get_con(); | |
| 347 | |
| 348 int init_offset = init * p.scale_in_bytes() + p.offset_in_bytes(); | |
| 349 assert(init_offset >= 0, "positive offset from object start"); | |
| 350 | |
| 351 if (span > 0) { | |
| 352 return (vw - (init_offset % vw)) % span == 0; | |
| 353 } else { | |
| 354 assert(span < 0, "nonzero stride * scale"); | |
| 355 return (init_offset % vw) % -span == 0; | |
| 356 } | |
| 357 } | |
| 358 } | |
| 359 return false; | |
| 360 } | |
| 361 | |
| 362 //---------------------------dependence_graph--------------------------- | |
| 363 // Construct dependency graph. | |
| 364 // Add dependence edges to load/store nodes for memory dependence | |
| 365 // A.out()->DependNode.in(1) and DependNode.out()->B.prec(x) | |
| 366 void SuperWord::dependence_graph() { | |
| 367 // First, assign a dependence node to each memory node | |
| 368 for (int i = 0; i < _block.length(); i++ ) { | |
| 369 Node *n = _block.at(i); | |
| 370 if (n->is_Mem() || n->is_Phi() && n->bottom_type() == Type::MEMORY) { | |
| 371 _dg.make_node(n); | |
| 372 } | |
| 373 } | |
| 374 | |
| 375 // For each memory slice, create the dependences | |
| 376 for (int i = 0; i < _mem_slice_head.length(); i++) { | |
| 377 Node* n = _mem_slice_head.at(i); | |
| 378 Node* n_tail = _mem_slice_tail.at(i); | |
| 379 | |
| 380 // Get slice in predecessor order (last is first) | |
| 381 mem_slice_preds(n_tail, n, _nlist); | |
| 382 | |
| 383 // Make the slice dependent on the root | |
| 384 DepMem* slice = _dg.dep(n); | |
| 385 _dg.make_edge(_dg.root(), slice); | |
| 386 | |
| 387 // Create a sink for the slice | |
| 388 DepMem* slice_sink = _dg.make_node(NULL); | |
| 389 _dg.make_edge(slice_sink, _dg.tail()); | |
| 390 | |
| 391 // Now visit each pair of memory ops, creating the edges | |
| 392 for (int j = _nlist.length() - 1; j >= 0 ; j--) { | |
| 393 Node* s1 = _nlist.at(j); | |
| 394 | |
| 395 // If no dependency yet, use slice | |
| 396 if (_dg.dep(s1)->in_cnt() == 0) { | |
| 397 _dg.make_edge(slice, s1); | |
| 398 } | |
| 399 SWPointer p1(s1->as_Mem(), this); | |
| 400 bool sink_dependent = true; | |
| 401 for (int k = j - 1; k >= 0; k--) { | |
| 402 Node* s2 = _nlist.at(k); | |
| 403 if (s1->is_Load() && s2->is_Load()) | |
| 404 continue; | |
| 405 SWPointer p2(s2->as_Mem(), this); | |
| 406 | |
| 407 int cmp = p1.cmp(p2); | |
| 408 if (SuperWordRTDepCheck && | |
| 409 p1.base() != p2.base() && p1.valid() && p2.valid()) { | |
| 410 // Create a runtime check to disambiguate | |
| 411 OrderedPair pp(p1.base(), p2.base()); | |
| 412 _disjoint_ptrs.append_if_missing(pp); | |
| 413 } else if (!SWPointer::not_equal(cmp)) { | |
| 414 // Possibly same address | |
| 415 _dg.make_edge(s1, s2); | |
| 416 sink_dependent = false; | |
| 417 } | |
| 418 } | |
| 419 if (sink_dependent) { | |
| 420 _dg.make_edge(s1, slice_sink); | |
| 421 } | |
| 422 } | |
| 423 #ifndef PRODUCT | |
| 424 if (TraceSuperWord) { | |
| 425 tty->print_cr("\nDependence graph for slice: %d", n->_idx); | |
| 426 for (int q = 0; q < _nlist.length(); q++) { | |
| 427 _dg.print(_nlist.at(q)); | |
| 428 } | |
| 429 tty->cr(); | |
| 430 } | |
| 431 #endif | |
| 432 _nlist.clear(); | |
| 433 } | |
| 434 | |
| 435 #ifndef PRODUCT | |
| 436 if (TraceSuperWord) { | |
| 437 tty->print_cr("\ndisjoint_ptrs: %s", _disjoint_ptrs.length() > 0 ? "" : "NONE"); | |
| 438 for (int r = 0; r < _disjoint_ptrs.length(); r++) { | |
| 439 _disjoint_ptrs.at(r).print(); | |
| 440 tty->cr(); | |
| 441 } | |
| 442 tty->cr(); | |
| 443 } | |
| 444 #endif | |
| 445 } | |
| 446 | |
| 447 //---------------------------mem_slice_preds--------------------------- | |
| 448 // Return a memory slice (node list) in predecessor order starting at "start" | |
| 449 void SuperWord::mem_slice_preds(Node* start, Node* stop, GrowableArray<Node*> &preds) { | |
| 450 assert(preds.length() == 0, "start empty"); | |
| 451 Node* n = start; | |
| 452 Node* prev = NULL; | |
| 453 while (true) { | |
| 454 assert(in_bb(n), "must be in block"); | |
| 455 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { | |
| 456 Node* out = n->fast_out(i); | |
| 457 if (out->is_Load()) { | |
| 458 if (in_bb(out)) { | |
| 459 preds.push(out); | |
| 460 } | |
| 461 } else { | |
| 462 // FIXME | |
| 463 if (out->is_MergeMem() && !in_bb(out)) { | |
| 464 // Either unrolling is causing a memory edge not to disappear, | |
| 465 // or need to run igvn.optimize() again before SLP | |
| 466 } else if (out->is_Phi() && out->bottom_type() == Type::MEMORY && !in_bb(out)) { | |
| 467 // Ditto. Not sure what else to check further. | |
| 667 | 468 } else if (out->Opcode() == Op_StoreCM && out->in(MemNode::OopStore) == n) { |
| 0 | 469 // StoreCM has an input edge used as a precedence edge. |
| 470 // Maybe an issue when oop stores are vectorized. | |
| 471 } else { | |
| 472 assert(out == prev || prev == NULL, "no branches off of store slice"); | |
| 473 } | |
| 474 } | |
| 475 } | |
| 476 if (n == stop) break; | |
| 477 preds.push(n); | |
| 478 prev = n; | |
| 479 n = n->in(MemNode::Memory); | |
| 480 } | |
| 481 } | |
| 482 | |
| 483 //------------------------------stmts_can_pack--------------------------- | |
| 605 | 484 // Can s1 and s2 be in a pack with s1 immediately preceding s2 and |
| 0 | 485 // s1 aligned at "align" |
| 486 bool SuperWord::stmts_can_pack(Node* s1, Node* s2, int align) { | |
|
987
00977607da34
6879921: CTW failure jdk6_18/hotspot/src/share/vm/utilities/globalDefinitions.cpp:268
cfang
parents:
985
diff
changeset
|
487 |
|
00977607da34
6879921: CTW failure jdk6_18/hotspot/src/share/vm/utilities/globalDefinitions.cpp:268
cfang
parents:
985
diff
changeset
|
488 // Do not use superword for non-primitives |
|
00977607da34
6879921: CTW failure jdk6_18/hotspot/src/share/vm/utilities/globalDefinitions.cpp:268
cfang
parents:
985
diff
changeset
|
489 if((s1->is_Mem() && !is_java_primitive(s1->as_Mem()->memory_type())) || |
|
00977607da34
6879921: CTW failure jdk6_18/hotspot/src/share/vm/utilities/globalDefinitions.cpp:268
cfang
parents:
985
diff
changeset
|
490 (s2->is_Mem() && !is_java_primitive(s2->as_Mem()->memory_type()))) |
|
00977607da34
6879921: CTW failure jdk6_18/hotspot/src/share/vm/utilities/globalDefinitions.cpp:268
cfang
parents:
985
diff
changeset
|
491 return false; |
|
00977607da34
6879921: CTW failure jdk6_18/hotspot/src/share/vm/utilities/globalDefinitions.cpp:268
cfang
parents:
985
diff
changeset
|
492 |
| 0 | 493 if (isomorphic(s1, s2)) { |
| 494 if (independent(s1, s2)) { | |
| 495 if (!exists_at(s1, 0) && !exists_at(s2, 1)) { | |
| 496 if (!s1->is_Mem() || are_adjacent_refs(s1, s2)) { | |
| 497 int s1_align = alignment(s1); | |
| 498 int s2_align = alignment(s2); | |
| 499 if (s1_align == top_align || s1_align == align) { | |
| 500 if (s2_align == top_align || s2_align == align + data_size(s1)) { | |
| 501 return true; | |
| 502 } | |
| 503 } | |
| 504 } | |
| 505 } | |
| 506 } | |
| 507 } | |
| 508 return false; | |
| 509 } | |
| 510 | |
| 511 //------------------------------exists_at--------------------------- | |
| 512 // Does s exist in a pack at position pos? | |
| 513 bool SuperWord::exists_at(Node* s, uint pos) { | |
| 514 for (int i = 0; i < _packset.length(); i++) { | |
| 515 Node_List* p = _packset.at(i); | |
| 516 if (p->at(pos) == s) { | |
| 517 return true; | |
| 518 } | |
| 519 } | |
| 520 return false; | |
| 521 } | |
| 522 | |
| 523 //------------------------------are_adjacent_refs--------------------------- | |
| 524 // Is s1 immediately before s2 in memory? | |
| 525 bool SuperWord::are_adjacent_refs(Node* s1, Node* s2) { | |
| 526 if (!s1->is_Mem() || !s2->is_Mem()) return false; | |
| 527 if (!in_bb(s1) || !in_bb(s2)) return false; | |
| 1585 | 528 |
| 529 // Do not use superword for non-primitives | |
| 530 if (!is_java_primitive(s1->as_Mem()->memory_type()) || | |
| 531 !is_java_primitive(s2->as_Mem()->memory_type())) { | |
| 532 return false; | |
| 533 } | |
| 534 | |
| 0 | 535 // FIXME - co_locate_pack fails on Stores in different mem-slices, so |
| 536 // only pack memops that are in the same alias set until that's fixed. | |
| 537 if (_phase->C->get_alias_index(s1->as_Mem()->adr_type()) != | |
| 538 _phase->C->get_alias_index(s2->as_Mem()->adr_type())) | |
| 539 return false; | |
| 540 SWPointer p1(s1->as_Mem(), this); | |
| 541 SWPointer p2(s2->as_Mem(), this); | |
| 542 if (p1.base() != p2.base() || !p1.comparable(p2)) return false; | |
| 543 int diff = p2.offset_in_bytes() - p1.offset_in_bytes(); | |
| 544 return diff == data_size(s1); | |
| 545 } | |
| 546 | |
| 547 //------------------------------isomorphic--------------------------- | |
| 548 // Are s1 and s2 similar? | |
| 549 bool SuperWord::isomorphic(Node* s1, Node* s2) { | |
| 550 if (s1->Opcode() != s2->Opcode()) return false; | |
| 551 if (s1->req() != s2->req()) return false; | |
| 552 if (s1->in(0) != s2->in(0)) return false; | |
| 553 if (velt_type(s1) != velt_type(s2)) return false; | |
| 554 return true; | |
| 555 } | |
| 556 | |
| 557 //------------------------------independent--------------------------- | |
| 558 // Is there no data path from s1 to s2 or s2 to s1? | |
| 559 bool SuperWord::independent(Node* s1, Node* s2) { | |
| 560 // assert(s1->Opcode() == s2->Opcode(), "check isomorphic first"); | |
| 561 int d1 = depth(s1); | |
| 562 int d2 = depth(s2); | |
| 563 if (d1 == d2) return s1 != s2; | |
| 564 Node* deep = d1 > d2 ? s1 : s2; | |
| 565 Node* shallow = d1 > d2 ? s2 : s1; | |
| 566 | |
| 567 visited_clear(); | |
| 568 | |
| 569 return independent_path(shallow, deep); | |
| 570 } | |
| 571 | |
| 572 //------------------------------independent_path------------------------------ | |
| 573 // Helper for independent | |
| 574 bool SuperWord::independent_path(Node* shallow, Node* deep, uint dp) { | |
| 575 if (dp >= 1000) return false; // stop deep recursion | |
| 576 visited_set(deep); | |
| 577 int shal_depth = depth(shallow); | |
| 578 assert(shal_depth <= depth(deep), "must be"); | |
| 579 for (DepPreds preds(deep, _dg); !preds.done(); preds.next()) { | |
| 580 Node* pred = preds.current(); | |
| 581 if (in_bb(pred) && !visited_test(pred)) { | |
| 582 if (shallow == pred) { | |
| 583 return false; | |
| 584 } | |
| 585 if (shal_depth < depth(pred) && !independent_path(shallow, pred, dp+1)) { | |
| 586 return false; | |
| 587 } | |
| 588 } | |
| 589 } | |
| 590 return true; | |
| 591 } | |
| 592 | |
| 593 //------------------------------set_alignment--------------------------- | |
| 594 void SuperWord::set_alignment(Node* s1, Node* s2, int align) { | |
| 595 set_alignment(s1, align); | |
| 596 set_alignment(s2, align + data_size(s1)); | |
| 597 } | |
| 598 | |
| 599 //------------------------------data_size--------------------------- | |
| 600 int SuperWord::data_size(Node* s) { | |
| 601 const Type* t = velt_type(s); | |
| 602 BasicType bt = t->array_element_basic_type(); | |
|
29
d5fc211aea19
6633953: type2aelembytes{T_ADDRESS} should be 8 bytes in 64 bit VM
kvn
parents:
0
diff
changeset
|
603 int bsize = type2aelembytes(bt); |
| 0 | 604 assert(bsize != 0, "valid size"); |
| 605 return bsize; | |
| 606 } | |
| 607 | |
| 608 //------------------------------extend_packlist--------------------------- | |
| 609 // Extend packset by following use->def and def->use links from pack members. | |
| 610 void SuperWord::extend_packlist() { | |
| 611 bool changed; | |
| 612 do { | |
| 613 changed = false; | |
| 614 for (int i = 0; i < _packset.length(); i++) { | |
| 615 Node_List* p = _packset.at(i); | |
| 616 changed |= follow_use_defs(p); | |
| 617 changed |= follow_def_uses(p); | |
| 618 } | |
| 619 } while (changed); | |
| 620 | |
| 621 #ifndef PRODUCT | |
| 622 if (TraceSuperWord) { | |
| 623 tty->print_cr("\nAfter extend_packlist"); | |
| 624 print_packset(); | |
| 625 } | |
| 626 #endif | |
| 627 } | |
| 628 | |
| 629 //------------------------------follow_use_defs--------------------------- | |
| 630 // Extend the packset by visiting operand definitions of nodes in pack p | |
| 631 bool SuperWord::follow_use_defs(Node_List* p) { | |
| 632 Node* s1 = p->at(0); | |
| 633 Node* s2 = p->at(1); | |
| 634 assert(p->size() == 2, "just checking"); | |
| 635 assert(s1->req() == s2->req(), "just checking"); | |
| 636 assert(alignment(s1) + data_size(s1) == alignment(s2), "just checking"); | |
| 637 | |
| 638 if (s1->is_Load()) return false; | |
| 639 | |
| 640 int align = alignment(s1); | |
| 641 bool changed = false; | |
| 642 int start = s1->is_Store() ? MemNode::ValueIn : 1; | |
| 643 int end = s1->is_Store() ? MemNode::ValueIn+1 : s1->req(); | |
| 644 for (int j = start; j < end; j++) { | |
| 645 Node* t1 = s1->in(j); | |
| 646 Node* t2 = s2->in(j); | |
| 647 if (!in_bb(t1) || !in_bb(t2)) | |
| 648 continue; | |
| 649 if (stmts_can_pack(t1, t2, align)) { | |
| 650 if (est_savings(t1, t2) >= 0) { | |
| 651 Node_List* pair = new Node_List(); | |
| 652 pair->push(t1); | |
| 653 pair->push(t2); | |
| 654 _packset.append(pair); | |
| 655 set_alignment(t1, t2, align); | |
| 656 changed = true; | |
| 657 } | |
| 658 } | |
| 659 } | |
| 660 return changed; | |
| 661 } | |
| 662 | |
| 663 //------------------------------follow_def_uses--------------------------- | |
| 664 // Extend the packset by visiting uses of nodes in pack p | |
| 665 bool SuperWord::follow_def_uses(Node_List* p) { | |
| 666 bool changed = false; | |
| 667 Node* s1 = p->at(0); | |
| 668 Node* s2 = p->at(1); | |
| 669 assert(p->size() == 2, "just checking"); | |
| 670 assert(s1->req() == s2->req(), "just checking"); | |
| 671 assert(alignment(s1) + data_size(s1) == alignment(s2), "just checking"); | |
| 672 | |
| 673 if (s1->is_Store()) return false; | |
| 674 | |
| 675 int align = alignment(s1); | |
| 676 int savings = -1; | |
| 677 Node* u1 = NULL; | |
| 678 Node* u2 = NULL; | |
| 679 for (DUIterator_Fast imax, i = s1->fast_outs(imax); i < imax; i++) { | |
| 680 Node* t1 = s1->fast_out(i); | |
| 681 if (!in_bb(t1)) continue; | |
| 682 for (DUIterator_Fast jmax, j = s2->fast_outs(jmax); j < jmax; j++) { | |
| 683 Node* t2 = s2->fast_out(j); | |
| 684 if (!in_bb(t2)) continue; | |
| 685 if (!opnd_positions_match(s1, t1, s2, t2)) | |
| 686 continue; | |
| 687 if (stmts_can_pack(t1, t2, align)) { | |
| 688 int my_savings = est_savings(t1, t2); | |
| 689 if (my_savings > savings) { | |
| 690 savings = my_savings; | |
| 691 u1 = t1; | |
| 692 u2 = t2; | |
| 693 } | |
| 694 } | |
| 695 } | |
| 696 } | |
| 697 if (savings >= 0) { | |
| 698 Node_List* pair = new Node_List(); | |
| 699 pair->push(u1); | |
| 700 pair->push(u2); | |
| 701 _packset.append(pair); | |
| 702 set_alignment(u1, u2, align); | |
| 703 changed = true; | |
| 704 } | |
| 705 return changed; | |
| 706 } | |
| 707 | |
| 708 //---------------------------opnd_positions_match------------------------- | |
| 709 // Is the use of d1 in u1 at the same operand position as d2 in u2? | |
| 710 bool SuperWord::opnd_positions_match(Node* d1, Node* u1, Node* d2, Node* u2) { | |
| 711 uint ct = u1->req(); | |
| 712 if (ct != u2->req()) return false; | |
| 713 uint i1 = 0; | |
| 714 uint i2 = 0; | |
| 715 do { | |
| 716 for (i1++; i1 < ct; i1++) if (u1->in(i1) == d1) break; | |
| 717 for (i2++; i2 < ct; i2++) if (u2->in(i2) == d2) break; | |
| 718 if (i1 != i2) { | |
| 719 return false; | |
| 720 } | |
| 721 } while (i1 < ct); | |
| 722 return true; | |
| 723 } | |
| 724 | |
| 725 //------------------------------est_savings--------------------------- | |
| 726 // Estimate the savings from executing s1 and s2 as a pack | |
| 727 int SuperWord::est_savings(Node* s1, Node* s2) { | |
| 728 int save = 2 - 1; // 2 operations per instruction in packed form | |
| 729 | |
| 730 // inputs | |
| 731 for (uint i = 1; i < s1->req(); i++) { | |
| 732 Node* x1 = s1->in(i); | |
| 733 Node* x2 = s2->in(i); | |
| 734 if (x1 != x2) { | |
| 735 if (are_adjacent_refs(x1, x2)) { | |
| 736 save += adjacent_profit(x1, x2); | |
| 737 } else if (!in_packset(x1, x2)) { | |
| 738 save -= pack_cost(2); | |
| 739 } else { | |
| 740 save += unpack_cost(2); | |
| 741 } | |
| 742 } | |
| 743 } | |
| 744 | |
| 745 // uses of result | |
| 746 uint ct = 0; | |
| 747 for (DUIterator_Fast imax, i = s1->fast_outs(imax); i < imax; i++) { | |
| 748 Node* s1_use = s1->fast_out(i); | |
| 749 for (int j = 0; j < _packset.length(); j++) { | |
| 750 Node_List* p = _packset.at(j); | |
| 751 if (p->at(0) == s1_use) { | |
| 752 for (DUIterator_Fast kmax, k = s2->fast_outs(kmax); k < kmax; k++) { | |
| 753 Node* s2_use = s2->fast_out(k); | |
| 754 if (p->at(p->size()-1) == s2_use) { | |
| 755 ct++; | |
| 756 if (are_adjacent_refs(s1_use, s2_use)) { | |
| 757 save += adjacent_profit(s1_use, s2_use); | |
| 758 } | |
| 759 } | |
| 760 } | |
| 761 } | |
| 762 } | |
| 763 } | |
| 764 | |
| 765 if (ct < s1->outcnt()) save += unpack_cost(1); | |
| 766 if (ct < s2->outcnt()) save += unpack_cost(1); | |
| 767 | |
| 768 return save; | |
| 769 } | |
| 770 | |
| 771 //------------------------------costs--------------------------- | |
| 772 int SuperWord::adjacent_profit(Node* s1, Node* s2) { return 2; } | |
| 773 int SuperWord::pack_cost(int ct) { return ct; } | |
| 774 int SuperWord::unpack_cost(int ct) { return ct; } | |
| 775 | |
| 776 //------------------------------combine_packs--------------------------- | |
| 777 // Combine packs A and B with A.last == B.first into A.first..,A.last,B.second,..B.last | |
| 778 void SuperWord::combine_packs() { | |
| 779 bool changed; | |
| 780 do { | |
| 781 changed = false; | |
| 782 for (int i = 0; i < _packset.length(); i++) { | |
| 783 Node_List* p1 = _packset.at(i); | |
| 784 if (p1 == NULL) continue; | |
| 785 for (int j = 0; j < _packset.length(); j++) { | |
| 786 Node_List* p2 = _packset.at(j); | |
| 787 if (p2 == NULL) continue; | |
| 788 if (p1->at(p1->size()-1) == p2->at(0)) { | |
| 789 for (uint k = 1; k < p2->size(); k++) { | |
| 790 p1->push(p2->at(k)); | |
| 791 } | |
| 792 _packset.at_put(j, NULL); | |
| 793 changed = true; | |
| 794 } | |
| 795 } | |
| 796 } | |
| 797 } while (changed); | |
| 798 | |
| 799 for (int i = _packset.length() - 1; i >= 0; i--) { | |
| 800 Node_List* p1 = _packset.at(i); | |
| 801 if (p1 == NULL) { | |
| 802 _packset.remove_at(i); | |
| 803 } | |
| 804 } | |
| 805 | |
| 806 #ifndef PRODUCT | |
| 807 if (TraceSuperWord) { | |
| 808 tty->print_cr("\nAfter combine_packs"); | |
| 809 print_packset(); | |
| 810 } | |
| 811 #endif | |
| 812 } | |
| 813 | |
| 814 //-----------------------------construct_my_pack_map-------------------------- | |
| 815 // Construct the map from nodes to packs. Only valid after the | |
| 816 // point where a node is only in one pack (after combine_packs). | |
| 817 void SuperWord::construct_my_pack_map() { | |
| 818 Node_List* rslt = NULL; | |
| 819 for (int i = 0; i < _packset.length(); i++) { | |
| 820 Node_List* p = _packset.at(i); | |
| 821 for (uint j = 0; j < p->size(); j++) { | |
| 822 Node* s = p->at(j); | |
| 823 assert(my_pack(s) == NULL, "only in one pack"); | |
| 824 set_my_pack(s, p); | |
| 825 } | |
| 826 } | |
| 827 } | |
| 828 | |
| 829 //------------------------------filter_packs--------------------------- | |
| 830 // Remove packs that are not implemented or not profitable. | |
| 831 void SuperWord::filter_packs() { | |
| 832 | |
| 833 // Remove packs that are not implemented | |
| 834 for (int i = _packset.length() - 1; i >= 0; i--) { | |
| 835 Node_List* pk = _packset.at(i); | |
| 836 bool impl = implemented(pk); | |
| 837 if (!impl) { | |
| 838 #ifndef PRODUCT | |
| 839 if (TraceSuperWord && Verbose) { | |
| 840 tty->print_cr("Unimplemented"); | |
| 841 pk->at(0)->dump(); | |
| 842 } | |
| 843 #endif | |
| 844 remove_pack_at(i); | |
| 845 } | |
| 846 } | |
| 847 | |
| 848 // Remove packs that are not profitable | |
| 849 bool changed; | |
| 850 do { | |
| 851 changed = false; | |
| 852 for (int i = _packset.length() - 1; i >= 0; i--) { | |
| 853 Node_List* pk = _packset.at(i); | |
| 854 bool prof = profitable(pk); | |
| 855 if (!prof) { | |
| 856 #ifndef PRODUCT | |
| 857 if (TraceSuperWord && Verbose) { | |
| 858 tty->print_cr("Unprofitable"); | |
| 859 pk->at(0)->dump(); | |
| 860 } | |
| 861 #endif | |
| 862 remove_pack_at(i); | |
| 863 changed = true; | |
| 864 } | |
| 865 } | |
| 866 } while (changed); | |
| 867 | |
| 868 #ifndef PRODUCT | |
| 869 if (TraceSuperWord) { | |
| 870 tty->print_cr("\nAfter filter_packs"); | |
| 871 print_packset(); | |
| 872 tty->cr(); | |
| 873 } | |
| 874 #endif | |
| 875 } | |
| 876 | |
| 877 //------------------------------implemented--------------------------- | |
| 878 // Can code be generated for pack p? | |
| 879 bool SuperWord::implemented(Node_List* p) { | |
| 880 Node* p0 = p->at(0); | |
| 881 int vopc = VectorNode::opcode(p0->Opcode(), p->size(), velt_type(p0)); | |
| 882 return vopc > 0 && Matcher::has_match_rule(vopc); | |
| 883 } | |
| 884 | |
| 885 //------------------------------profitable--------------------------- | |
| 886 // For pack p, are all operands and all uses (with in the block) vector? | |
| 887 bool SuperWord::profitable(Node_List* p) { | |
| 888 Node* p0 = p->at(0); | |
| 889 uint start, end; | |
| 890 vector_opd_range(p0, &start, &end); | |
| 891 | |
| 892 // Return false if some input is not vector and inside block | |
| 893 for (uint i = start; i < end; i++) { | |
| 894 if (!is_vector_use(p0, i)) { | |
| 895 // For now, return false if not scalar promotion case (inputs are the same.) | |
| 605 | 896 // Later, implement PackNode and allow differing, non-vector inputs |
| 0 | 897 // (maybe just the ones from outside the block.) |
| 898 Node* p0_def = p0->in(i); | |
| 899 for (uint j = 1; j < p->size(); j++) { | |
| 900 Node* use = p->at(j); | |
| 901 Node* def = use->in(i); | |
| 902 if (p0_def != def) | |
| 903 return false; | |
| 904 } | |
| 905 } | |
| 906 } | |
| 907 if (!p0->is_Store()) { | |
| 908 // For now, return false if not all uses are vector. | |
| 909 // Later, implement ExtractNode and allow non-vector uses (maybe | |
| 910 // just the ones outside the block.) | |
| 911 for (uint i = 0; i < p->size(); i++) { | |
| 912 Node* def = p->at(i); | |
| 913 for (DUIterator_Fast jmax, j = def->fast_outs(jmax); j < jmax; j++) { | |
| 914 Node* use = def->fast_out(j); | |
| 915 for (uint k = 0; k < use->req(); k++) { | |
| 916 Node* n = use->in(k); | |
| 917 if (def == n) { | |
| 918 if (!is_vector_use(use, k)) { | |
| 919 return false; | |
| 920 } | |
| 921 } | |
| 922 } | |
| 923 } | |
| 924 } | |
| 925 } | |
| 926 return true; | |
| 927 } | |
| 928 | |
| 929 //------------------------------schedule--------------------------- | |
| 930 // Adjust the memory graph for the packed operations | |
| 931 void SuperWord::schedule() { | |
| 932 | |
| 933 // Co-locate in the memory graph the members of each memory pack | |
| 934 for (int i = 0; i < _packset.length(); i++) { | |
| 935 co_locate_pack(_packset.at(i)); | |
| 936 } | |
| 937 } | |
| 938 | |
| 667 | 939 //-------------------------------remove_and_insert------------------- |
| 940 //remove "current" from its current position in the memory graph and insert | |
| 941 //it after the appropriate insertion point (lip or uip) | |
| 942 void SuperWord::remove_and_insert(MemNode *current, MemNode *prev, MemNode *lip, | |
| 943 Node *uip, Unique_Node_List &sched_before) { | |
| 944 Node* my_mem = current->in(MemNode::Memory); | |
| 945 _igvn.hash_delete(current); | |
| 946 _igvn.hash_delete(my_mem); | |
| 947 | |
| 948 //remove current_store from its current position in the memmory graph | |
| 949 for (DUIterator i = current->outs(); current->has_out(i); i++) { | |
| 950 Node* use = current->out(i); | |
| 951 if (use->is_Mem()) { | |
| 952 assert(use->in(MemNode::Memory) == current, "must be"); | |
| 953 _igvn.hash_delete(use); | |
| 954 if (use == prev) { // connect prev to my_mem | |
| 955 use->set_req(MemNode::Memory, my_mem); | |
| 956 } else if (sched_before.member(use)) { | |
| 957 _igvn.hash_delete(uip); | |
| 958 use->set_req(MemNode::Memory, uip); | |
| 959 } else { | |
| 960 _igvn.hash_delete(lip); | |
| 961 use->set_req(MemNode::Memory, lip); | |
| 962 } | |
| 963 _igvn._worklist.push(use); | |
| 964 --i; //deleted this edge; rescan position | |
| 965 } | |
| 966 } | |
| 967 | |
| 968 bool sched_up = sched_before.member(current); | |
| 969 Node *insert_pt = sched_up ? uip : lip; | |
| 970 _igvn.hash_delete(insert_pt); | |
| 971 | |
| 972 // all uses of insert_pt's memory state should use current's instead | |
| 973 for (DUIterator i = insert_pt->outs(); insert_pt->has_out(i); i++) { | |
| 974 Node* use = insert_pt->out(i); | |
| 975 if (use->is_Mem()) { | |
| 976 assert(use->in(MemNode::Memory) == insert_pt, "must be"); | |
| 977 _igvn.hash_delete(use); | |
| 978 use->set_req(MemNode::Memory, current); | |
| 979 _igvn._worklist.push(use); | |
| 980 --i; //deleted this edge; rescan position | |
| 981 } else if (!sched_up && use->is_Phi() && use->bottom_type() == Type::MEMORY) { | |
| 982 uint pos; //lip (lower insert point) must be the last one in the memory slice | |
| 983 _igvn.hash_delete(use); | |
| 984 for (pos=1; pos < use->req(); pos++) { | |
| 985 if (use->in(pos) == insert_pt) break; | |
| 986 } | |
| 987 use->set_req(pos, current); | |
| 988 _igvn._worklist.push(use); | |
| 989 --i; | |
| 990 } | |
| 991 } | |
| 992 | |
| 993 //connect current to insert_pt | |
| 994 current->set_req(MemNode::Memory, insert_pt); | |
| 995 _igvn._worklist.push(current); | |
| 996 } | |
| 997 | |
| 998 //------------------------------co_locate_pack---------------------------------- | |
| 999 // To schedule a store pack, we need to move any sandwiched memory ops either before | |
| 1000 // or after the pack, based upon dependence information: | |
| 1001 // (1) If any store in the pack depends on the sandwiched memory op, the | |
| 1002 // sandwiched memory op must be scheduled BEFORE the pack; | |
| 1003 // (2) If a sandwiched memory op depends on any store in the pack, the | |
| 1004 // sandwiched memory op must be scheduled AFTER the pack; | |
| 1005 // (3) If a sandwiched memory op (say, memA) depends on another sandwiched | |
| 1006 // memory op (say memB), memB must be scheduled before memA. So, if memA is | |
| 1007 // scheduled before the pack, memB must also be scheduled before the pack; | |
| 1008 // (4) If there is no dependence restriction for a sandwiched memory op, we simply | |
| 1009 // schedule this store AFTER the pack | |
| 1010 // (5) We know there is no dependence cycle, so there in no other case; | |
| 1011 // (6) Finally, all memory ops in another single pack should be moved in the same direction. | |
| 1012 // | |
| 952 | 1013 // To schedule a load pack, we use the memory state of either the first or the last load in |
| 1014 // the pack, based on the dependence constraint. | |
| 0 | 1015 void SuperWord::co_locate_pack(Node_List* pk) { |
| 1016 if (pk->at(0)->is_Store()) { | |
| 1017 MemNode* first = executed_first(pk)->as_Mem(); | |
| 1018 MemNode* last = executed_last(pk)->as_Mem(); | |
| 667 | 1019 Unique_Node_List schedule_before_pack; |
| 1020 Unique_Node_List memops; | |
| 1021 | |
| 0 | 1022 MemNode* current = last->in(MemNode::Memory)->as_Mem(); |
| 667 | 1023 MemNode* previous = last; |
| 0 | 1024 while (true) { |
| 1025 assert(in_bb(current), "stay in block"); | |
| 667 | 1026 memops.push(previous); |
| 1027 for (DUIterator i = current->outs(); current->has_out(i); i++) { | |
| 1028 Node* use = current->out(i); | |
| 1029 if (use->is_Mem() && use != previous) | |
| 1030 memops.push(use); | |
| 1031 } | |
| 1032 if(current == first) break; | |
| 1033 previous = current; | |
| 1034 current = current->in(MemNode::Memory)->as_Mem(); | |
| 1035 } | |
| 1036 | |
| 1037 // determine which memory operations should be scheduled before the pack | |
| 1038 for (uint i = 1; i < memops.size(); i++) { | |
| 1039 Node *s1 = memops.at(i); | |
| 1040 if (!in_pack(s1, pk) && !schedule_before_pack.member(s1)) { | |
| 1041 for (uint j = 0; j< i; j++) { | |
| 1042 Node *s2 = memops.at(j); | |
| 1043 if (!independent(s1, s2)) { | |
| 1044 if (in_pack(s2, pk) || schedule_before_pack.member(s2)) { | |
| 1045 schedule_before_pack.push(s1); //s1 must be scheduled before | |
| 1046 Node_List* mem_pk = my_pack(s1); | |
| 1047 if (mem_pk != NULL) { | |
| 1048 for (uint ii = 0; ii < mem_pk->size(); ii++) { | |
| 1049 Node* s = mem_pk->at(ii); // follow partner | |
| 1050 if (memops.member(s) && !schedule_before_pack.member(s)) | |
| 1051 schedule_before_pack.push(s); | |
| 1052 } | |
| 1053 } | |
| 1054 } | |
| 1055 } | |
| 1056 } | |
| 1057 } | |
| 1058 } | |
| 1059 | |
| 1060 MemNode* lower_insert_pt = last; | |
| 1061 Node* upper_insert_pt = first->in(MemNode::Memory); | |
| 1062 previous = last; //previous store in pk | |
| 1063 current = last->in(MemNode::Memory)->as_Mem(); | |
| 1064 | |
| 1065 //start scheduling from "last" to "first" | |
| 1066 while (true) { | |
| 1067 assert(in_bb(current), "stay in block"); | |
| 1068 assert(in_pack(previous, pk), "previous stays in pack"); | |
| 0 | 1069 Node* my_mem = current->in(MemNode::Memory); |
| 667 | 1070 |
| 0 | 1071 if (in_pack(current, pk)) { |
| 667 | 1072 // Forward users of my memory state (except "previous) to my input memory state |
| 0 | 1073 _igvn.hash_delete(current); |
| 1074 for (DUIterator i = current->outs(); current->has_out(i); i++) { | |
| 1075 Node* use = current->out(i); | |
| 667 | 1076 if (use->is_Mem() && use != previous) { |
| 0 | 1077 assert(use->in(MemNode::Memory) == current, "must be"); |
| 1078 _igvn.hash_delete(use); | |
| 667 | 1079 if (schedule_before_pack.member(use)) { |
| 1080 _igvn.hash_delete(upper_insert_pt); | |
| 1081 use->set_req(MemNode::Memory, upper_insert_pt); | |
| 1082 } else { | |
| 1083 _igvn.hash_delete(lower_insert_pt); | |
| 1084 use->set_req(MemNode::Memory, lower_insert_pt); | |
| 1085 } | |
| 0 | 1086 _igvn._worklist.push(use); |
| 1087 --i; // deleted this edge; rescan position | |
| 1088 } | |
| 1089 } | |
| 667 | 1090 previous = current; |
| 1091 } else { // !in_pack(current, pk) ==> a sandwiched store | |
| 1092 remove_and_insert(current, previous, lower_insert_pt, upper_insert_pt, schedule_before_pack); | |
| 0 | 1093 } |
| 667 | 1094 |
| 0 | 1095 if (current == first) break; |
| 1096 current = my_mem->as_Mem(); | |
| 667 | 1097 } // end while |
| 1098 } else if (pk->at(0)->is_Load()) { //load | |
| 952 | 1099 // all loads in the pack should have the same memory state. By default, |
| 1100 // we use the memory state of the last load. However, if any load could | |
| 1101 // not be moved down due to the dependence constraint, we use the memory | |
| 1102 // state of the first load. | |
| 1103 Node* last_mem = executed_last(pk)->in(MemNode::Memory); | |
| 1104 Node* first_mem = executed_first(pk)->in(MemNode::Memory); | |
| 1105 bool schedule_last = true; | |
| 1106 for (uint i = 0; i < pk->size(); i++) { | |
| 1107 Node* ld = pk->at(i); | |
| 1108 for (Node* current = last_mem; current != ld->in(MemNode::Memory); | |
| 1109 current=current->in(MemNode::Memory)) { | |
| 1110 assert(current != first_mem, "corrupted memory graph"); | |
| 1111 if(current->is_Mem() && !independent(current, ld)){ | |
| 1112 schedule_last = false; // a later store depends on this load | |
| 1113 break; | |
| 1114 } | |
| 1115 } | |
| 1116 } | |
| 1117 | |
| 1118 Node* mem_input = schedule_last ? last_mem : first_mem; | |
| 1119 _igvn.hash_delete(mem_input); | |
| 1120 // Give each load the same memory state | |
| 0 | 1121 for (uint i = 0; i < pk->size(); i++) { |
| 1122 LoadNode* ld = pk->at(i)->as_Load(); | |
| 1123 _igvn.hash_delete(ld); | |
| 952 | 1124 ld->set_req(MemNode::Memory, mem_input); |
| 0 | 1125 _igvn._worklist.push(ld); |
| 1126 } | |
| 1127 } | |
| 1128 } | |
| 1129 | |
| 1130 //------------------------------output--------------------------- | |
| 1131 // Convert packs into vector node operations | |
| 1132 void SuperWord::output() { | |
| 1133 if (_packset.length() == 0) return; | |
| 1134 | |
| 1135 // MUST ENSURE main loop's initial value is properly aligned: | |
| 1136 // (iv_initial_value + min_iv_offset) % vector_width_in_bytes() == 0 | |
| 1137 | |
| 1138 align_initial_loop_index(align_to_ref()); | |
| 1139 | |
| 1140 // Insert extract (unpack) operations for scalar uses | |
| 1141 for (int i = 0; i < _packset.length(); i++) { | |
| 1142 insert_extracts(_packset.at(i)); | |
| 1143 } | |
| 1144 | |
| 1145 for (int i = 0; i < _block.length(); i++) { | |
| 1146 Node* n = _block.at(i); | |
| 1147 Node_List* p = my_pack(n); | |
| 1148 if (p && n == executed_last(p)) { | |
| 1149 uint vlen = p->size(); | |
| 1150 Node* vn = NULL; | |
| 1151 Node* low_adr = p->at(0); | |
| 1152 Node* first = executed_first(p); | |
| 1153 if (n->is_Load()) { | |
| 1154 int opc = n->Opcode(); | |
| 1155 Node* ctl = n->in(MemNode::Control); | |
| 1156 Node* mem = first->in(MemNode::Memory); | |
| 1157 Node* adr = low_adr->in(MemNode::Address); | |
| 1158 const TypePtr* atyp = n->adr_type(); | |
| 1159 vn = VectorLoadNode::make(_phase->C, opc, ctl, mem, adr, atyp, vlen); | |
| 1160 | |
| 1161 } else if (n->is_Store()) { | |
| 1162 // Promote value to be stored to vector | |
| 1163 VectorNode* val = vector_opd(p, MemNode::ValueIn); | |
| 1164 | |
| 1165 int opc = n->Opcode(); | |
| 1166 Node* ctl = n->in(MemNode::Control); | |
| 1167 Node* mem = first->in(MemNode::Memory); | |
| 1168 Node* adr = low_adr->in(MemNode::Address); | |
| 1169 const TypePtr* atyp = n->adr_type(); | |
| 1170 vn = VectorStoreNode::make(_phase->C, opc, ctl, mem, adr, atyp, val, vlen); | |
| 1171 | |
| 1172 } else if (n->req() == 3) { | |
| 1173 // Promote operands to vector | |
| 1174 Node* in1 = vector_opd(p, 1); | |
| 1175 Node* in2 = vector_opd(p, 2); | |
| 1176 vn = VectorNode::make(_phase->C, n->Opcode(), in1, in2, vlen, velt_type(n)); | |
| 1177 | |
| 1178 } else { | |
| 1179 ShouldNotReachHere(); | |
| 1180 } | |
| 1181 | |
| 1182 _phase->_igvn.register_new_node_with_optimizer(vn); | |
| 1183 _phase->set_ctrl(vn, _phase->get_ctrl(p->at(0))); | |
| 1184 for (uint j = 0; j < p->size(); j++) { | |
| 1185 Node* pm = p->at(j); | |
|
1621
6027dddc26c6
6677629: PhaseIterGVN::subsume_node() should call hash_delete() and add_users_to_worklist()
kvn
parents:
1585
diff
changeset
|
1186 _igvn.replace_node(pm, vn); |
| 0 | 1187 } |
| 1188 _igvn._worklist.push(vn); | |
| 1189 } | |
| 1190 } | |
| 1191 } | |
| 1192 | |
| 1193 //------------------------------vector_opd--------------------------- | |
| 1194 // Create a vector operand for the nodes in pack p for operand: in(opd_idx) | |
| 1195 VectorNode* SuperWord::vector_opd(Node_List* p, int opd_idx) { | |
| 1196 Node* p0 = p->at(0); | |
| 1197 uint vlen = p->size(); | |
| 1198 Node* opd = p0->in(opd_idx); | |
| 1199 | |
| 1200 bool same_opd = true; | |
| 1201 for (uint i = 1; i < vlen; i++) { | |
| 1202 Node* pi = p->at(i); | |
| 1203 Node* in = pi->in(opd_idx); | |
| 1204 if (opd != in) { | |
| 1205 same_opd = false; | |
| 1206 break; | |
| 1207 } | |
| 1208 } | |
| 1209 | |
| 1210 if (same_opd) { | |
| 1211 if (opd->is_Vector()) { | |
| 1212 return (VectorNode*)opd; // input is matching vector | |
| 1213 } | |
| 1214 // Convert scalar input to vector. Use p0's type because it's container | |
| 1215 // maybe smaller than the operand's container. | |
| 1216 const Type* opd_t = velt_type(!in_bb(opd) ? p0 : opd); | |
| 1217 const Type* p0_t = velt_type(p0); | |
| 1218 if (p0_t->higher_equal(opd_t)) opd_t = p0_t; | |
| 1219 VectorNode* vn = VectorNode::scalar2vector(_phase->C, opd, vlen, opd_t); | |
| 1220 | |
| 1221 _phase->_igvn.register_new_node_with_optimizer(vn); | |
| 1222 _phase->set_ctrl(vn, _phase->get_ctrl(opd)); | |
| 1223 return vn; | |
| 1224 } | |
| 1225 | |
| 1226 // Insert pack operation | |
| 1227 const Type* opd_t = velt_type(!in_bb(opd) ? p0 : opd); | |
| 1228 PackNode* pk = PackNode::make(_phase->C, opd, opd_t); | |
| 1229 | |
| 1230 for (uint i = 1; i < vlen; i++) { | |
| 1231 Node* pi = p->at(i); | |
| 1232 Node* in = pi->in(opd_idx); | |
| 1233 assert(my_pack(in) == NULL, "Should already have been unpacked"); | |
| 1234 assert(opd_t == velt_type(!in_bb(in) ? pi : in), "all same type"); | |
| 1235 pk->add_opd(in); | |
| 1236 } | |
| 1237 _phase->_igvn.register_new_node_with_optimizer(pk); | |
| 1238 _phase->set_ctrl(pk, _phase->get_ctrl(opd)); | |
| 1239 return pk; | |
| 1240 } | |
| 1241 | |
| 1242 //------------------------------insert_extracts--------------------------- | |
| 1243 // If a use of pack p is not a vector use, then replace the | |
| 1244 // use with an extract operation. | |
| 1245 void SuperWord::insert_extracts(Node_List* p) { | |
| 1246 if (p->at(0)->is_Store()) return; | |
| 1247 assert(_n_idx_list.is_empty(), "empty (node,index) list"); | |
| 1248 | |
| 1249 // Inspect each use of each pack member. For each use that is | |
| 1250 // not a vector use, replace the use with an extract operation. | |
| 1251 | |
| 1252 for (uint i = 0; i < p->size(); i++) { | |
| 1253 Node* def = p->at(i); | |
| 1254 for (DUIterator_Fast jmax, j = def->fast_outs(jmax); j < jmax; j++) { | |
| 1255 Node* use = def->fast_out(j); | |
| 1256 for (uint k = 0; k < use->req(); k++) { | |
| 1257 Node* n = use->in(k); | |
| 1258 if (def == n) { | |
| 1259 if (!is_vector_use(use, k)) { | |
| 1260 _n_idx_list.push(use, k); | |
| 1261 } | |
| 1262 } | |
| 1263 } | |
| 1264 } | |
| 1265 } | |
| 1266 | |
| 1267 while (_n_idx_list.is_nonempty()) { | |
| 1268 Node* use = _n_idx_list.node(); | |
| 1269 int idx = _n_idx_list.index(); | |
| 1270 _n_idx_list.pop(); | |
| 1271 Node* def = use->in(idx); | |
| 1272 | |
| 1273 // Insert extract operation | |
| 1274 _igvn.hash_delete(def); | |
| 1275 _igvn.hash_delete(use); | |
| 1276 int def_pos = alignment(def) / data_size(def); | |
| 1277 const Type* def_t = velt_type(def); | |
| 1278 | |
| 1279 Node* ex = ExtractNode::make(_phase->C, def, def_pos, def_t); | |
| 1280 _phase->_igvn.register_new_node_with_optimizer(ex); | |
| 1281 _phase->set_ctrl(ex, _phase->get_ctrl(def)); | |
| 1282 use->set_req(idx, ex); | |
| 1283 _igvn._worklist.push(def); | |
| 1284 _igvn._worklist.push(use); | |
| 1285 | |
| 1286 bb_insert_after(ex, bb_idx(def)); | |
| 1287 set_velt_type(ex, def_t); | |
| 1288 } | |
| 1289 } | |
| 1290 | |
| 1291 //------------------------------is_vector_use--------------------------- | |
| 1292 // Is use->in(u_idx) a vector use? | |
| 1293 bool SuperWord::is_vector_use(Node* use, int u_idx) { | |
| 1294 Node_List* u_pk = my_pack(use); | |
| 1295 if (u_pk == NULL) return false; | |
| 1296 Node* def = use->in(u_idx); | |
| 1297 Node_List* d_pk = my_pack(def); | |
| 1298 if (d_pk == NULL) { | |
| 1299 // check for scalar promotion | |
| 1300 Node* n = u_pk->at(0)->in(u_idx); | |
| 1301 for (uint i = 1; i < u_pk->size(); i++) { | |
| 1302 if (u_pk->at(i)->in(u_idx) != n) return false; | |
| 1303 } | |
| 1304 return true; | |
| 1305 } | |
| 1306 if (u_pk->size() != d_pk->size()) | |
| 1307 return false; | |
| 1308 for (uint i = 0; i < u_pk->size(); i++) { | |
| 1309 Node* ui = u_pk->at(i); | |
| 1310 Node* di = d_pk->at(i); | |
| 1311 if (ui->in(u_idx) != di || alignment(ui) != alignment(di)) | |
| 1312 return false; | |
| 1313 } | |
| 1314 return true; | |
| 1315 } | |
| 1316 | |
| 1317 //------------------------------construct_bb--------------------------- | |
| 1318 // Construct reverse postorder list of block members | |
| 1319 void SuperWord::construct_bb() { | |
| 1320 Node* entry = bb(); | |
| 1321 | |
| 1322 assert(_stk.length() == 0, "stk is empty"); | |
| 1323 assert(_block.length() == 0, "block is empty"); | |
| 1324 assert(_data_entry.length() == 0, "data_entry is empty"); | |
| 1325 assert(_mem_slice_head.length() == 0, "mem_slice_head is empty"); | |
| 1326 assert(_mem_slice_tail.length() == 0, "mem_slice_tail is empty"); | |
| 1327 | |
| 1328 // Find non-control nodes with no inputs from within block, | |
| 1329 // create a temporary map from node _idx to bb_idx for use | |
| 1330 // by the visited and post_visited sets, | |
| 1331 // and count number of nodes in block. | |
| 1332 int bb_ct = 0; | |
| 1333 for (uint i = 0; i < lpt()->_body.size(); i++ ) { | |
| 1334 Node *n = lpt()->_body.at(i); | |
| 1335 set_bb_idx(n, i); // Create a temporary map | |
| 1336 if (in_bb(n)) { | |
| 1337 bb_ct++; | |
| 1338 if (!n->is_CFG()) { | |
| 1339 bool found = false; | |
| 1340 for (uint j = 0; j < n->req(); j++) { | |
| 1341 Node* def = n->in(j); | |
| 1342 if (def && in_bb(def)) { | |
| 1343 found = true; | |
| 1344 break; | |
| 1345 } | |
| 1346 } | |
| 1347 if (!found) { | |
| 1348 assert(n != entry, "can't be entry"); | |
| 1349 _data_entry.push(n); | |
| 1350 } | |
| 1351 } | |
| 1352 } | |
| 1353 } | |
| 1354 | |
| 1355 // Find memory slices (head and tail) | |
| 1356 for (DUIterator_Fast imax, i = lp()->fast_outs(imax); i < imax; i++) { | |
| 1357 Node *n = lp()->fast_out(i); | |
| 1358 if (in_bb(n) && (n->is_Phi() && n->bottom_type() == Type::MEMORY)) { | |
| 1359 Node* n_tail = n->in(LoopNode::LoopBackControl); | |
|
253
b0fe4deeb9fb
6726999: nsk/stress/jck12a/jck12a010 assert(n != null,"Bad immediate dominator info.")
kvn
parents:
235
diff
changeset
|
1360 if (n_tail != n->in(LoopNode::EntryControl)) { |
|
b0fe4deeb9fb
6726999: nsk/stress/jck12a/jck12a010 assert(n != null,"Bad immediate dominator info.")
kvn
parents:
235
diff
changeset
|
1361 _mem_slice_head.push(n); |
|
b0fe4deeb9fb
6726999: nsk/stress/jck12a/jck12a010 assert(n != null,"Bad immediate dominator info.")
kvn
parents:
235
diff
changeset
|
1362 _mem_slice_tail.push(n_tail); |
|
b0fe4deeb9fb
6726999: nsk/stress/jck12a/jck12a010 assert(n != null,"Bad immediate dominator info.")
kvn
parents:
235
diff
changeset
|
1363 } |
| 0 | 1364 } |
| 1365 } | |
| 1366 | |
| 1367 // Create an RPO list of nodes in block | |
| 1368 | |
| 1369 visited_clear(); | |
| 1370 post_visited_clear(); | |
| 1371 | |
| 1372 // Push all non-control nodes with no inputs from within block, then control entry | |
| 1373 for (int j = 0; j < _data_entry.length(); j++) { | |
| 1374 Node* n = _data_entry.at(j); | |
| 1375 visited_set(n); | |
| 1376 _stk.push(n); | |
| 1377 } | |
| 1378 visited_set(entry); | |
| 1379 _stk.push(entry); | |
| 1380 | |
| 1381 // Do a depth first walk over out edges | |
| 1382 int rpo_idx = bb_ct - 1; | |
| 1383 int size; | |
| 1384 while ((size = _stk.length()) > 0) { | |
| 1385 Node* n = _stk.top(); // Leave node on stack | |
| 1386 if (!visited_test_set(n)) { | |
| 1387 // forward arc in graph | |
| 1388 } else if (!post_visited_test(n)) { | |
| 1389 // cross or back arc | |
| 1390 for (DUIterator_Fast imax, i = n->fast_outs(imax); i < imax; i++) { | |
| 1391 Node *use = n->fast_out(i); | |
| 1392 if (in_bb(use) && !visited_test(use) && | |
| 1393 // Don't go around backedge | |
| 1394 (!use->is_Phi() || n == entry)) { | |
| 1395 _stk.push(use); | |
| 1396 } | |
| 1397 } | |
| 1398 if (_stk.length() == size) { | |
| 1399 // There were no additional uses, post visit node now | |
| 1400 _stk.pop(); // Remove node from stack | |
| 1401 assert(rpo_idx >= 0, ""); | |
| 1402 _block.at_put_grow(rpo_idx, n); | |
| 1403 rpo_idx--; | |
| 1404 post_visited_set(n); | |
| 1405 assert(rpo_idx >= 0 || _stk.is_empty(), ""); | |
| 1406 } | |
| 1407 } else { | |
| 1408 _stk.pop(); // Remove post-visited node from stack | |
| 1409 } | |
| 1410 } | |
| 1411 | |
| 1412 // Create real map of block indices for nodes | |
| 1413 for (int j = 0; j < _block.length(); j++) { | |
| 1414 Node* n = _block.at(j); | |
| 1415 set_bb_idx(n, j); | |
| 1416 } | |
| 1417 | |
| 1418 initialize_bb(); // Ensure extra info is allocated. | |
| 1419 | |
| 1420 #ifndef PRODUCT | |
| 1421 if (TraceSuperWord) { | |
| 1422 print_bb(); | |
| 1423 tty->print_cr("\ndata entry nodes: %s", _data_entry.length() > 0 ? "" : "NONE"); | |
| 1424 for (int m = 0; m < _data_entry.length(); m++) { | |
| 1425 tty->print("%3d ", m); | |
| 1426 _data_entry.at(m)->dump(); | |
| 1427 } | |
| 1428 tty->print_cr("\nmemory slices: %s", _mem_slice_head.length() > 0 ? "" : "NONE"); | |
| 1429 for (int m = 0; m < _mem_slice_head.length(); m++) { | |
| 1430 tty->print("%3d ", m); _mem_slice_head.at(m)->dump(); | |
| 1431 tty->print(" "); _mem_slice_tail.at(m)->dump(); | |
| 1432 } | |
| 1433 } | |
| 1434 #endif | |
| 1435 assert(rpo_idx == -1 && bb_ct == _block.length(), "all block members found"); | |
| 1436 } | |
| 1437 | |
| 1438 //------------------------------initialize_bb--------------------------- | |
| 1439 // Initialize per node info | |
| 1440 void SuperWord::initialize_bb() { | |
| 1441 Node* last = _block.at(_block.length() - 1); | |
| 1442 grow_node_info(bb_idx(last)); | |
| 1443 } | |
| 1444 | |
| 1445 //------------------------------bb_insert_after--------------------------- | |
| 1446 // Insert n into block after pos | |
| 1447 void SuperWord::bb_insert_after(Node* n, int pos) { | |
| 1448 int n_pos = pos + 1; | |
| 1449 // Make room | |
| 1450 for (int i = _block.length() - 1; i >= n_pos; i--) { | |
| 1451 _block.at_put_grow(i+1, _block.at(i)); | |
| 1452 } | |
| 1453 for (int j = _node_info.length() - 1; j >= n_pos; j--) { | |
| 1454 _node_info.at_put_grow(j+1, _node_info.at(j)); | |
| 1455 } | |
| 1456 // Set value | |
| 1457 _block.at_put_grow(n_pos, n); | |
| 1458 _node_info.at_put_grow(n_pos, SWNodeInfo::initial); | |
| 1459 // Adjust map from node->_idx to _block index | |
| 1460 for (int i = n_pos; i < _block.length(); i++) { | |
| 1461 set_bb_idx(_block.at(i), i); | |
| 1462 } | |
| 1463 } | |
| 1464 | |
| 1465 //------------------------------compute_max_depth--------------------------- | |
| 1466 // Compute max depth for expressions from beginning of block | |
| 1467 // Use to prune search paths during test for independence. | |
| 1468 void SuperWord::compute_max_depth() { | |
| 1469 int ct = 0; | |
| 1470 bool again; | |
| 1471 do { | |
| 1472 again = false; | |
| 1473 for (int i = 0; i < _block.length(); i++) { | |
| 1474 Node* n = _block.at(i); | |
| 1475 if (!n->is_Phi()) { | |
| 1476 int d_orig = depth(n); | |
| 1477 int d_in = 0; | |
| 1478 for (DepPreds preds(n, _dg); !preds.done(); preds.next()) { | |
| 1479 Node* pred = preds.current(); | |
| 1480 if (in_bb(pred)) { | |
| 1481 d_in = MAX2(d_in, depth(pred)); | |
| 1482 } | |
| 1483 } | |
| 1484 if (d_in + 1 != d_orig) { | |
| 1485 set_depth(n, d_in + 1); | |
| 1486 again = true; | |
| 1487 } | |
| 1488 } | |
| 1489 } | |
| 1490 ct++; | |
| 1491 } while (again); | |
| 1492 #ifndef PRODUCT | |
| 1493 if (TraceSuperWord && Verbose) | |
| 1494 tty->print_cr("compute_max_depth iterated: %d times", ct); | |
| 1495 #endif | |
| 1496 } | |
| 1497 | |
| 1498 //-------------------------compute_vector_element_type----------------------- | |
| 1499 // Compute necessary vector element type for expressions | |
| 1500 // This propagates backwards a narrower integer type when the | |
| 1501 // upper bits of the value are not needed. | |
| 1502 // Example: char a,b,c; a = b + c; | |
| 1503 // Normally the type of the add is integer, but for packed character | |
| 1504 // operations the type of the add needs to be char. | |
| 1505 void SuperWord::compute_vector_element_type() { | |
| 1506 #ifndef PRODUCT | |
| 1507 if (TraceSuperWord && Verbose) | |
| 1508 tty->print_cr("\ncompute_velt_type:"); | |
| 1509 #endif | |
| 1510 | |
| 1511 // Initial type | |
| 1512 for (int i = 0; i < _block.length(); i++) { | |
| 1513 Node* n = _block.at(i); | |
| 1514 const Type* t = n->is_Mem() ? Type::get_const_basic_type(n->as_Mem()->memory_type()) | |
| 1515 : _igvn.type(n); | |
| 1516 const Type* vt = container_type(t); | |
| 1517 set_velt_type(n, vt); | |
| 1518 } | |
| 1519 | |
| 1520 // Propagate narrowed type backwards through operations | |
| 1521 // that don't depend on higher order bits | |
| 1522 for (int i = _block.length() - 1; i >= 0; i--) { | |
| 1523 Node* n = _block.at(i); | |
| 1524 // Only integer types need be examined | |
| 1525 if (n->bottom_type()->isa_int()) { | |
| 1526 uint start, end; | |
| 1527 vector_opd_range(n, &start, &end); | |
| 1528 const Type* vt = velt_type(n); | |
| 1529 | |
| 1530 for (uint j = start; j < end; j++) { | |
| 1531 Node* in = n->in(j); | |
| 1532 // Don't propagate through a type conversion | |
| 1533 if (n->bottom_type() != in->bottom_type()) | |
| 1534 continue; | |
| 1535 switch(in->Opcode()) { | |
| 1536 case Op_AddI: case Op_AddL: | |
| 1537 case Op_SubI: case Op_SubL: | |
| 1538 case Op_MulI: case Op_MulL: | |
| 1539 case Op_AndI: case Op_AndL: | |
| 1540 case Op_OrI: case Op_OrL: | |
| 1541 case Op_XorI: case Op_XorL: | |
| 1542 case Op_LShiftI: case Op_LShiftL: | |
| 1543 case Op_CMoveI: case Op_CMoveL: | |
| 1544 if (in_bb(in)) { | |
| 1545 bool same_type = true; | |
| 1546 for (DUIterator_Fast kmax, k = in->fast_outs(kmax); k < kmax; k++) { | |
| 1547 Node *use = in->fast_out(k); | |
| 1548 if (!in_bb(use) || velt_type(use) != vt) { | |
| 1549 same_type = false; | |
| 1550 break; | |
| 1551 } | |
| 1552 } | |
| 1553 if (same_type) { | |
| 1554 set_velt_type(in, vt); | |
| 1555 } | |
| 1556 } | |
| 1557 } | |
| 1558 } | |
| 1559 } | |
| 1560 } | |
| 1561 #ifndef PRODUCT | |
| 1562 if (TraceSuperWord && Verbose) { | |
| 1563 for (int i = 0; i < _block.length(); i++) { | |
| 1564 Node* n = _block.at(i); | |
| 1565 velt_type(n)->dump(); | |
| 1566 tty->print("\t"); | |
| 1567 n->dump(); | |
| 1568 } | |
| 1569 } | |
| 1570 #endif | |
| 1571 } | |
| 1572 | |
| 1573 //------------------------------memory_alignment--------------------------- | |
| 1574 // Alignment within a vector memory reference | |
| 1575 int SuperWord::memory_alignment(MemNode* s, int iv_adjust_in_bytes) { | |
| 1576 SWPointer p(s, this); | |
| 1577 if (!p.valid()) { | |
| 1578 return bottom_align; | |
| 1579 } | |
| 1580 int offset = p.offset_in_bytes(); | |
| 1581 offset += iv_adjust_in_bytes; | |
| 1582 int off_rem = offset % vector_width_in_bytes(); | |
| 1583 int off_mod = off_rem >= 0 ? off_rem : off_rem + vector_width_in_bytes(); | |
| 1584 return off_mod; | |
| 1585 } | |
| 1586 | |
| 1587 //---------------------------container_type--------------------------- | |
| 1588 // Smallest type containing range of values | |
| 1589 const Type* SuperWord::container_type(const Type* t) { | |
|
221
1e026f8da827
6710487: More than half of JDI Regression tests hang with COOPs in -Xcomp mode
kvn
parents:
113
diff
changeset
|
1590 const Type* tp = t->make_ptr(); |
|
1e026f8da827
6710487: More than half of JDI Regression tests hang with COOPs in -Xcomp mode
kvn
parents:
113
diff
changeset
|
1591 if (tp && tp->isa_aryptr()) { |
|
1e026f8da827
6710487: More than half of JDI Regression tests hang with COOPs in -Xcomp mode
kvn
parents:
113
diff
changeset
|
1592 t = tp->is_aryptr()->elem(); |
| 0 | 1593 } |
| 1594 if (t->basic_type() == T_INT) { | |
| 1595 if (t->higher_equal(TypeInt::BOOL)) return TypeInt::BOOL; | |
| 1596 if (t->higher_equal(TypeInt::BYTE)) return TypeInt::BYTE; | |
| 1597 if (t->higher_equal(TypeInt::CHAR)) return TypeInt::CHAR; | |
| 1598 if (t->higher_equal(TypeInt::SHORT)) return TypeInt::SHORT; | |
| 1599 return TypeInt::INT; | |
| 1600 } | |
| 1601 return t; | |
| 1602 } | |
| 1603 | |
| 1604 //-------------------------vector_opd_range----------------------- | |
| 1605 // (Start, end] half-open range defining which operands are vector | |
| 1606 void SuperWord::vector_opd_range(Node* n, uint* start, uint* end) { | |
| 1607 switch (n->Opcode()) { | |
|
558
3b5ac9e7e6ea
6796746: rename LoadC (char) opcode class to LoadUS (unsigned short)
twisti
parents:
253
diff
changeset
|
1608 case Op_LoadB: case Op_LoadUS: |
| 0 | 1609 case Op_LoadI: case Op_LoadL: |
| 1610 case Op_LoadF: case Op_LoadD: | |
| 1611 case Op_LoadP: | |
| 1612 *start = 0; | |
| 1613 *end = 0; | |
| 1614 return; | |
| 1615 case Op_StoreB: case Op_StoreC: | |
| 1616 case Op_StoreI: case Op_StoreL: | |
| 1617 case Op_StoreF: case Op_StoreD: | |
| 1618 case Op_StoreP: | |
| 1619 *start = MemNode::ValueIn; | |
| 1620 *end = *start + 1; | |
| 1621 return; | |
| 1622 case Op_LShiftI: case Op_LShiftL: | |
| 1623 *start = 1; | |
| 1624 *end = 2; | |
| 1625 return; | |
| 1626 case Op_CMoveI: case Op_CMoveL: case Op_CMoveF: case Op_CMoveD: | |
| 1627 *start = 2; | |
| 1628 *end = n->req(); | |
| 1629 return; | |
| 1630 } | |
| 1631 *start = 1; | |
| 1632 *end = n->req(); // default is all operands | |
| 1633 } | |
| 1634 | |
| 1635 //------------------------------in_packset--------------------------- | |
| 1636 // Are s1 and s2 in a pack pair and ordered as s1,s2? | |
| 1637 bool SuperWord::in_packset(Node* s1, Node* s2) { | |
| 1638 for (int i = 0; i < _packset.length(); i++) { | |
| 1639 Node_List* p = _packset.at(i); | |
| 1640 assert(p->size() == 2, "must be"); | |
| 1641 if (p->at(0) == s1 && p->at(p->size()-1) == s2) { | |
| 1642 return true; | |
| 1643 } | |
| 1644 } | |
| 1645 return false; | |
| 1646 } | |
| 1647 | |
| 1648 //------------------------------in_pack--------------------------- | |
| 1649 // Is s in pack p? | |
| 1650 Node_List* SuperWord::in_pack(Node* s, Node_List* p) { | |
| 1651 for (uint i = 0; i < p->size(); i++) { | |
| 1652 if (p->at(i) == s) { | |
| 1653 return p; | |
| 1654 } | |
| 1655 } | |
| 1656 return NULL; | |
| 1657 } | |
| 1658 | |
| 1659 //------------------------------remove_pack_at--------------------------- | |
| 1660 // Remove the pack at position pos in the packset | |
| 1661 void SuperWord::remove_pack_at(int pos) { | |
| 1662 Node_List* p = _packset.at(pos); | |
| 1663 for (uint i = 0; i < p->size(); i++) { | |
| 1664 Node* s = p->at(i); | |
| 1665 set_my_pack(s, NULL); | |
| 1666 } | |
| 1667 _packset.remove_at(pos); | |
| 1668 } | |
| 1669 | |
| 1670 //------------------------------executed_first--------------------------- | |
| 1671 // Return the node executed first in pack p. Uses the RPO block list | |
| 1672 // to determine order. | |
| 1673 Node* SuperWord::executed_first(Node_List* p) { | |
| 1674 Node* n = p->at(0); | |
| 1675 int n_rpo = bb_idx(n); | |
| 1676 for (uint i = 1; i < p->size(); i++) { | |
| 1677 Node* s = p->at(i); | |
| 1678 int s_rpo = bb_idx(s); | |
| 1679 if (s_rpo < n_rpo) { | |
| 1680 n = s; | |
| 1681 n_rpo = s_rpo; | |
| 1682 } | |
| 1683 } | |
| 1684 return n; | |
| 1685 } | |
| 1686 | |
| 1687 //------------------------------executed_last--------------------------- | |
| 1688 // Return the node executed last in pack p. | |
| 1689 Node* SuperWord::executed_last(Node_List* p) { | |
| 1690 Node* n = p->at(0); | |
| 1691 int n_rpo = bb_idx(n); | |
| 1692 for (uint i = 1; i < p->size(); i++) { | |
| 1693 Node* s = p->at(i); | |
| 1694 int s_rpo = bb_idx(s); | |
| 1695 if (s_rpo > n_rpo) { | |
| 1696 n = s; | |
| 1697 n_rpo = s_rpo; | |
| 1698 } | |
| 1699 } | |
| 1700 return n; | |
| 1701 } | |
| 1702 | |
| 1703 //----------------------------align_initial_loop_index--------------------------- | |
| 1704 // Adjust pre-loop limit so that in main loop, a load/store reference | |
| 1705 // to align_to_ref will be a position zero in the vector. | |
| 1706 // (iv + k) mod vector_align == 0 | |
| 1707 void SuperWord::align_initial_loop_index(MemNode* align_to_ref) { | |
| 1708 CountedLoopNode *main_head = lp()->as_CountedLoop(); | |
| 1709 assert(main_head->is_main_loop(), ""); | |
| 1710 CountedLoopEndNode* pre_end = get_pre_loop_end(main_head); | |
| 1711 assert(pre_end != NULL, ""); | |
| 1712 Node *pre_opaq1 = pre_end->limit(); | |
| 1713 assert(pre_opaq1->Opcode() == Op_Opaque1, ""); | |
| 1714 Opaque1Node *pre_opaq = (Opaque1Node*)pre_opaq1; | |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1715 Node *lim0 = pre_opaq->in(1); |
| 0 | 1716 |
| 1717 // Where we put new limit calculations | |
| 1718 Node *pre_ctrl = pre_end->loopnode()->in(LoopNode::EntryControl); | |
| 1719 | |
| 1720 // Ensure the original loop limit is available from the | |
| 1721 // pre-loop Opaque1 node. | |
| 1722 Node *orig_limit = pre_opaq->original_loop_limit(); | |
| 1723 assert(orig_limit != NULL && _igvn.type(orig_limit) != Type::TOP, ""); | |
| 1724 | |
| 1725 SWPointer align_to_ref_p(align_to_ref, this); | |
| 1726 | |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1727 // Given: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1728 // lim0 == original pre loop limit |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1729 // V == v_align (power of 2) |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1730 // invar == extra invariant piece of the address expression |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1731 // e == k [ +/- invar ] |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1732 // |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1733 // When reassociating expressions involving '%' the basic rules are: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1734 // (a - b) % k == 0 => a % k == b % k |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1735 // and: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1736 // (a + b) % k == 0 => a % k == (k - b) % k |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1737 // |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1738 // For stride > 0 && scale > 0, |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1739 // Derive the new pre-loop limit "lim" such that the two constraints: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1740 // (1) lim = lim0 + N (where N is some positive integer < V) |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1741 // (2) (e + lim) % V == 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1742 // are true. |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1743 // |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1744 // Substituting (1) into (2), |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1745 // (e + lim0 + N) % V == 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1746 // solve for N: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1747 // N = (V - (e + lim0)) % V |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1748 // substitute back into (1), so that new limit |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1749 // lim = lim0 + (V - (e + lim0)) % V |
| 0 | 1750 // |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1751 // For stride > 0 && scale < 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1752 // Constraints: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1753 // lim = lim0 + N |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1754 // (e - lim) % V == 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1755 // Solving for lim: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1756 // (e - lim0 - N) % V == 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1757 // N = (e - lim0) % V |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1758 // lim = lim0 + (e - lim0) % V |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1759 // |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1760 // For stride < 0 && scale > 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1761 // Constraints: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1762 // lim = lim0 - N |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1763 // (e + lim) % V == 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1764 // Solving for lim: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1765 // (e + lim0 - N) % V == 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1766 // N = (e + lim0) % V |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1767 // lim = lim0 - (e + lim0) % V |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1768 // |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1769 // For stride < 0 && scale < 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1770 // Constraints: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1771 // lim = lim0 - N |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1772 // (e - lim) % V == 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1773 // Solving for lim: |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1774 // (e - lim0 + N) % V == 0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1775 // N = (V - (e - lim0)) % V |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1776 // lim = lim0 - (V - (e - lim0)) % V |
| 0 | 1777 |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1778 int stride = iv_stride(); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1779 int scale = align_to_ref_p.scale_in_bytes(); |
| 0 | 1780 int elt_size = align_to_ref_p.memory_size(); |
| 1781 int v_align = vector_width_in_bytes() / elt_size; | |
| 1782 int k = align_to_ref_p.offset_in_bytes() / elt_size; | |
| 1783 | |
| 1784 Node *kn = _igvn.intcon(k); | |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1785 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1786 Node *e = kn; |
| 0 | 1787 if (align_to_ref_p.invar() != NULL) { |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1788 // incorporate any extra invariant piece producing k +/- invar >>> log2(elt) |
| 0 | 1789 Node* log2_elt = _igvn.intcon(exact_log2(elt_size)); |
| 1790 Node* aref = new (_phase->C, 3) URShiftINode(align_to_ref_p.invar(), log2_elt); | |
| 1791 _phase->_igvn.register_new_node_with_optimizer(aref); | |
| 1792 _phase->set_ctrl(aref, pre_ctrl); | |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1793 if (align_to_ref_p.negate_invar()) { |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1794 e = new (_phase->C, 3) SubINode(e, aref); |
| 0 | 1795 } else { |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1796 e = new (_phase->C, 3) AddINode(e, aref); |
| 0 | 1797 } |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1798 _phase->_igvn.register_new_node_with_optimizer(e); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1799 _phase->set_ctrl(e, pre_ctrl); |
| 0 | 1800 } |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1801 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1802 // compute e +/- lim0 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1803 if (scale < 0) { |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1804 e = new (_phase->C, 3) SubINode(e, lim0); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1805 } else { |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1806 e = new (_phase->C, 3) AddINode(e, lim0); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1807 } |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1808 _phase->_igvn.register_new_node_with_optimizer(e); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1809 _phase->set_ctrl(e, pre_ctrl); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1810 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1811 if (stride * scale > 0) { |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1812 // compute V - (e +/- lim0) |
| 0 | 1813 Node* va = _igvn.intcon(v_align); |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1814 e = new (_phase->C, 3) SubINode(va, e); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1815 _phase->_igvn.register_new_node_with_optimizer(e); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1816 _phase->set_ctrl(e, pre_ctrl); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1817 } |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1818 // compute N = (exp) % V |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1819 Node* va_msk = _igvn.intcon(v_align - 1); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1820 Node* N = new (_phase->C, 3) AndINode(e, va_msk); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1821 _phase->_igvn.register_new_node_with_optimizer(N); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1822 _phase->set_ctrl(N, pre_ctrl); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1823 |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1824 // substitute back into (1), so that new limit |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1825 // lim = lim0 + N |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1826 Node* lim; |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1827 if (stride < 0) { |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1828 lim = new (_phase->C, 3) SubINode(lim0, N); |
| 0 | 1829 } else { |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1830 lim = new (_phase->C, 3) AddINode(lim0, N); |
| 0 | 1831 } |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1832 _phase->_igvn.register_new_node_with_optimizer(lim); |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1833 _phase->set_ctrl(lim, pre_ctrl); |
| 0 | 1834 Node* constrained = |
|
72
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1835 (stride > 0) ? (Node*) new (_phase->C,3) MinINode(lim, orig_limit) |
|
f705f25597eb
6663621: JVM crashes while trying to execute api/java_security/Signature/SignatureTests.html#initSign tests.
never
parents:
29
diff
changeset
|
1836 : (Node*) new (_phase->C,3) MaxINode(lim, orig_limit); |
| 0 | 1837 _phase->_igvn.register_new_node_with_optimizer(constrained); |
| 1838 _phase->set_ctrl(constrained, pre_ctrl); | |
| 1839 _igvn.hash_delete(pre_opaq); | |
| 1840 pre_opaq->set_req(1, constrained); | |
| 1841 } | |
| 1842 | |
| 1843 //----------------------------get_pre_loop_end--------------------------- | |
| 1844 // Find pre loop end from main loop. Returns null if none. | |
| 1845 CountedLoopEndNode* SuperWord::get_pre_loop_end(CountedLoopNode *cl) { | |
| 1846 Node *ctrl = cl->in(LoopNode::EntryControl); | |
| 1847 if (!ctrl->is_IfTrue() && !ctrl->is_IfFalse()) return NULL; | |
| 1848 Node *iffm = ctrl->in(0); | |
| 1849 if (!iffm->is_If()) return NULL; | |
| 1850 Node *p_f = iffm->in(0); | |
| 1851 if (!p_f->is_IfFalse()) return NULL; | |
| 1852 if (!p_f->in(0)->is_CountedLoopEnd()) return NULL; | |
| 1853 CountedLoopEndNode *pre_end = p_f->in(0)->as_CountedLoopEnd(); | |
| 1854 if (!pre_end->loopnode()->is_pre_loop()) return NULL; | |
| 1855 return pre_end; | |
| 1856 } | |
| 1857 | |
| 1858 | |
| 1859 //------------------------------init--------------------------- | |
| 1860 void SuperWord::init() { | |
| 1861 _dg.init(); | |
| 1862 _packset.clear(); | |
| 1863 _disjoint_ptrs.clear(); | |
| 1864 _block.clear(); | |
| 1865 _data_entry.clear(); | |
| 1866 _mem_slice_head.clear(); | |
| 1867 _mem_slice_tail.clear(); | |
| 1868 _node_info.clear(); | |
| 1869 _align_to_ref = NULL; | |
| 1870 _lpt = NULL; | |
| 1871 _lp = NULL; | |
| 1872 _bb = NULL; | |
| 1873 _iv = NULL; | |
| 1874 } | |
| 1875 | |
| 1876 //------------------------------print_packset--------------------------- | |
| 1877 void SuperWord::print_packset() { | |
| 1878 #ifndef PRODUCT | |
| 1879 tty->print_cr("packset"); | |
| 1880 for (int i = 0; i < _packset.length(); i++) { | |
| 1881 tty->print_cr("Pack: %d", i); | |
| 1882 Node_List* p = _packset.at(i); | |
| 1883 print_pack(p); | |
| 1884 } | |
| 1885 #endif | |
| 1886 } | |
| 1887 | |
| 1888 //------------------------------print_pack--------------------------- | |
| 1889 void SuperWord::print_pack(Node_List* p) { | |
| 1890 for (uint i = 0; i < p->size(); i++) { | |
| 1891 print_stmt(p->at(i)); | |
| 1892 } | |
| 1893 } | |
| 1894 | |
| 1895 //------------------------------print_bb--------------------------- | |
| 1896 void SuperWord::print_bb() { | |
| 1897 #ifndef PRODUCT | |
| 1898 tty->print_cr("\nBlock"); | |
| 1899 for (int i = 0; i < _block.length(); i++) { | |
| 1900 Node* n = _block.at(i); | |
| 1901 tty->print("%d ", i); | |
| 1902 if (n) { | |
| 1903 n->dump(); | |
| 1904 } | |
| 1905 } | |
| 1906 #endif | |
| 1907 } | |
| 1908 | |
| 1909 //------------------------------print_stmt--------------------------- | |
| 1910 void SuperWord::print_stmt(Node* s) { | |
| 1911 #ifndef PRODUCT | |
| 1912 tty->print(" align: %d \t", alignment(s)); | |
| 1913 s->dump(); | |
| 1914 #endif | |
| 1915 } | |
| 1916 | |
| 1917 //------------------------------blank--------------------------- | |
| 1918 char* SuperWord::blank(uint depth) { | |
| 1919 static char blanks[101]; | |
| 1920 assert(depth < 101, "too deep"); | |
| 1921 for (uint i = 0; i < depth; i++) blanks[i] = ' '; | |
| 1922 blanks[depth] = '\0'; | |
| 1923 return blanks; | |
| 1924 } | |
| 1925 | |
| 1926 | |
| 1927 //==============================SWPointer=========================== | |
| 1928 | |
| 1929 //----------------------------SWPointer------------------------ | |
| 1930 SWPointer::SWPointer(MemNode* mem, SuperWord* slp) : | |
| 1931 _mem(mem), _slp(slp), _base(NULL), _adr(NULL), | |
| 1932 _scale(0), _offset(0), _invar(NULL), _negate_invar(false) { | |
| 1933 | |
| 1934 Node* adr = mem->in(MemNode::Address); | |
| 1935 if (!adr->is_AddP()) { | |
| 1936 assert(!valid(), "too complex"); | |
| 1937 return; | |
| 1938 } | |
| 1939 // Match AddP(base, AddP(ptr, k*iv [+ invariant]), constant) | |
| 1940 Node* base = adr->in(AddPNode::Base); | |
|
1058
73a726751507
6852078: HSX 14/16 in jdk 5.0: api/javax_management api/org_omg jck tests crashes or make tnameserv crash
cfang
parents:
987
diff
changeset
|
1941 //unsafe reference could not be aligned appropriately without runtime checking |
|
73a726751507
6852078: HSX 14/16 in jdk 5.0: api/javax_management api/org_omg jck tests crashes or make tnameserv crash
cfang
parents:
987
diff
changeset
|
1942 if (base == NULL || base->bottom_type() == Type::TOP) { |
|
73a726751507
6852078: HSX 14/16 in jdk 5.0: api/javax_management api/org_omg jck tests crashes or make tnameserv crash
cfang
parents:
987
diff
changeset
|
1943 assert(!valid(), "unsafe access"); |
|
73a726751507
6852078: HSX 14/16 in jdk 5.0: api/javax_management api/org_omg jck tests crashes or make tnameserv crash
cfang
parents:
987
diff
changeset
|
1944 return; |
|
73a726751507
6852078: HSX 14/16 in jdk 5.0: api/javax_management api/org_omg jck tests crashes or make tnameserv crash
cfang
parents:
987
diff
changeset
|
1945 } |
| 0 | 1946 for (int i = 0; i < 3; i++) { |
| 1947 if (!scaled_iv_plus_offset(adr->in(AddPNode::Offset))) { | |
| 1948 assert(!valid(), "too complex"); | |
| 1949 return; | |
| 1950 } | |
| 1951 adr = adr->in(AddPNode::Address); | |
| 1952 if (base == adr || !adr->is_AddP()) { | |
| 1953 break; // stop looking at addp's | |
| 1954 } | |
| 1955 } | |
| 1956 _base = base; | |
| 1957 _adr = adr; | |
| 1958 assert(valid(), "Usable"); | |
| 1959 } | |
| 1960 | |
| 1961 // Following is used to create a temporary object during | |
| 1962 // the pattern match of an address expression. | |
| 1963 SWPointer::SWPointer(SWPointer* p) : | |
| 1964 _mem(p->_mem), _slp(p->_slp), _base(NULL), _adr(NULL), | |
| 1965 _scale(0), _offset(0), _invar(NULL), _negate_invar(false) {} | |
| 1966 | |
| 1967 //------------------------scaled_iv_plus_offset-------------------- | |
| 1968 // Match: k*iv + offset | |
| 1969 // where: k is a constant that maybe zero, and | |
| 1970 // offset is (k2 [+/- invariant]) where k2 maybe zero and invariant is optional | |
| 1971 bool SWPointer::scaled_iv_plus_offset(Node* n) { | |
| 1972 if (scaled_iv(n)) { | |
| 1973 return true; | |
| 1974 } | |
| 1975 if (offset_plus_k(n)) { | |
| 1976 return true; | |
| 1977 } | |
| 1978 int opc = n->Opcode(); | |
| 1979 if (opc == Op_AddI) { | |
| 1980 if (scaled_iv(n->in(1)) && offset_plus_k(n->in(2))) { | |
| 1981 return true; | |
| 1982 } | |
| 1983 if (scaled_iv(n->in(2)) && offset_plus_k(n->in(1))) { | |
| 1984 return true; | |
| 1985 } | |
| 1986 } else if (opc == Op_SubI) { | |
| 1987 if (scaled_iv(n->in(1)) && offset_plus_k(n->in(2), true)) { | |
| 1988 return true; | |
| 1989 } | |
| 1990 if (scaled_iv(n->in(2)) && offset_plus_k(n->in(1))) { | |
| 1991 _scale *= -1; | |
| 1992 return true; | |
| 1993 } | |
| 1994 } | |
| 1995 return false; | |
| 1996 } | |
| 1997 | |
| 1998 //----------------------------scaled_iv------------------------ | |
| 1999 // Match: k*iv where k is a constant that's not zero | |
| 2000 bool SWPointer::scaled_iv(Node* n) { | |
| 2001 if (_scale != 0) { | |
| 2002 return false; // already found a scale | |
| 2003 } | |
| 2004 if (n == iv()) { | |
| 2005 _scale = 1; | |
| 2006 return true; | |
| 2007 } | |
| 2008 int opc = n->Opcode(); | |
| 2009 if (opc == Op_MulI) { | |
| 2010 if (n->in(1) == iv() && n->in(2)->is_Con()) { | |
| 2011 _scale = n->in(2)->get_int(); | |
| 2012 return true; | |
| 2013 } else if (n->in(2) == iv() && n->in(1)->is_Con()) { | |
| 2014 _scale = n->in(1)->get_int(); | |
| 2015 return true; | |
| 2016 } | |
| 2017 } else if (opc == Op_LShiftI) { | |
| 2018 if (n->in(1) == iv() && n->in(2)->is_Con()) { | |
| 2019 _scale = 1 << n->in(2)->get_int(); | |
| 2020 return true; | |
| 2021 } | |
| 2022 } else if (opc == Op_ConvI2L) { | |
| 2023 if (scaled_iv_plus_offset(n->in(1))) { | |
| 2024 return true; | |
| 2025 } | |
| 2026 } else if (opc == Op_LShiftL) { | |
| 2027 if (!has_iv() && _invar == NULL) { | |
| 2028 // Need to preserve the current _offset value, so | |
| 2029 // create a temporary object for this expression subtree. | |
| 2030 // Hacky, so should re-engineer the address pattern match. | |
| 2031 SWPointer tmp(this); | |
| 2032 if (tmp.scaled_iv_plus_offset(n->in(1))) { | |
| 2033 if (tmp._invar == NULL) { | |
| 2034 int mult = 1 << n->in(2)->get_int(); | |
| 2035 _scale = tmp._scale * mult; | |
| 2036 _offset += tmp._offset * mult; | |
| 2037 return true; | |
| 2038 } | |
| 2039 } | |
| 2040 } | |
| 2041 } | |
| 2042 return false; | |
| 2043 } | |
| 2044 | |
| 2045 //----------------------------offset_plus_k------------------------ | |
| 2046 // Match: offset is (k [+/- invariant]) | |
| 2047 // where k maybe zero and invariant is optional, but not both. | |
| 2048 bool SWPointer::offset_plus_k(Node* n, bool negate) { | |
| 2049 int opc = n->Opcode(); | |
| 2050 if (opc == Op_ConI) { | |
| 2051 _offset += negate ? -(n->get_int()) : n->get_int(); | |
| 2052 return true; | |
| 2053 } else if (opc == Op_ConL) { | |
| 2054 // Okay if value fits into an int | |
| 2055 const TypeLong* t = n->find_long_type(); | |
| 2056 if (t->higher_equal(TypeLong::INT)) { | |
| 2057 jlong loff = n->get_long(); | |
| 2058 jint off = (jint)loff; | |
| 2059 _offset += negate ? -off : loff; | |
| 2060 return true; | |
| 2061 } | |
| 2062 return false; | |
| 2063 } | |
| 2064 if (_invar != NULL) return false; // already have an invariant | |
| 2065 if (opc == Op_AddI) { | |
| 2066 if (n->in(2)->is_Con() && invariant(n->in(1))) { | |
| 2067 _negate_invar = negate; | |
| 2068 _invar = n->in(1); | |
| 2069 _offset += negate ? -(n->in(2)->get_int()) : n->in(2)->get_int(); | |
| 2070 return true; | |
| 2071 } else if (n->in(1)->is_Con() && invariant(n->in(2))) { | |
| 2072 _offset += negate ? -(n->in(1)->get_int()) : n->in(1)->get_int(); | |
| 2073 _negate_invar = negate; | |
| 2074 _invar = n->in(2); | |
| 2075 return true; | |
| 2076 } | |
| 2077 } | |
| 2078 if (opc == Op_SubI) { | |
| 2079 if (n->in(2)->is_Con() && invariant(n->in(1))) { | |
| 2080 _negate_invar = negate; | |
| 2081 _invar = n->in(1); | |
| 2082 _offset += !negate ? -(n->in(2)->get_int()) : n->in(2)->get_int(); | |
| 2083 return true; | |
| 2084 } else if (n->in(1)->is_Con() && invariant(n->in(2))) { | |
| 2085 _offset += negate ? -(n->in(1)->get_int()) : n->in(1)->get_int(); | |
| 2086 _negate_invar = !negate; | |
| 2087 _invar = n->in(2); | |
| 2088 return true; | |
| 2089 } | |
| 2090 } | |
| 2091 if (invariant(n)) { | |
| 2092 _negate_invar = negate; | |
| 2093 _invar = n; | |
| 2094 return true; | |
| 2095 } | |
| 2096 return false; | |
| 2097 } | |
| 2098 | |
| 2099 //----------------------------print------------------------ | |
| 2100 void SWPointer::print() { | |
| 2101 #ifndef PRODUCT | |
| 2102 tty->print("base: %d adr: %d scale: %d offset: %d invar: %c%d\n", | |
| 2103 _base != NULL ? _base->_idx : 0, | |
| 2104 _adr != NULL ? _adr->_idx : 0, | |
| 2105 _scale, _offset, | |
| 2106 _negate_invar?'-':'+', | |
| 2107 _invar != NULL ? _invar->_idx : 0); | |
| 2108 #endif | |
| 2109 } | |
| 2110 | |
| 2111 // ========================= OrderedPair ===================== | |
| 2112 | |
| 2113 const OrderedPair OrderedPair::initial; | |
| 2114 | |
| 2115 // ========================= SWNodeInfo ===================== | |
| 2116 | |
| 2117 const SWNodeInfo SWNodeInfo::initial; | |
| 2118 | |
| 2119 | |
| 2120 // ============================ DepGraph =========================== | |
| 2121 | |
| 2122 //------------------------------make_node--------------------------- | |
| 2123 // Make a new dependence graph node for an ideal node. | |
| 2124 DepMem* DepGraph::make_node(Node* node) { | |
| 2125 DepMem* m = new (_arena) DepMem(node); | |
| 2126 if (node != NULL) { | |
| 2127 assert(_map.at_grow(node->_idx) == NULL, "one init only"); | |
| 2128 _map.at_put_grow(node->_idx, m); | |
| 2129 } | |
| 2130 return m; | |
| 2131 } | |
| 2132 | |
| 2133 //------------------------------make_edge--------------------------- | |
| 2134 // Make a new dependence graph edge from dpred -> dsucc | |
| 2135 DepEdge* DepGraph::make_edge(DepMem* dpred, DepMem* dsucc) { | |
| 2136 DepEdge* e = new (_arena) DepEdge(dpred, dsucc, dsucc->in_head(), dpred->out_head()); | |
| 2137 dpred->set_out_head(e); | |
| 2138 dsucc->set_in_head(e); | |
| 2139 return e; | |
| 2140 } | |
| 2141 | |
| 2142 // ========================== DepMem ======================== | |
| 2143 | |
| 2144 //------------------------------in_cnt--------------------------- | |
| 2145 int DepMem::in_cnt() { | |
| 2146 int ct = 0; | |
| 2147 for (DepEdge* e = _in_head; e != NULL; e = e->next_in()) ct++; | |
| 2148 return ct; | |
| 2149 } | |
| 2150 | |
| 2151 //------------------------------out_cnt--------------------------- | |
| 2152 int DepMem::out_cnt() { | |
| 2153 int ct = 0; | |
| 2154 for (DepEdge* e = _out_head; e != NULL; e = e->next_out()) ct++; | |
| 2155 return ct; | |
| 2156 } | |
| 2157 | |
| 2158 //------------------------------print----------------------------- | |
| 2159 void DepMem::print() { | |
| 2160 #ifndef PRODUCT | |
| 2161 tty->print(" DepNode %d (", _node->_idx); | |
| 2162 for (DepEdge* p = _in_head; p != NULL; p = p->next_in()) { | |
| 2163 Node* pred = p->pred()->node(); | |
| 2164 tty->print(" %d", pred != NULL ? pred->_idx : 0); | |
| 2165 } | |
| 2166 tty->print(") ["); | |
| 2167 for (DepEdge* s = _out_head; s != NULL; s = s->next_out()) { | |
| 2168 Node* succ = s->succ()->node(); | |
| 2169 tty->print(" %d", succ != NULL ? succ->_idx : 0); | |
| 2170 } | |
| 2171 tty->print_cr(" ]"); | |
| 2172 #endif | |
| 2173 } | |
| 2174 | |
| 2175 // =========================== DepEdge ========================= | |
| 2176 | |
| 2177 //------------------------------DepPreds--------------------------- | |
| 2178 void DepEdge::print() { | |
| 2179 #ifndef PRODUCT | |
| 2180 tty->print_cr("DepEdge: %d [ %d ]", _pred->node()->_idx, _succ->node()->_idx); | |
| 2181 #endif | |
| 2182 } | |
| 2183 | |
| 2184 // =========================== DepPreds ========================= | |
| 2185 // Iterator over predecessor edges in the dependence graph. | |
| 2186 | |
| 2187 //------------------------------DepPreds--------------------------- | |
| 2188 DepPreds::DepPreds(Node* n, DepGraph& dg) { | |
| 2189 _n = n; | |
| 2190 _done = false; | |
| 2191 if (_n->is_Store() || _n->is_Load()) { | |
| 2192 _next_idx = MemNode::Address; | |
| 2193 _end_idx = n->req(); | |
| 2194 _dep_next = dg.dep(_n)->in_head(); | |
| 2195 } else if (_n->is_Mem()) { | |
| 2196 _next_idx = 0; | |
| 2197 _end_idx = 0; | |
| 2198 _dep_next = dg.dep(_n)->in_head(); | |
| 2199 } else { | |
| 2200 _next_idx = 1; | |
| 2201 _end_idx = _n->req(); | |
| 2202 _dep_next = NULL; | |
| 2203 } | |
| 2204 next(); | |
| 2205 } | |
| 2206 | |
| 2207 //------------------------------next--------------------------- | |
| 2208 void DepPreds::next() { | |
| 2209 if (_dep_next != NULL) { | |
| 2210 _current = _dep_next->pred()->node(); | |
| 2211 _dep_next = _dep_next->next_in(); | |
| 2212 } else if (_next_idx < _end_idx) { | |
| 2213 _current = _n->in(_next_idx++); | |
| 2214 } else { | |
| 2215 _done = true; | |
| 2216 } | |
| 2217 } | |
| 2218 | |
| 2219 // =========================== DepSuccs ========================= | |
| 2220 // Iterator over successor edges in the dependence graph. | |
| 2221 | |
| 2222 //------------------------------DepSuccs--------------------------- | |
| 2223 DepSuccs::DepSuccs(Node* n, DepGraph& dg) { | |
| 2224 _n = n; | |
| 2225 _done = false; | |
| 2226 if (_n->is_Load()) { | |
| 2227 _next_idx = 0; | |
| 2228 _end_idx = _n->outcnt(); | |
| 2229 _dep_next = dg.dep(_n)->out_head(); | |
| 2230 } else if (_n->is_Mem() || _n->is_Phi() && _n->bottom_type() == Type::MEMORY) { | |
| 2231 _next_idx = 0; | |
| 2232 _end_idx = 0; | |
| 2233 _dep_next = dg.dep(_n)->out_head(); | |
| 2234 } else { | |
| 2235 _next_idx = 0; | |
| 2236 _end_idx = _n->outcnt(); | |
| 2237 _dep_next = NULL; | |
| 2238 } | |
| 2239 next(); | |
| 2240 } | |
| 2241 | |
| 2242 //-------------------------------next--------------------------- | |
| 2243 void DepSuccs::next() { | |
| 2244 if (_dep_next != NULL) { | |
| 2245 _current = _dep_next->succ()->node(); | |
| 2246 _dep_next = _dep_next->next_out(); | |
| 2247 } else if (_next_idx < _end_idx) { | |
| 2248 _current = _n->raw_out(_next_idx++); | |
| 2249 } else { | |
| 2250 _done = true; | |
| 2251 } | |
| 2252 } |