Mercurial > hg > truffle
annotate src/share/vm/c1/c1_LinearScan.cpp @ 1994:6cd6d394f280
7001033: assert(gch->gc_cause() == GCCause::_scavenge_alot || !gch->incremental_collection_failed())
7002546: regression on SpecJbb2005 on 7b118 comparing to 7b117 on small heaps
Summary: Relaxed assertion checking related to incremental_collection_failed flag to allow for ExplicitGCInvokesConcurrent behaviour where we do not want a failing scavenge to bail to a stop-world collection. Parameterized incremental_collection_will_fail() so we can selectively use, or not use, as appropriate, the statistical prediction at specific use sites. This essentially reverts the scavenge bail-out logic to what it was prior to some recent changes that had inadvertently started using the statistical prediction which can be noisy in the presence of bursty loads. Added some associated verbose non-product debugging messages.
Reviewed-by: johnc, tonyp
| author | ysr |
|---|---|
| date | Tue, 07 Dec 2010 21:55:53 -0800 |
| parents | f95d63e2154a |
| children | ac637b7220d1 |
| rev | line source |
|---|---|
| 0 | 1 /* |
|
1552
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1397
diff
changeset
|
2 * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. |
| 0 | 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| 4 * | |
| 5 * This code is free software; you can redistribute it and/or modify it | |
| 6 * under the terms of the GNU General Public License version 2 only, as | |
| 7 * published by the Free Software Foundation. | |
| 8 * | |
| 9 * This code is distributed in the hope that it will be useful, but WITHOUT | |
| 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
| 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
| 12 * version 2 for more details (a copy is included in the LICENSE file that | |
| 13 * accompanied this code). | |
| 14 * | |
| 15 * You should have received a copy of the GNU General Public License version | |
| 16 * 2 along with this work; if not, write to the Free Software Foundation, | |
| 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |
| 18 * | |
|
1552
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1397
diff
changeset
|
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
|
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1397
diff
changeset
|
20 * or visit www.oracle.com if you need additional information or have any |
|
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1397
diff
changeset
|
21 * questions. |
| 0 | 22 * |
| 23 */ | |
| 24 | |
| 1972 | 25 #include "precompiled.hpp" |
| 26 #include "c1/c1_CFGPrinter.hpp" | |
| 27 #include "c1/c1_CodeStubs.hpp" | |
| 28 #include "c1/c1_Compilation.hpp" | |
| 29 #include "c1/c1_FrameMap.hpp" | |
| 30 #include "c1/c1_IR.hpp" | |
| 31 #include "c1/c1_LIRGenerator.hpp" | |
| 32 #include "c1/c1_LinearScan.hpp" | |
| 33 #include "c1/c1_ValueStack.hpp" | |
| 34 #include "utilities/bitMap.inline.hpp" | |
| 35 #ifdef TARGET_ARCH_x86 | |
| 36 # include "vmreg_x86.inline.hpp" | |
| 37 #endif | |
| 38 #ifdef TARGET_ARCH_sparc | |
| 39 # include "vmreg_sparc.inline.hpp" | |
| 40 #endif | |
| 41 #ifdef TARGET_ARCH_zero | |
| 42 # include "vmreg_zero.inline.hpp" | |
| 43 #endif | |
| 0 | 44 |
| 45 | |
| 46 #ifndef PRODUCT | |
| 47 | |
| 48 static LinearScanStatistic _stat_before_alloc; | |
| 49 static LinearScanStatistic _stat_after_asign; | |
| 50 static LinearScanStatistic _stat_final; | |
| 51 | |
| 52 static LinearScanTimers _total_timer; | |
| 53 | |
| 54 // helper macro for short definition of timer | |
| 55 #define TIME_LINEAR_SCAN(timer_name) TraceTime _block_timer("", _total_timer.timer(LinearScanTimers::timer_name), TimeLinearScan || TimeEachLinearScan, Verbose); | |
| 56 | |
| 57 // helper macro for short definition of trace-output inside code | |
| 58 #define TRACE_LINEAR_SCAN(level, code) \ | |
| 59 if (TraceLinearScanLevel >= level) { \ | |
| 60 code; \ | |
| 61 } | |
| 62 | |
| 63 #else | |
| 64 | |
| 65 #define TIME_LINEAR_SCAN(timer_name) | |
| 66 #define TRACE_LINEAR_SCAN(level, code) | |
| 67 | |
| 68 #endif | |
| 69 | |
| 70 // Map BasicType to spill size in 32-bit words, matching VMReg's notion of words | |
| 71 #ifdef _LP64 | |
| 72 static int type2spill_size[T_CONFLICT+1]={ -1, 0, 0, 0, 1, 1, 1, 2, 1, 1, 1, 2, 2, 2, 0, 1, -1}; | |
| 73 #else | |
| 74 static int type2spill_size[T_CONFLICT+1]={ -1, 0, 0, 0, 1, 1, 1, 2, 1, 1, 1, 2, 1, 1, 0, 1, -1}; | |
| 75 #endif | |
| 76 | |
| 77 | |
| 78 // Implementation of LinearScan | |
| 79 | |
| 80 LinearScan::LinearScan(IR* ir, LIRGenerator* gen, FrameMap* frame_map) | |
| 81 : _compilation(ir->compilation()) | |
| 82 , _ir(ir) | |
| 83 , _gen(gen) | |
| 84 , _frame_map(frame_map) | |
| 85 , _num_virtual_regs(gen->max_virtual_register_number()) | |
| 86 , _has_fpu_registers(false) | |
| 87 , _num_calls(-1) | |
| 88 , _max_spills(0) | |
| 89 , _unused_spill_slot(-1) | |
| 90 , _intervals(0) // initialized later with correct length | |
| 91 , _new_intervals_from_allocation(new IntervalList()) | |
| 92 , _sorted_intervals(NULL) | |
| 93 , _lir_ops(0) // initialized later with correct length | |
| 94 , _block_of_op(0) // initialized later with correct length | |
| 95 , _has_info(0) | |
| 96 , _has_call(0) | |
| 97 , _scope_value_cache(0) // initialized later with correct length | |
| 98 , _interval_in_loop(0, 0) // initialized later with correct length | |
| 99 , _cached_blocks(*ir->linear_scan_order()) | |
| 304 | 100 #ifdef X86 |
| 0 | 101 , _fpu_stack_allocator(NULL) |
| 102 #endif | |
| 103 { | |
| 104 assert(this->ir() != NULL, "check if valid"); | |
| 105 assert(this->compilation() != NULL, "check if valid"); | |
| 106 assert(this->gen() != NULL, "check if valid"); | |
| 107 assert(this->frame_map() != NULL, "check if valid"); | |
| 108 } | |
| 109 | |
| 110 | |
| 111 // ********** functions for converting LIR-Operands to register numbers | |
| 112 // | |
| 113 // Emulate a flat register file comprising physical integer registers, | |
| 114 // physical floating-point registers and virtual registers, in that order. | |
| 115 // Virtual registers already have appropriate numbers, since V0 is | |
| 116 // the number of physical registers. | |
| 117 // Returns -1 for hi word if opr is a single word operand. | |
| 118 // | |
| 119 // Note: the inverse operation (calculating an operand for register numbers) | |
| 120 // is done in calc_operand_for_interval() | |
| 121 | |
| 122 int LinearScan::reg_num(LIR_Opr opr) { | |
| 123 assert(opr->is_register(), "should not call this otherwise"); | |
| 124 | |
| 125 if (opr->is_virtual_register()) { | |
| 126 assert(opr->vreg_number() >= nof_regs, "found a virtual register with a fixed-register number"); | |
| 127 return opr->vreg_number(); | |
| 128 } else if (opr->is_single_cpu()) { | |
| 129 return opr->cpu_regnr(); | |
| 130 } else if (opr->is_double_cpu()) { | |
| 131 return opr->cpu_regnrLo(); | |
| 304 | 132 #ifdef X86 |
| 0 | 133 } else if (opr->is_single_xmm()) { |
| 134 return opr->fpu_regnr() + pd_first_xmm_reg; | |
| 135 } else if (opr->is_double_xmm()) { | |
| 136 return opr->fpu_regnrLo() + pd_first_xmm_reg; | |
| 137 #endif | |
| 138 } else if (opr->is_single_fpu()) { | |
| 139 return opr->fpu_regnr() + pd_first_fpu_reg; | |
| 140 } else if (opr->is_double_fpu()) { | |
| 141 return opr->fpu_regnrLo() + pd_first_fpu_reg; | |
| 142 } else { | |
| 143 ShouldNotReachHere(); | |
| 304 | 144 return -1; |
| 0 | 145 } |
| 146 } | |
| 147 | |
| 148 int LinearScan::reg_numHi(LIR_Opr opr) { | |
| 149 assert(opr->is_register(), "should not call this otherwise"); | |
| 150 | |
| 151 if (opr->is_virtual_register()) { | |
| 152 return -1; | |
| 153 } else if (opr->is_single_cpu()) { | |
| 154 return -1; | |
| 155 } else if (opr->is_double_cpu()) { | |
| 156 return opr->cpu_regnrHi(); | |
| 304 | 157 #ifdef X86 |
| 0 | 158 } else if (opr->is_single_xmm()) { |
| 159 return -1; | |
| 160 } else if (opr->is_double_xmm()) { | |
| 161 return -1; | |
| 162 #endif | |
| 163 } else if (opr->is_single_fpu()) { | |
| 164 return -1; | |
| 165 } else if (opr->is_double_fpu()) { | |
| 166 return opr->fpu_regnrHi() + pd_first_fpu_reg; | |
| 167 } else { | |
| 168 ShouldNotReachHere(); | |
| 304 | 169 return -1; |
| 0 | 170 } |
| 171 } | |
| 172 | |
| 173 | |
| 174 // ********** functions for classification of intervals | |
| 175 | |
| 176 bool LinearScan::is_precolored_interval(const Interval* i) { | |
| 177 return i->reg_num() < LinearScan::nof_regs; | |
| 178 } | |
| 179 | |
| 180 bool LinearScan::is_virtual_interval(const Interval* i) { | |
| 181 return i->reg_num() >= LIR_OprDesc::vreg_base; | |
| 182 } | |
| 183 | |
| 184 bool LinearScan::is_precolored_cpu_interval(const Interval* i) { | |
| 185 return i->reg_num() < LinearScan::nof_cpu_regs; | |
| 186 } | |
| 187 | |
| 188 bool LinearScan::is_virtual_cpu_interval(const Interval* i) { | |
|
1681
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
189 #if defined(__SOFTFP__) || defined(E500V2) |
|
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
190 return i->reg_num() >= LIR_OprDesc::vreg_base; |
|
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
191 #else |
| 0 | 192 return i->reg_num() >= LIR_OprDesc::vreg_base && (i->type() != T_FLOAT && i->type() != T_DOUBLE); |
|
1681
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
193 #endif // __SOFTFP__ or E500V2 |
| 0 | 194 } |
| 195 | |
| 196 bool LinearScan::is_precolored_fpu_interval(const Interval* i) { | |
| 197 return i->reg_num() >= LinearScan::nof_cpu_regs && i->reg_num() < LinearScan::nof_regs; | |
| 198 } | |
| 199 | |
| 200 bool LinearScan::is_virtual_fpu_interval(const Interval* i) { | |
|
1681
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
201 #if defined(__SOFTFP__) || defined(E500V2) |
|
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
202 return false; |
|
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
203 #else |
| 0 | 204 return i->reg_num() >= LIR_OprDesc::vreg_base && (i->type() == T_FLOAT || i->type() == T_DOUBLE); |
|
1681
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
205 #endif // __SOFTFP__ or E500V2 |
| 0 | 206 } |
| 207 | |
| 208 bool LinearScan::is_in_fpu_register(const Interval* i) { | |
| 209 // fixed intervals not needed for FPU stack allocation | |
| 210 return i->reg_num() >= nof_regs && pd_first_fpu_reg <= i->assigned_reg() && i->assigned_reg() <= pd_last_fpu_reg; | |
| 211 } | |
| 212 | |
| 213 bool LinearScan::is_oop_interval(const Interval* i) { | |
| 214 // fixed intervals never contain oops | |
| 215 return i->reg_num() >= nof_regs && i->type() == T_OBJECT; | |
| 216 } | |
| 217 | |
| 218 | |
| 219 // ********** General helper functions | |
| 220 | |
| 221 // compute next unused stack index that can be used for spilling | |
| 222 int LinearScan::allocate_spill_slot(bool double_word) { | |
| 223 int spill_slot; | |
| 224 if (double_word) { | |
| 225 if ((_max_spills & 1) == 1) { | |
| 226 // alignment of double-word values | |
| 227 // the hole because of the alignment is filled with the next single-word value | |
| 228 assert(_unused_spill_slot == -1, "wasting a spill slot"); | |
| 229 _unused_spill_slot = _max_spills; | |
| 230 _max_spills++; | |
| 231 } | |
| 232 spill_slot = _max_spills; | |
| 233 _max_spills += 2; | |
| 234 | |
| 235 } else if (_unused_spill_slot != -1) { | |
| 236 // re-use hole that was the result of a previous double-word alignment | |
| 237 spill_slot = _unused_spill_slot; | |
| 238 _unused_spill_slot = -1; | |
| 239 | |
| 240 } else { | |
| 241 spill_slot = _max_spills; | |
| 242 _max_spills++; | |
| 243 } | |
| 244 | |
| 245 int result = spill_slot + LinearScan::nof_regs + frame_map()->argcount(); | |
| 246 | |
| 247 // the class OopMapValue uses only 11 bits for storing the name of the | |
| 248 // oop location. So a stack slot bigger than 2^11 leads to an overflow | |
| 249 // that is not reported in product builds. Prevent this by checking the | |
| 250 // spill slot here (altough this value and the later used location name | |
| 251 // are slightly different) | |
| 252 if (result > 2000) { | |
| 253 bailout("too many stack slots used"); | |
| 254 } | |
| 255 | |
| 256 return result; | |
| 257 } | |
| 258 | |
| 259 void LinearScan::assign_spill_slot(Interval* it) { | |
| 260 // assign the canonical spill slot of the parent (if a part of the interval | |
| 261 // is already spilled) or allocate a new spill slot | |
| 262 if (it->canonical_spill_slot() >= 0) { | |
| 263 it->assign_reg(it->canonical_spill_slot()); | |
| 264 } else { | |
| 265 int spill = allocate_spill_slot(type2spill_size[it->type()] == 2); | |
| 266 it->set_canonical_spill_slot(spill); | |
| 267 it->assign_reg(spill); | |
| 268 } | |
| 269 } | |
| 270 | |
| 271 void LinearScan::propagate_spill_slots() { | |
| 272 if (!frame_map()->finalize_frame(max_spills())) { | |
| 273 bailout("frame too large"); | |
| 274 } | |
| 275 } | |
| 276 | |
| 277 // create a new interval with a predefined reg_num | |
| 278 // (only used for parent intervals that are created during the building phase) | |
| 279 Interval* LinearScan::create_interval(int reg_num) { | |
| 280 assert(_intervals.at(reg_num) == NULL, "overwriting exisiting interval"); | |
| 281 | |
| 282 Interval* interval = new Interval(reg_num); | |
| 283 _intervals.at_put(reg_num, interval); | |
| 284 | |
| 285 // assign register number for precolored intervals | |
| 286 if (reg_num < LIR_OprDesc::vreg_base) { | |
| 287 interval->assign_reg(reg_num); | |
| 288 } | |
| 289 return interval; | |
| 290 } | |
| 291 | |
| 292 // assign a new reg_num to the interval and append it to the list of intervals | |
| 293 // (only used for child intervals that are created during register allocation) | |
| 294 void LinearScan::append_interval(Interval* it) { | |
| 295 it->set_reg_num(_intervals.length()); | |
| 296 _intervals.append(it); | |
| 297 _new_intervals_from_allocation->append(it); | |
| 298 } | |
| 299 | |
| 300 // copy the vreg-flags if an interval is split | |
| 301 void LinearScan::copy_register_flags(Interval* from, Interval* to) { | |
| 302 if (gen()->is_vreg_flag_set(from->reg_num(), LIRGenerator::byte_reg)) { | |
| 303 gen()->set_vreg_flag(to->reg_num(), LIRGenerator::byte_reg); | |
| 304 } | |
| 305 if (gen()->is_vreg_flag_set(from->reg_num(), LIRGenerator::callee_saved)) { | |
| 306 gen()->set_vreg_flag(to->reg_num(), LIRGenerator::callee_saved); | |
| 307 } | |
| 308 | |
| 309 // Note: do not copy the must_start_in_memory flag because it is not necessary for child | |
| 310 // intervals (only the very beginning of the interval must be in memory) | |
| 311 } | |
| 312 | |
| 313 | |
| 314 // ********** spill move optimization | |
| 315 // eliminate moves from register to stack if stack slot is known to be correct | |
| 316 | |
| 317 // called during building of intervals | |
| 318 void LinearScan::change_spill_definition_pos(Interval* interval, int def_pos) { | |
| 319 assert(interval->is_split_parent(), "can only be called for split parents"); | |
| 320 | |
| 321 switch (interval->spill_state()) { | |
| 322 case noDefinitionFound: | |
| 323 assert(interval->spill_definition_pos() == -1, "must no be set before"); | |
| 324 interval->set_spill_definition_pos(def_pos); | |
| 325 interval->set_spill_state(oneDefinitionFound); | |
| 326 break; | |
| 327 | |
| 328 case oneDefinitionFound: | |
| 329 assert(def_pos <= interval->spill_definition_pos(), "positions are processed in reverse order when intervals are created"); | |
| 330 if (def_pos < interval->spill_definition_pos() - 2) { | |
| 331 // second definition found, so no spill optimization possible for this interval | |
| 332 interval->set_spill_state(noOptimization); | |
| 333 } else { | |
| 334 // two consecutive definitions (because of two-operand LIR form) | |
| 335 assert(block_of_op_with_id(def_pos) == block_of_op_with_id(interval->spill_definition_pos()), "block must be equal"); | |
| 336 } | |
| 337 break; | |
| 338 | |
| 339 case noOptimization: | |
| 340 // nothing to do | |
| 341 break; | |
| 342 | |
| 343 default: | |
| 344 assert(false, "other states not allowed at this time"); | |
| 345 } | |
| 346 } | |
| 347 | |
| 348 // called during register allocation | |
| 349 void LinearScan::change_spill_state(Interval* interval, int spill_pos) { | |
| 350 switch (interval->spill_state()) { | |
| 351 case oneDefinitionFound: { | |
| 352 int def_loop_depth = block_of_op_with_id(interval->spill_definition_pos())->loop_depth(); | |
| 353 int spill_loop_depth = block_of_op_with_id(spill_pos)->loop_depth(); | |
| 354 | |
| 355 if (def_loop_depth < spill_loop_depth) { | |
| 356 // the loop depth of the spilling position is higher then the loop depth | |
| 357 // at the definition of the interval -> move write to memory out of loop | |
| 358 // by storing at definitin of the interval | |
| 359 interval->set_spill_state(storeAtDefinition); | |
| 360 } else { | |
| 361 // the interval is currently spilled only once, so for now there is no | |
| 362 // reason to store the interval at the definition | |
| 363 interval->set_spill_state(oneMoveInserted); | |
| 364 } | |
| 365 break; | |
| 366 } | |
| 367 | |
| 368 case oneMoveInserted: { | |
| 369 // the interval is spilled more then once, so it is better to store it to | |
| 370 // memory at the definition | |
| 371 interval->set_spill_state(storeAtDefinition); | |
| 372 break; | |
| 373 } | |
| 374 | |
| 375 case storeAtDefinition: | |
| 376 case startInMemory: | |
| 377 case noOptimization: | |
| 378 case noDefinitionFound: | |
| 379 // nothing to do | |
| 380 break; | |
| 381 | |
| 382 default: | |
| 383 assert(false, "other states not allowed at this time"); | |
| 384 } | |
| 385 } | |
| 386 | |
| 387 | |
| 388 bool LinearScan::must_store_at_definition(const Interval* i) { | |
| 389 return i->is_split_parent() && i->spill_state() == storeAtDefinition; | |
| 390 } | |
| 391 | |
| 392 // called once before asignment of register numbers | |
| 393 void LinearScan::eliminate_spill_moves() { | |
| 394 TIME_LINEAR_SCAN(timer_eliminate_spill_moves); | |
| 395 TRACE_LINEAR_SCAN(3, tty->print_cr("***** Eliminating unnecessary spill moves")); | |
| 396 | |
| 397 // collect all intervals that must be stored after their definion. | |
| 398 // the list is sorted by Interval::spill_definition_pos | |
| 399 Interval* interval; | |
| 400 Interval* temp_list; | |
| 401 create_unhandled_lists(&interval, &temp_list, must_store_at_definition, NULL); | |
| 402 | |
| 403 #ifdef ASSERT | |
| 404 Interval* prev = NULL; | |
| 405 Interval* temp = interval; | |
| 406 while (temp != Interval::end()) { | |
| 407 assert(temp->spill_definition_pos() > 0, "invalid spill definition pos"); | |
| 408 if (prev != NULL) { | |
| 409 assert(temp->from() >= prev->from(), "intervals not sorted"); | |
| 410 assert(temp->spill_definition_pos() >= prev->spill_definition_pos(), "when intervals are sorted by from, then they must also be sorted by spill_definition_pos"); | |
| 411 } | |
| 412 | |
| 413 assert(temp->canonical_spill_slot() >= LinearScan::nof_regs, "interval has no spill slot assigned"); | |
| 414 assert(temp->spill_definition_pos() >= temp->from(), "invalid order"); | |
| 415 assert(temp->spill_definition_pos() <= temp->from() + 2, "only intervals defined once at their start-pos can be optimized"); | |
| 416 | |
| 417 TRACE_LINEAR_SCAN(4, tty->print_cr("interval %d (from %d to %d) must be stored at %d", temp->reg_num(), temp->from(), temp->to(), temp->spill_definition_pos())); | |
| 418 | |
| 419 temp = temp->next(); | |
| 420 } | |
| 421 #endif | |
| 422 | |
| 423 LIR_InsertionBuffer insertion_buffer; | |
| 424 int num_blocks = block_count(); | |
| 425 for (int i = 0; i < num_blocks; i++) { | |
| 426 BlockBegin* block = block_at(i); | |
| 427 LIR_OpList* instructions = block->lir()->instructions_list(); | |
| 428 int num_inst = instructions->length(); | |
| 429 bool has_new = false; | |
| 430 | |
| 431 // iterate all instructions of the block. skip the first because it is always a label | |
| 432 for (int j = 1; j < num_inst; j++) { | |
| 433 LIR_Op* op = instructions->at(j); | |
| 434 int op_id = op->id(); | |
| 435 | |
| 436 if (op_id == -1) { | |
| 437 // remove move from register to stack if the stack slot is guaranteed to be correct. | |
| 438 // only moves that have been inserted by LinearScan can be removed. | |
| 439 assert(op->code() == lir_move, "only moves can have a op_id of -1"); | |
| 440 assert(op->as_Op1() != NULL, "move must be LIR_Op1"); | |
| 441 assert(op->as_Op1()->result_opr()->is_virtual(), "LinearScan inserts only moves to virtual registers"); | |
| 442 | |
| 443 LIR_Op1* op1 = (LIR_Op1*)op; | |
| 444 Interval* interval = interval_at(op1->result_opr()->vreg_number()); | |
| 445 | |
| 446 if (interval->assigned_reg() >= LinearScan::nof_regs && interval->always_in_memory()) { | |
| 447 // move target is a stack slot that is always correct, so eliminate instruction | |
| 448 TRACE_LINEAR_SCAN(4, tty->print_cr("eliminating move from interval %d to %d", op1->in_opr()->vreg_number(), op1->result_opr()->vreg_number())); | |
| 449 instructions->at_put(j, NULL); // NULL-instructions are deleted by assign_reg_num | |
| 450 } | |
| 451 | |
| 452 } else { | |
| 453 // insert move from register to stack just after the beginning of the interval | |
| 454 assert(interval == Interval::end() || interval->spill_definition_pos() >= op_id, "invalid order"); | |
| 455 assert(interval == Interval::end() || (interval->is_split_parent() && interval->spill_state() == storeAtDefinition), "invalid interval"); | |
| 456 | |
| 457 while (interval != Interval::end() && interval->spill_definition_pos() == op_id) { | |
| 458 if (!has_new) { | |
| 459 // prepare insertion buffer (appended when all instructions of the block are processed) | |
| 460 insertion_buffer.init(block->lir()); | |
| 461 has_new = true; | |
| 462 } | |
| 463 | |
| 464 LIR_Opr from_opr = operand_for_interval(interval); | |
| 465 LIR_Opr to_opr = canonical_spill_opr(interval); | |
| 466 assert(from_opr->is_fixed_cpu() || from_opr->is_fixed_fpu(), "from operand must be a register"); | |
| 467 assert(to_opr->is_stack(), "to operand must be a stack slot"); | |
| 468 | |
| 469 insertion_buffer.move(j, from_opr, to_opr); | |
| 470 TRACE_LINEAR_SCAN(4, tty->print_cr("inserting move after definition of interval %d to stack slot %d at op_id %d", interval->reg_num(), interval->canonical_spill_slot() - LinearScan::nof_regs, op_id)); | |
| 471 | |
| 472 interval = interval->next(); | |
| 473 } | |
| 474 } | |
| 475 } // end of instruction iteration | |
| 476 | |
| 477 if (has_new) { | |
| 478 block->lir()->append(&insertion_buffer); | |
| 479 } | |
| 480 } // end of block iteration | |
| 481 | |
| 482 assert(interval == Interval::end(), "missed an interval"); | |
| 483 } | |
| 484 | |
| 485 | |
| 486 // ********** Phase 1: number all instructions in all blocks | |
| 487 // Compute depth-first and linear scan block orders, and number LIR_Op nodes for linear scan. | |
| 488 | |
| 489 void LinearScan::number_instructions() { | |
| 490 { | |
| 491 // dummy-timer to measure the cost of the timer itself | |
| 492 // (this time is then subtracted from all other timers to get the real value) | |
| 493 TIME_LINEAR_SCAN(timer_do_nothing); | |
| 494 } | |
| 495 TIME_LINEAR_SCAN(timer_number_instructions); | |
| 496 | |
| 497 // Assign IDs to LIR nodes and build a mapping, lir_ops, from ID to LIR_Op node. | |
| 498 int num_blocks = block_count(); | |
| 499 int num_instructions = 0; | |
| 500 int i; | |
| 501 for (i = 0; i < num_blocks; i++) { | |
| 502 num_instructions += block_at(i)->lir()->instructions_list()->length(); | |
| 503 } | |
| 504 | |
| 505 // initialize with correct length | |
| 506 _lir_ops = LIR_OpArray(num_instructions); | |
| 507 _block_of_op = BlockBeginArray(num_instructions); | |
| 508 | |
| 509 int op_id = 0; | |
| 510 int idx = 0; | |
| 511 | |
| 512 for (i = 0; i < num_blocks; i++) { | |
| 513 BlockBegin* block = block_at(i); | |
| 514 block->set_first_lir_instruction_id(op_id); | |
| 515 LIR_OpList* instructions = block->lir()->instructions_list(); | |
| 516 | |
| 517 int num_inst = instructions->length(); | |
| 518 for (int j = 0; j < num_inst; j++) { | |
| 519 LIR_Op* op = instructions->at(j); | |
| 520 op->set_id(op_id); | |
| 521 | |
| 522 _lir_ops.at_put(idx, op); | |
| 523 _block_of_op.at_put(idx, block); | |
| 524 assert(lir_op_with_id(op_id) == op, "must match"); | |
| 525 | |
| 526 idx++; | |
| 527 op_id += 2; // numbering of lir_ops by two | |
| 528 } | |
| 529 block->set_last_lir_instruction_id(op_id - 2); | |
| 530 } | |
| 531 assert(idx == num_instructions, "must match"); | |
| 532 assert(idx * 2 == op_id, "must match"); | |
| 533 | |
| 534 _has_call = BitMap(num_instructions); _has_call.clear(); | |
| 535 _has_info = BitMap(num_instructions); _has_info.clear(); | |
| 536 } | |
| 537 | |
| 538 | |
| 539 // ********** Phase 2: compute local live sets separately for each block | |
| 540 // (sets live_gen and live_kill for each block) | |
| 541 | |
| 542 void LinearScan::set_live_gen_kill(Value value, LIR_Op* op, BitMap& live_gen, BitMap& live_kill) { | |
| 543 LIR_Opr opr = value->operand(); | |
| 544 Constant* con = value->as_Constant(); | |
| 545 | |
| 546 // check some asumptions about debug information | |
| 547 assert(!value->type()->is_illegal(), "if this local is used by the interpreter it shouldn't be of indeterminate type"); | |
| 548 assert(con == NULL || opr->is_virtual() || opr->is_constant() || opr->is_illegal(), "asumption: Constant instructions have only constant operands"); | |
| 549 assert(con != NULL || opr->is_virtual(), "asumption: non-Constant instructions have only virtual operands"); | |
| 550 | |
| 551 if ((con == NULL || con->is_pinned()) && opr->is_register()) { | |
| 552 assert(reg_num(opr) == opr->vreg_number() && !is_valid_reg_num(reg_numHi(opr)), "invalid optimization below"); | |
| 553 int reg = opr->vreg_number(); | |
| 554 if (!live_kill.at(reg)) { | |
| 555 live_gen.set_bit(reg); | |
| 556 TRACE_LINEAR_SCAN(4, tty->print_cr(" Setting live_gen for value %c%d, LIR op_id %d, register number %d", value->type()->tchar(), value->id(), op->id(), reg)); | |
| 557 } | |
| 558 } | |
| 559 } | |
| 560 | |
| 561 | |
| 562 void LinearScan::compute_local_live_sets() { | |
| 563 TIME_LINEAR_SCAN(timer_compute_local_live_sets); | |
| 564 | |
| 565 int num_blocks = block_count(); | |
| 566 int live_size = live_set_size(); | |
| 567 bool local_has_fpu_registers = false; | |
| 568 int local_num_calls = 0; | |
| 569 LIR_OpVisitState visitor; | |
| 570 | |
| 571 BitMap2D local_interval_in_loop = BitMap2D(_num_virtual_regs, num_loops()); | |
| 572 local_interval_in_loop.clear(); | |
| 573 | |
| 574 // iterate all blocks | |
| 575 for (int i = 0; i < num_blocks; i++) { | |
| 576 BlockBegin* block = block_at(i); | |
| 577 | |
| 578 BitMap live_gen(live_size); live_gen.clear(); | |
| 579 BitMap live_kill(live_size); live_kill.clear(); | |
| 580 | |
| 581 if (block->is_set(BlockBegin::exception_entry_flag)) { | |
| 582 // Phi functions at the begin of an exception handler are | |
| 583 // implicitly defined (= killed) at the beginning of the block. | |
| 584 for_each_phi_fun(block, phi, | |
| 585 live_kill.set_bit(phi->operand()->vreg_number()) | |
| 586 ); | |
| 587 } | |
| 588 | |
| 589 LIR_OpList* instructions = block->lir()->instructions_list(); | |
| 590 int num_inst = instructions->length(); | |
| 591 | |
| 592 // iterate all instructions of the block. skip the first because it is always a label | |
| 593 assert(visitor.no_operands(instructions->at(0)), "first operation must always be a label"); | |
| 594 for (int j = 1; j < num_inst; j++) { | |
| 595 LIR_Op* op = instructions->at(j); | |
| 596 | |
| 597 // visit operation to collect all operands | |
| 598 visitor.visit(op); | |
| 599 | |
| 600 if (visitor.has_call()) { | |
| 601 _has_call.set_bit(op->id() >> 1); | |
| 602 local_num_calls++; | |
| 603 } | |
| 604 if (visitor.info_count() > 0) { | |
| 605 _has_info.set_bit(op->id() >> 1); | |
| 606 } | |
| 607 | |
| 608 // iterate input operands of instruction | |
| 609 int k, n, reg; | |
| 610 n = visitor.opr_count(LIR_OpVisitState::inputMode); | |
| 611 for (k = 0; k < n; k++) { | |
| 612 LIR_Opr opr = visitor.opr_at(LIR_OpVisitState::inputMode, k); | |
| 613 assert(opr->is_register(), "visitor should only return register operands"); | |
| 614 | |
| 615 if (opr->is_virtual_register()) { | |
| 616 assert(reg_num(opr) == opr->vreg_number() && !is_valid_reg_num(reg_numHi(opr)), "invalid optimization below"); | |
| 617 reg = opr->vreg_number(); | |
| 618 if (!live_kill.at(reg)) { | |
| 619 live_gen.set_bit(reg); | |
| 620 TRACE_LINEAR_SCAN(4, tty->print_cr(" Setting live_gen for register %d at instruction %d", reg, op->id())); | |
| 621 } | |
| 622 if (block->loop_index() >= 0) { | |
| 623 local_interval_in_loop.set_bit(reg, block->loop_index()); | |
| 624 } | |
| 625 local_has_fpu_registers = local_has_fpu_registers || opr->is_virtual_fpu(); | |
| 626 } | |
| 627 | |
| 628 #ifdef ASSERT | |
| 629 // fixed intervals are never live at block boundaries, so | |
| 630 // they need not be processed in live sets. | |
| 631 // this is checked by these assertions to be sure about it. | |
| 632 // the entry block may have incoming values in registers, which is ok. | |
| 633 if (!opr->is_virtual_register() && block != ir()->start()) { | |
| 634 reg = reg_num(opr); | |
| 635 if (is_processed_reg_num(reg)) { | |
| 636 assert(live_kill.at(reg), "using fixed register that is not defined in this block"); | |
| 637 } | |
| 638 reg = reg_numHi(opr); | |
| 639 if (is_valid_reg_num(reg) && is_processed_reg_num(reg)) { | |
| 640 assert(live_kill.at(reg), "using fixed register that is not defined in this block"); | |
| 641 } | |
| 642 } | |
| 643 #endif | |
| 644 } | |
| 645 | |
| 646 // Add uses of live locals from interpreter's point of view for proper debug information generation | |
| 647 n = visitor.info_count(); | |
| 648 for (k = 0; k < n; k++) { | |
| 649 CodeEmitInfo* info = visitor.info_at(k); | |
| 650 ValueStack* stack = info->stack(); | |
| 651 for_each_state_value(stack, value, | |
| 652 set_live_gen_kill(value, op, live_gen, live_kill) | |
| 653 ); | |
| 654 } | |
| 655 | |
| 656 // iterate temp operands of instruction | |
| 657 n = visitor.opr_count(LIR_OpVisitState::tempMode); | |
| 658 for (k = 0; k < n; k++) { | |
| 659 LIR_Opr opr = visitor.opr_at(LIR_OpVisitState::tempMode, k); | |
| 660 assert(opr->is_register(), "visitor should only return register operands"); | |
| 661 | |
| 662 if (opr->is_virtual_register()) { | |
| 663 assert(reg_num(opr) == opr->vreg_number() && !is_valid_reg_num(reg_numHi(opr)), "invalid optimization below"); | |
| 664 reg = opr->vreg_number(); | |
| 665 live_kill.set_bit(reg); | |
| 666 if (block->loop_index() >= 0) { | |
| 667 local_interval_in_loop.set_bit(reg, block->loop_index()); | |
| 668 } | |
| 669 local_has_fpu_registers = local_has_fpu_registers || opr->is_virtual_fpu(); | |
| 670 } | |
| 671 | |
| 672 #ifdef ASSERT | |
| 673 // fixed intervals are never live at block boundaries, so | |
| 674 // they need not be processed in live sets | |
| 675 // process them only in debug mode so that this can be checked | |
| 676 if (!opr->is_virtual_register()) { | |
| 677 reg = reg_num(opr); | |
| 678 if (is_processed_reg_num(reg)) { | |
| 679 live_kill.set_bit(reg_num(opr)); | |
| 680 } | |
| 681 reg = reg_numHi(opr); | |
| 682 if (is_valid_reg_num(reg) && is_processed_reg_num(reg)) { | |
| 683 live_kill.set_bit(reg); | |
| 684 } | |
| 685 } | |
| 686 #endif | |
| 687 } | |
| 688 | |
| 689 // iterate output operands of instruction | |
| 690 n = visitor.opr_count(LIR_OpVisitState::outputMode); | |
| 691 for (k = 0; k < n; k++) { | |
| 692 LIR_Opr opr = visitor.opr_at(LIR_OpVisitState::outputMode, k); | |
| 693 assert(opr->is_register(), "visitor should only return register operands"); | |
| 694 | |
| 695 if (opr->is_virtual_register()) { | |
| 696 assert(reg_num(opr) == opr->vreg_number() && !is_valid_reg_num(reg_numHi(opr)), "invalid optimization below"); | |
| 697 reg = opr->vreg_number(); | |
| 698 live_kill.set_bit(reg); | |
| 699 if (block->loop_index() >= 0) { | |
| 700 local_interval_in_loop.set_bit(reg, block->loop_index()); | |
| 701 } | |
| 702 local_has_fpu_registers = local_has_fpu_registers || opr->is_virtual_fpu(); | |
| 703 } | |
| 704 | |
| 705 #ifdef ASSERT | |
| 706 // fixed intervals are never live at block boundaries, so | |
| 707 // they need not be processed in live sets | |
| 708 // process them only in debug mode so that this can be checked | |
| 709 if (!opr->is_virtual_register()) { | |
| 710 reg = reg_num(opr); | |
| 711 if (is_processed_reg_num(reg)) { | |
| 712 live_kill.set_bit(reg_num(opr)); | |
| 713 } | |
| 714 reg = reg_numHi(opr); | |
| 715 if (is_valid_reg_num(reg) && is_processed_reg_num(reg)) { | |
| 716 live_kill.set_bit(reg); | |
| 717 } | |
| 718 } | |
| 719 #endif | |
| 720 } | |
| 721 } // end of instruction iteration | |
| 722 | |
| 723 block->set_live_gen (live_gen); | |
| 724 block->set_live_kill(live_kill); | |
| 725 block->set_live_in (BitMap(live_size)); block->live_in().clear(); | |
| 726 block->set_live_out (BitMap(live_size)); block->live_out().clear(); | |
| 727 | |
| 728 TRACE_LINEAR_SCAN(4, tty->print("live_gen B%d ", block->block_id()); print_bitmap(block->live_gen())); | |
| 729 TRACE_LINEAR_SCAN(4, tty->print("live_kill B%d ", block->block_id()); print_bitmap(block->live_kill())); | |
| 730 } // end of block iteration | |
| 731 | |
| 732 // propagate local calculated information into LinearScan object | |
| 733 _has_fpu_registers = local_has_fpu_registers; | |
| 734 compilation()->set_has_fpu_code(local_has_fpu_registers); | |
| 735 | |
| 736 _num_calls = local_num_calls; | |
| 737 _interval_in_loop = local_interval_in_loop; | |
| 738 } | |
| 739 | |
| 740 | |
| 741 // ********** Phase 3: perform a backward dataflow analysis to compute global live sets | |
| 742 // (sets live_in and live_out for each block) | |
| 743 | |
| 744 void LinearScan::compute_global_live_sets() { | |
| 745 TIME_LINEAR_SCAN(timer_compute_global_live_sets); | |
| 746 | |
| 747 int num_blocks = block_count(); | |
| 748 bool change_occurred; | |
| 749 bool change_occurred_in_block; | |
| 750 int iteration_count = 0; | |
| 751 BitMap live_out(live_set_size()); live_out.clear(); // scratch set for calculations | |
| 752 | |
| 753 // Perform a backward dataflow analysis to compute live_out and live_in for each block. | |
| 754 // The loop is executed until a fixpoint is reached (no changes in an iteration) | |
| 755 // Exception handlers must be processed because not all live values are | |
| 756 // present in the state array, e.g. because of global value numbering | |
| 757 do { | |
| 758 change_occurred = false; | |
| 759 | |
| 760 // iterate all blocks in reverse order | |
| 761 for (int i = num_blocks - 1; i >= 0; i--) { | |
| 762 BlockBegin* block = block_at(i); | |
| 763 | |
| 764 change_occurred_in_block = false; | |
| 765 | |
| 766 // live_out(block) is the union of live_in(sux), for successors sux of block | |
| 767 int n = block->number_of_sux(); | |
| 768 int e = block->number_of_exception_handlers(); | |
| 769 if (n + e > 0) { | |
| 770 // block has successors | |
| 771 if (n > 0) { | |
| 772 live_out.set_from(block->sux_at(0)->live_in()); | |
| 773 for (int j = 1; j < n; j++) { | |
| 774 live_out.set_union(block->sux_at(j)->live_in()); | |
| 775 } | |
| 776 } else { | |
| 777 live_out.clear(); | |
| 778 } | |
| 779 for (int j = 0; j < e; j++) { | |
| 780 live_out.set_union(block->exception_handler_at(j)->live_in()); | |
| 781 } | |
| 782 | |
| 783 if (!block->live_out().is_same(live_out)) { | |
| 784 // A change occurred. Swap the old and new live out sets to avoid copying. | |
| 785 BitMap temp = block->live_out(); | |
| 786 block->set_live_out(live_out); | |
| 787 live_out = temp; | |
| 788 | |
| 789 change_occurred = true; | |
| 790 change_occurred_in_block = true; | |
| 791 } | |
| 792 } | |
| 793 | |
| 794 if (iteration_count == 0 || change_occurred_in_block) { | |
| 795 // live_in(block) is the union of live_gen(block) with (live_out(block) & !live_kill(block)) | |
| 796 // note: live_in has to be computed only in first iteration or if live_out has changed! | |
| 797 BitMap live_in = block->live_in(); | |
| 798 live_in.set_from(block->live_out()); | |
| 799 live_in.set_difference(block->live_kill()); | |
| 800 live_in.set_union(block->live_gen()); | |
| 801 } | |
| 802 | |
| 803 #ifndef PRODUCT | |
| 804 if (TraceLinearScanLevel >= 4) { | |
| 805 char c = ' '; | |
| 806 if (iteration_count == 0 || change_occurred_in_block) { | |
| 807 c = '*'; | |
| 808 } | |
| 809 tty->print("(%d) live_in%c B%d ", iteration_count, c, block->block_id()); print_bitmap(block->live_in()); | |
| 810 tty->print("(%d) live_out%c B%d ", iteration_count, c, block->block_id()); print_bitmap(block->live_out()); | |
| 811 } | |
| 812 #endif | |
| 813 } | |
| 814 iteration_count++; | |
| 815 | |
| 816 if (change_occurred && iteration_count > 50) { | |
| 817 BAILOUT("too many iterations in compute_global_live_sets"); | |
| 818 } | |
| 819 } while (change_occurred); | |
| 820 | |
| 821 | |
| 822 #ifdef ASSERT | |
| 823 // check that fixed intervals are not live at block boundaries | |
| 824 // (live set must be empty at fixed intervals) | |
| 825 for (int i = 0; i < num_blocks; i++) { | |
| 826 BlockBegin* block = block_at(i); | |
| 827 for (int j = 0; j < LIR_OprDesc::vreg_base; j++) { | |
| 828 assert(block->live_in().at(j) == false, "live_in set of fixed register must be empty"); | |
| 829 assert(block->live_out().at(j) == false, "live_out set of fixed register must be empty"); | |
| 830 assert(block->live_gen().at(j) == false, "live_gen set of fixed register must be empty"); | |
| 831 } | |
| 832 } | |
| 833 #endif | |
| 834 | |
| 835 // check that the live_in set of the first block is empty | |
| 836 BitMap live_in_args(ir()->start()->live_in().size()); | |
| 837 live_in_args.clear(); | |
| 838 if (!ir()->start()->live_in().is_same(live_in_args)) { | |
| 839 #ifdef ASSERT | |
| 840 tty->print_cr("Error: live_in set of first block must be empty (when this fails, virtual registers are used before they are defined)"); | |
| 841 tty->print_cr("affected registers:"); | |
| 842 print_bitmap(ir()->start()->live_in()); | |
| 843 | |
| 844 // print some additional information to simplify debugging | |
| 845 for (unsigned int i = 0; i < ir()->start()->live_in().size(); i++) { | |
| 846 if (ir()->start()->live_in().at(i)) { | |
| 847 Instruction* instr = gen()->instruction_for_vreg(i); | |
| 848 tty->print_cr("* vreg %d (HIR instruction %c%d)", i, instr == NULL ? ' ' : instr->type()->tchar(), instr == NULL ? 0 : instr->id()); | |
| 849 | |
| 850 for (int j = 0; j < num_blocks; j++) { | |
| 851 BlockBegin* block = block_at(j); | |
| 852 if (block->live_gen().at(i)) { | |
| 853 tty->print_cr(" used in block B%d", block->block_id()); | |
| 854 } | |
| 855 if (block->live_kill().at(i)) { | |
| 856 tty->print_cr(" defined in block B%d", block->block_id()); | |
| 857 } | |
| 858 } | |
| 859 } | |
| 860 } | |
| 861 | |
| 862 #endif | |
| 863 // when this fails, virtual registers are used before they are defined. | |
| 864 assert(false, "live_in set of first block must be empty"); | |
| 865 // bailout of if this occurs in product mode. | |
| 866 bailout("live_in set of first block not empty"); | |
| 867 } | |
| 868 } | |
| 869 | |
| 870 | |
| 871 // ********** Phase 4: build intervals | |
| 872 // (fills the list _intervals) | |
| 873 | |
| 874 void LinearScan::add_use(Value value, int from, int to, IntervalUseKind use_kind) { | |
| 875 assert(!value->type()->is_illegal(), "if this value is used by the interpreter it shouldn't be of indeterminate type"); | |
| 876 LIR_Opr opr = value->operand(); | |
| 877 Constant* con = value->as_Constant(); | |
| 878 | |
| 879 if ((con == NULL || con->is_pinned()) && opr->is_register()) { | |
| 880 assert(reg_num(opr) == opr->vreg_number() && !is_valid_reg_num(reg_numHi(opr)), "invalid optimization below"); | |
| 881 add_use(opr, from, to, use_kind); | |
| 882 } | |
| 883 } | |
| 884 | |
| 885 | |
| 886 void LinearScan::add_def(LIR_Opr opr, int def_pos, IntervalUseKind use_kind) { | |
| 887 TRACE_LINEAR_SCAN(2, tty->print(" def "); opr->print(tty); tty->print_cr(" def_pos %d (%d)", def_pos, use_kind)); | |
| 888 assert(opr->is_register(), "should not be called otherwise"); | |
| 889 | |
| 890 if (opr->is_virtual_register()) { | |
| 891 assert(reg_num(opr) == opr->vreg_number() && !is_valid_reg_num(reg_numHi(opr)), "invalid optimization below"); | |
| 892 add_def(opr->vreg_number(), def_pos, use_kind, opr->type_register()); | |
| 893 | |
| 894 } else { | |
| 895 int reg = reg_num(opr); | |
| 896 if (is_processed_reg_num(reg)) { | |
| 897 add_def(reg, def_pos, use_kind, opr->type_register()); | |
| 898 } | |
| 899 reg = reg_numHi(opr); | |
| 900 if (is_valid_reg_num(reg) && is_processed_reg_num(reg)) { | |
| 901 add_def(reg, def_pos, use_kind, opr->type_register()); | |
| 902 } | |
| 903 } | |
| 904 } | |
| 905 | |
| 906 void LinearScan::add_use(LIR_Opr opr, int from, int to, IntervalUseKind use_kind) { | |
| 907 TRACE_LINEAR_SCAN(2, tty->print(" use "); opr->print(tty); tty->print_cr(" from %d to %d (%d)", from, to, use_kind)); | |
| 908 assert(opr->is_register(), "should not be called otherwise"); | |
| 909 | |
| 910 if (opr->is_virtual_register()) { | |
| 911 assert(reg_num(opr) == opr->vreg_number() && !is_valid_reg_num(reg_numHi(opr)), "invalid optimization below"); | |
| 912 add_use(opr->vreg_number(), from, to, use_kind, opr->type_register()); | |
| 913 | |
| 914 } else { | |
| 915 int reg = reg_num(opr); | |
| 916 if (is_processed_reg_num(reg)) { | |
| 917 add_use(reg, from, to, use_kind, opr->type_register()); | |
| 918 } | |
| 919 reg = reg_numHi(opr); | |
| 920 if (is_valid_reg_num(reg) && is_processed_reg_num(reg)) { | |
| 921 add_use(reg, from, to, use_kind, opr->type_register()); | |
| 922 } | |
| 923 } | |
| 924 } | |
| 925 | |
| 926 void LinearScan::add_temp(LIR_Opr opr, int temp_pos, IntervalUseKind use_kind) { | |
| 927 TRACE_LINEAR_SCAN(2, tty->print(" temp "); opr->print(tty); tty->print_cr(" temp_pos %d (%d)", temp_pos, use_kind)); | |
| 928 assert(opr->is_register(), "should not be called otherwise"); | |
| 929 | |
| 930 if (opr->is_virtual_register()) { | |
| 931 assert(reg_num(opr) == opr->vreg_number() && !is_valid_reg_num(reg_numHi(opr)), "invalid optimization below"); | |
| 932 add_temp(opr->vreg_number(), temp_pos, use_kind, opr->type_register()); | |
| 933 | |
| 934 } else { | |
| 935 int reg = reg_num(opr); | |
| 936 if (is_processed_reg_num(reg)) { | |
| 937 add_temp(reg, temp_pos, use_kind, opr->type_register()); | |
| 938 } | |
| 939 reg = reg_numHi(opr); | |
| 940 if (is_valid_reg_num(reg) && is_processed_reg_num(reg)) { | |
| 941 add_temp(reg, temp_pos, use_kind, opr->type_register()); | |
| 942 } | |
| 943 } | |
| 944 } | |
| 945 | |
| 946 | |
| 947 void LinearScan::add_def(int reg_num, int def_pos, IntervalUseKind use_kind, BasicType type) { | |
| 948 Interval* interval = interval_at(reg_num); | |
| 949 if (interval != NULL) { | |
| 950 assert(interval->reg_num() == reg_num, "wrong interval"); | |
| 951 | |
| 952 if (type != T_ILLEGAL) { | |
| 953 interval->set_type(type); | |
| 954 } | |
| 955 | |
| 956 Range* r = interval->first(); | |
| 957 if (r->from() <= def_pos) { | |
| 958 // Update the starting point (when a range is first created for a use, its | |
| 959 // start is the beginning of the current block until a def is encountered.) | |
| 960 r->set_from(def_pos); | |
| 961 interval->add_use_pos(def_pos, use_kind); | |
| 962 | |
| 963 } else { | |
| 964 // Dead value - make vacuous interval | |
| 965 // also add use_kind for dead intervals | |
| 966 interval->add_range(def_pos, def_pos + 1); | |
| 967 interval->add_use_pos(def_pos, use_kind); | |
| 968 TRACE_LINEAR_SCAN(2, tty->print_cr("Warning: def of reg %d at %d occurs without use", reg_num, def_pos)); | |
| 969 } | |
| 970 | |
| 971 } else { | |
| 972 // Dead value - make vacuous interval | |
| 973 // also add use_kind for dead intervals | |
| 974 interval = create_interval(reg_num); | |
| 975 if (type != T_ILLEGAL) { | |
| 976 interval->set_type(type); | |
| 977 } | |
| 978 | |
| 979 interval->add_range(def_pos, def_pos + 1); | |
| 980 interval->add_use_pos(def_pos, use_kind); | |
| 981 TRACE_LINEAR_SCAN(2, tty->print_cr("Warning: dead value %d at %d in live intervals", reg_num, def_pos)); | |
| 982 } | |
| 983 | |
| 984 change_spill_definition_pos(interval, def_pos); | |
| 985 if (use_kind == noUse && interval->spill_state() <= startInMemory) { | |
| 986 // detection of method-parameters and roundfp-results | |
| 987 // TODO: move this directly to position where use-kind is computed | |
| 988 interval->set_spill_state(startInMemory); | |
| 989 } | |
| 990 } | |
| 991 | |
| 992 void LinearScan::add_use(int reg_num, int from, int to, IntervalUseKind use_kind, BasicType type) { | |
| 993 Interval* interval = interval_at(reg_num); | |
| 994 if (interval == NULL) { | |
| 995 interval = create_interval(reg_num); | |
| 996 } | |
| 997 assert(interval->reg_num() == reg_num, "wrong interval"); | |
| 998 | |
| 999 if (type != T_ILLEGAL) { | |
| 1000 interval->set_type(type); | |
| 1001 } | |
| 1002 | |
| 1003 interval->add_range(from, to); | |
| 1004 interval->add_use_pos(to, use_kind); | |
| 1005 } | |
| 1006 | |
| 1007 void LinearScan::add_temp(int reg_num, int temp_pos, IntervalUseKind use_kind, BasicType type) { | |
| 1008 Interval* interval = interval_at(reg_num); | |
| 1009 if (interval == NULL) { | |
| 1010 interval = create_interval(reg_num); | |
| 1011 } | |
| 1012 assert(interval->reg_num() == reg_num, "wrong interval"); | |
| 1013 | |
| 1014 if (type != T_ILLEGAL) { | |
| 1015 interval->set_type(type); | |
| 1016 } | |
| 1017 | |
| 1018 interval->add_range(temp_pos, temp_pos + 1); | |
| 1019 interval->add_use_pos(temp_pos, use_kind); | |
| 1020 } | |
| 1021 | |
| 1022 | |
| 1023 // the results of this functions are used for optimizing spilling and reloading | |
| 1024 // if the functions return shouldHaveRegister and the interval is spilled, | |
| 1025 // it is not reloaded to a register. | |
| 1026 IntervalUseKind LinearScan::use_kind_of_output_operand(LIR_Op* op, LIR_Opr opr) { | |
| 1027 if (op->code() == lir_move) { | |
| 1028 assert(op->as_Op1() != NULL, "lir_move must be LIR_Op1"); | |
| 1029 LIR_Op1* move = (LIR_Op1*)op; | |
| 1030 LIR_Opr res = move->result_opr(); | |
| 1031 bool result_in_memory = res->is_virtual() && gen()->is_vreg_flag_set(res->vreg_number(), LIRGenerator::must_start_in_memory); | |
| 1032 | |
| 1033 if (result_in_memory) { | |
| 1034 // Begin of an interval with must_start_in_memory set. | |
| 1035 // This interval will always get a stack slot first, so return noUse. | |
| 1036 return noUse; | |
| 1037 | |
| 1038 } else if (move->in_opr()->is_stack()) { | |
| 1039 // method argument (condition must be equal to handle_method_arguments) | |
| 1040 return noUse; | |
| 1041 | |
| 1042 } else if (move->in_opr()->is_register() && move->result_opr()->is_register()) { | |
| 1043 // Move from register to register | |
| 1044 if (block_of_op_with_id(op->id())->is_set(BlockBegin::osr_entry_flag)) { | |
| 1045 // special handling of phi-function moves inside osr-entry blocks | |
| 1046 // input operand must have a register instead of output operand (leads to better register allocation) | |
| 1047 return shouldHaveRegister; | |
| 1048 } | |
| 1049 } | |
| 1050 } | |
| 1051 | |
| 1052 if (opr->is_virtual() && | |
| 1053 gen()->is_vreg_flag_set(opr->vreg_number(), LIRGenerator::must_start_in_memory)) { | |
| 1054 // result is a stack-slot, so prevent immediate reloading | |
| 1055 return noUse; | |
| 1056 } | |
| 1057 | |
| 1058 // all other operands require a register | |
| 1059 return mustHaveRegister; | |
| 1060 } | |
| 1061 | |
| 1062 IntervalUseKind LinearScan::use_kind_of_input_operand(LIR_Op* op, LIR_Opr opr) { | |
| 1063 if (op->code() == lir_move) { | |
| 1064 assert(op->as_Op1() != NULL, "lir_move must be LIR_Op1"); | |
| 1065 LIR_Op1* move = (LIR_Op1*)op; | |
| 1066 LIR_Opr res = move->result_opr(); | |
| 1067 bool result_in_memory = res->is_virtual() && gen()->is_vreg_flag_set(res->vreg_number(), LIRGenerator::must_start_in_memory); | |
| 1068 | |
| 1069 if (result_in_memory) { | |
| 1070 // Move to an interval with must_start_in_memory set. | |
| 1071 // To avoid moves from stack to stack (not allowed) force the input operand to a register | |
| 1072 return mustHaveRegister; | |
| 1073 | |
| 1074 } else if (move->in_opr()->is_register() && move->result_opr()->is_register()) { | |
| 1075 // Move from register to register | |
| 1076 if (block_of_op_with_id(op->id())->is_set(BlockBegin::osr_entry_flag)) { | |
| 1077 // special handling of phi-function moves inside osr-entry blocks | |
| 1078 // input operand must have a register instead of output operand (leads to better register allocation) | |
| 1079 return mustHaveRegister; | |
| 1080 } | |
| 1081 | |
| 1082 // The input operand is not forced to a register (moves from stack to register are allowed), | |
| 1083 // but it is faster if the input operand is in a register | |
| 1084 return shouldHaveRegister; | |
| 1085 } | |
| 1086 } | |
| 1087 | |
| 1088 | |
| 304 | 1089 #ifdef X86 |
| 0 | 1090 if (op->code() == lir_cmove) { |
| 1091 // conditional moves can handle stack operands | |
| 1092 assert(op->result_opr()->is_register(), "result must always be in a register"); | |
| 1093 return shouldHaveRegister; | |
| 1094 } | |
| 1095 | |
| 1096 // optimizations for second input operand of arithmehtic operations on Intel | |
| 1097 // this operand is allowed to be on the stack in some cases | |
| 1098 BasicType opr_type = opr->type_register(); | |
| 1099 if (opr_type == T_FLOAT || opr_type == T_DOUBLE) { | |
| 1100 if ((UseSSE == 1 && opr_type == T_FLOAT) || UseSSE >= 2) { | |
| 1101 // SSE float instruction (T_DOUBLE only supported with SSE2) | |
| 1102 switch (op->code()) { | |
| 1103 case lir_cmp: | |
| 1104 case lir_add: | |
| 1105 case lir_sub: | |
| 1106 case lir_mul: | |
| 1107 case lir_div: | |
| 1108 { | |
| 1109 assert(op->as_Op2() != NULL, "must be LIR_Op2"); | |
| 1110 LIR_Op2* op2 = (LIR_Op2*)op; | |
| 1111 if (op2->in_opr1() != op2->in_opr2() && op2->in_opr2() == opr) { | |
| 1112 assert((op2->result_opr()->is_register() || op->code() == lir_cmp) && op2->in_opr1()->is_register(), "cannot mark second operand as stack if others are not in register"); | |
| 1113 return shouldHaveRegister; | |
| 1114 } | |
| 1115 } | |
| 1116 } | |
| 1117 } else { | |
| 1118 // FPU stack float instruction | |
| 1119 switch (op->code()) { | |
| 1120 case lir_add: | |
| 1121 case lir_sub: | |
| 1122 case lir_mul: | |
| 1123 case lir_div: | |
| 1124 { | |
| 1125 assert(op->as_Op2() != NULL, "must be LIR_Op2"); | |
| 1126 LIR_Op2* op2 = (LIR_Op2*)op; | |
| 1127 if (op2->in_opr1() != op2->in_opr2() && op2->in_opr2() == opr) { | |
| 1128 assert((op2->result_opr()->is_register() || op->code() == lir_cmp) && op2->in_opr1()->is_register(), "cannot mark second operand as stack if others are not in register"); | |
| 1129 return shouldHaveRegister; | |
| 1130 } | |
| 1131 } | |
| 1132 } | |
| 1133 } | |
| 1134 | |
| 1135 } else if (opr_type != T_LONG) { | |
| 1136 // integer instruction (note: long operands must always be in register) | |
| 1137 switch (op->code()) { | |
| 1138 case lir_cmp: | |
| 1139 case lir_add: | |
| 1140 case lir_sub: | |
| 1141 case lir_logic_and: | |
| 1142 case lir_logic_or: | |
| 1143 case lir_logic_xor: | |
| 1144 { | |
| 1145 assert(op->as_Op2() != NULL, "must be LIR_Op2"); | |
| 1146 LIR_Op2* op2 = (LIR_Op2*)op; | |
| 1147 if (op2->in_opr1() != op2->in_opr2() && op2->in_opr2() == opr) { | |
| 1148 assert((op2->result_opr()->is_register() || op->code() == lir_cmp) && op2->in_opr1()->is_register(), "cannot mark second operand as stack if others are not in register"); | |
| 1149 return shouldHaveRegister; | |
| 1150 } | |
| 1151 } | |
| 1152 } | |
| 1153 } | |
| 304 | 1154 #endif // X86 |
| 0 | 1155 |
| 1156 // all other operands require a register | |
| 1157 return mustHaveRegister; | |
| 1158 } | |
| 1159 | |
| 1160 | |
| 1161 void LinearScan::handle_method_arguments(LIR_Op* op) { | |
| 1162 // special handling for method arguments (moves from stack to virtual register): | |
| 1163 // the interval gets no register assigned, but the stack slot. | |
| 1164 // it is split before the first use by the register allocator. | |
| 1165 | |
| 1166 if (op->code() == lir_move) { | |
| 1167 assert(op->as_Op1() != NULL, "must be LIR_Op1"); | |
| 1168 LIR_Op1* move = (LIR_Op1*)op; | |
| 1169 | |
| 1170 if (move->in_opr()->is_stack()) { | |
| 1171 #ifdef ASSERT | |
| 1172 int arg_size = compilation()->method()->arg_size(); | |
| 1173 LIR_Opr o = move->in_opr(); | |
| 1174 if (o->is_single_stack()) { | |
| 1175 assert(o->single_stack_ix() >= 0 && o->single_stack_ix() < arg_size, "out of range"); | |
| 1176 } else if (o->is_double_stack()) { | |
| 1177 assert(o->double_stack_ix() >= 0 && o->double_stack_ix() < arg_size, "out of range"); | |
| 1178 } else { | |
| 1179 ShouldNotReachHere(); | |
| 1180 } | |
| 1181 | |
| 1182 assert(move->id() > 0, "invalid id"); | |
| 1183 assert(block_of_op_with_id(move->id())->number_of_preds() == 0, "move from stack must be in first block"); | |
| 1184 assert(move->result_opr()->is_virtual(), "result of move must be a virtual register"); | |
| 1185 | |
| 1186 TRACE_LINEAR_SCAN(4, tty->print_cr("found move from stack slot %d to vreg %d", o->is_single_stack() ? o->single_stack_ix() : o->double_stack_ix(), reg_num(move->result_opr()))); | |
| 1187 #endif | |
| 1188 | |
| 1189 Interval* interval = interval_at(reg_num(move->result_opr())); | |
| 1190 | |
| 1191 int stack_slot = LinearScan::nof_regs + (move->in_opr()->is_single_stack() ? move->in_opr()->single_stack_ix() : move->in_opr()->double_stack_ix()); | |
| 1192 interval->set_canonical_spill_slot(stack_slot); | |
| 1193 interval->assign_reg(stack_slot); | |
| 1194 } | |
| 1195 } | |
| 1196 } | |
| 1197 | |
| 1198 void LinearScan::handle_doubleword_moves(LIR_Op* op) { | |
| 1199 // special handling for doubleword move from memory to register: | |
| 1200 // in this case the registers of the input address and the result | |
| 1201 // registers must not overlap -> add a temp range for the input registers | |
| 1202 if (op->code() == lir_move) { | |
| 1203 assert(op->as_Op1() != NULL, "must be LIR_Op1"); | |
| 1204 LIR_Op1* move = (LIR_Op1*)op; | |
| 1205 | |
| 1206 if (move->result_opr()->is_double_cpu() && move->in_opr()->is_pointer()) { | |
| 1207 LIR_Address* address = move->in_opr()->as_address_ptr(); | |
| 1208 if (address != NULL) { | |
| 1209 if (address->base()->is_valid()) { | |
| 1210 add_temp(address->base(), op->id(), noUse); | |
| 1211 } | |
| 1212 if (address->index()->is_valid()) { | |
| 1213 add_temp(address->index(), op->id(), noUse); | |
| 1214 } | |
| 1215 } | |
| 1216 } | |
| 1217 } | |
| 1218 } | |
| 1219 | |
| 1220 void LinearScan::add_register_hints(LIR_Op* op) { | |
| 1221 switch (op->code()) { | |
| 1222 case lir_move: // fall through | |
| 1223 case lir_convert: { | |
| 1224 assert(op->as_Op1() != NULL, "lir_move, lir_convert must be LIR_Op1"); | |
| 1225 LIR_Op1* move = (LIR_Op1*)op; | |
| 1226 | |
| 1227 LIR_Opr move_from = move->in_opr(); | |
| 1228 LIR_Opr move_to = move->result_opr(); | |
| 1229 | |
| 1230 if (move_to->is_register() && move_from->is_register()) { | |
| 1231 Interval* from = interval_at(reg_num(move_from)); | |
| 1232 Interval* to = interval_at(reg_num(move_to)); | |
| 1233 if (from != NULL && to != NULL) { | |
| 1234 to->set_register_hint(from); | |
| 1235 TRACE_LINEAR_SCAN(4, tty->print_cr("operation at op_id %d: added hint from interval %d to %d", move->id(), from->reg_num(), to->reg_num())); | |
| 1236 } | |
| 1237 } | |
| 1238 break; | |
| 1239 } | |
| 1240 case lir_cmove: { | |
| 1241 assert(op->as_Op2() != NULL, "lir_cmove must be LIR_Op2"); | |
| 1242 LIR_Op2* cmove = (LIR_Op2*)op; | |
| 1243 | |
| 1244 LIR_Opr move_from = cmove->in_opr1(); | |
| 1245 LIR_Opr move_to = cmove->result_opr(); | |
| 1246 | |
| 1247 if (move_to->is_register() && move_from->is_register()) { | |
| 1248 Interval* from = interval_at(reg_num(move_from)); | |
| 1249 Interval* to = interval_at(reg_num(move_to)); | |
| 1250 if (from != NULL && to != NULL) { | |
| 1251 to->set_register_hint(from); | |
| 1252 TRACE_LINEAR_SCAN(4, tty->print_cr("operation at op_id %d: added hint from interval %d to %d", cmove->id(), from->reg_num(), to->reg_num())); | |
| 1253 } | |
| 1254 } | |
| 1255 break; | |
| 1256 } | |
| 1257 } | |
| 1258 } | |
| 1259 | |
| 1260 | |
| 1261 void LinearScan::build_intervals() { | |
| 1262 TIME_LINEAR_SCAN(timer_build_intervals); | |
| 1263 | |
| 1264 // initialize interval list with expected number of intervals | |
| 1265 // (32 is added to have some space for split children without having to resize the list) | |
| 1266 _intervals = IntervalList(num_virtual_regs() + 32); | |
| 1267 // initialize all slots that are used by build_intervals | |
| 1268 _intervals.at_put_grow(num_virtual_regs() - 1, NULL, NULL); | |
| 1269 | |
| 1270 // create a list with all caller-save registers (cpu, fpu, xmm) | |
| 1271 // when an instruction is a call, a temp range is created for all these registers | |
| 1272 int num_caller_save_registers = 0; | |
| 1273 int caller_save_registers[LinearScan::nof_regs]; | |
| 1274 | |
| 1275 int i; | |
| 1276 for (i = 0; i < FrameMap::nof_caller_save_cpu_regs; i++) { | |
| 1277 LIR_Opr opr = FrameMap::caller_save_cpu_reg_at(i); | |
| 1278 assert(opr->is_valid() && opr->is_register(), "FrameMap should not return invalid operands"); | |
| 1279 assert(reg_numHi(opr) == -1, "missing addition of range for hi-register"); | |
| 1280 caller_save_registers[num_caller_save_registers++] = reg_num(opr); | |
| 1281 } | |
| 1282 | |
| 1283 // temp ranges for fpu registers are only created when the method has | |
| 1284 // virtual fpu operands. Otherwise no allocation for fpu registers is | |
| 1285 // perfomed and so the temp ranges would be useless | |
| 1286 if (has_fpu_registers()) { | |
| 304 | 1287 #ifdef X86 |
| 0 | 1288 if (UseSSE < 2) { |
| 1289 #endif | |
| 1290 for (i = 0; i < FrameMap::nof_caller_save_fpu_regs; i++) { | |
| 1291 LIR_Opr opr = FrameMap::caller_save_fpu_reg_at(i); | |
| 1292 assert(opr->is_valid() && opr->is_register(), "FrameMap should not return invalid operands"); | |
| 1293 assert(reg_numHi(opr) == -1, "missing addition of range for hi-register"); | |
| 1294 caller_save_registers[num_caller_save_registers++] = reg_num(opr); | |
| 1295 } | |
| 304 | 1296 #ifdef X86 |
| 0 | 1297 } |
| 1298 if (UseSSE > 0) { | |
| 1299 for (i = 0; i < FrameMap::nof_caller_save_xmm_regs; i++) { | |
| 1300 LIR_Opr opr = FrameMap::caller_save_xmm_reg_at(i); | |
| 1301 assert(opr->is_valid() && opr->is_register(), "FrameMap should not return invalid operands"); | |
| 1302 assert(reg_numHi(opr) == -1, "missing addition of range for hi-register"); | |
| 1303 caller_save_registers[num_caller_save_registers++] = reg_num(opr); | |
| 1304 } | |
| 1305 } | |
| 1306 #endif | |
| 1307 } | |
| 1308 assert(num_caller_save_registers <= LinearScan::nof_regs, "out of bounds"); | |
| 1309 | |
| 1310 | |
| 1311 LIR_OpVisitState visitor; | |
| 1312 | |
| 1313 // iterate all blocks in reverse order | |
| 1314 for (i = block_count() - 1; i >= 0; i--) { | |
| 1315 BlockBegin* block = block_at(i); | |
| 1316 LIR_OpList* instructions = block->lir()->instructions_list(); | |
| 1317 int block_from = block->first_lir_instruction_id(); | |
| 1318 int block_to = block->last_lir_instruction_id(); | |
| 1319 | |
| 1320 assert(block_from == instructions->at(0)->id(), "must be"); | |
| 1321 assert(block_to == instructions->at(instructions->length() - 1)->id(), "must be"); | |
| 1322 | |
| 1323 // Update intervals for registers live at the end of this block; | |
| 1324 BitMap live = block->live_out(); | |
| 304 | 1325 int size = (int)live.size(); |
| 1326 for (int number = (int)live.get_next_one_offset(0, size); number < size; number = (int)live.get_next_one_offset(number + 1, size)) { | |
| 0 | 1327 assert(live.at(number), "should not stop here otherwise"); |
| 1328 assert(number >= LIR_OprDesc::vreg_base, "fixed intervals must not be live on block bounds"); | |
| 1329 TRACE_LINEAR_SCAN(2, tty->print_cr("live in %d to %d", number, block_to + 2)); | |
| 1330 | |
| 1331 add_use(number, block_from, block_to + 2, noUse, T_ILLEGAL); | |
| 1332 | |
| 1333 // add special use positions for loop-end blocks when the | |
| 1334 // interval is used anywhere inside this loop. It's possible | |
| 1335 // that the block was part of a non-natural loop, so it might | |
| 1336 // have an invalid loop index. | |
| 1337 if (block->is_set(BlockBegin::linear_scan_loop_end_flag) && | |
| 1338 block->loop_index() != -1 && | |
| 1339 is_interval_in_loop(number, block->loop_index())) { | |
| 1340 interval_at(number)->add_use_pos(block_to + 1, loopEndMarker); | |
| 1341 } | |
| 1342 } | |
| 1343 | |
| 1344 // iterate all instructions of the block in reverse order. | |
| 1345 // skip the first instruction because it is always a label | |
| 1346 // definitions of intervals are processed before uses | |
| 1347 assert(visitor.no_operands(instructions->at(0)), "first operation must always be a label"); | |
| 1348 for (int j = instructions->length() - 1; j >= 1; j--) { | |
| 1349 LIR_Op* op = instructions->at(j); | |
| 1350 int op_id = op->id(); | |
| 1351 | |
| 1352 // visit operation to collect all operands | |
| 1353 visitor.visit(op); | |
| 1354 | |
| 1355 // add a temp range for each register if operation destroys caller-save registers | |
| 1356 if (visitor.has_call()) { | |
| 1357 for (int k = 0; k < num_caller_save_registers; k++) { | |
| 1358 add_temp(caller_save_registers[k], op_id, noUse, T_ILLEGAL); | |
| 1359 } | |
| 1360 TRACE_LINEAR_SCAN(4, tty->print_cr("operation destroys all caller-save registers")); | |
| 1361 } | |
| 1362 | |
| 1363 // Add any platform dependent temps | |
| 1364 pd_add_temps(op); | |
| 1365 | |
| 1366 // visit definitions (output and temp operands) | |
| 1367 int k, n; | |
| 1368 n = visitor.opr_count(LIR_OpVisitState::outputMode); | |
| 1369 for (k = 0; k < n; k++) { | |
| 1370 LIR_Opr opr = visitor.opr_at(LIR_OpVisitState::outputMode, k); | |
| 1371 assert(opr->is_register(), "visitor should only return register operands"); | |
| 1372 add_def(opr, op_id, use_kind_of_output_operand(op, opr)); | |
| 1373 } | |
| 1374 | |
| 1375 n = visitor.opr_count(LIR_OpVisitState::tempMode); | |
| 1376 for (k = 0; k < n; k++) { | |
| 1377 LIR_Opr opr = visitor.opr_at(LIR_OpVisitState::tempMode, k); | |
| 1378 assert(opr->is_register(), "visitor should only return register operands"); | |
| 1379 add_temp(opr, op_id, mustHaveRegister); | |
| 1380 } | |
| 1381 | |
| 1382 // visit uses (input operands) | |
| 1383 n = visitor.opr_count(LIR_OpVisitState::inputMode); | |
| 1384 for (k = 0; k < n; k++) { | |
| 1385 LIR_Opr opr = visitor.opr_at(LIR_OpVisitState::inputMode, k); | |
| 1386 assert(opr->is_register(), "visitor should only return register operands"); | |
| 1387 add_use(opr, block_from, op_id, use_kind_of_input_operand(op, opr)); | |
| 1388 } | |
| 1389 | |
| 1390 // Add uses of live locals from interpreter's point of view for proper | |
| 1391 // debug information generation | |
| 1392 // Treat these operands as temp values (if the life range is extended | |
| 1393 // to a call site, the value would be in a register at the call otherwise) | |
| 1394 n = visitor.info_count(); | |
| 1395 for (k = 0; k < n; k++) { | |
| 1396 CodeEmitInfo* info = visitor.info_at(k); | |
| 1397 ValueStack* stack = info->stack(); | |
| 1398 for_each_state_value(stack, value, | |
| 1399 add_use(value, block_from, op_id + 1, noUse); | |
| 1400 ); | |
| 1401 } | |
| 1402 | |
| 1403 // special steps for some instructions (especially moves) | |
| 1404 handle_method_arguments(op); | |
| 1405 handle_doubleword_moves(op); | |
| 1406 add_register_hints(op); | |
| 1407 | |
| 1408 } // end of instruction iteration | |
| 1409 } // end of block iteration | |
| 1410 | |
| 1411 | |
| 1412 // add the range [0, 1[ to all fixed intervals | |
| 1413 // -> the register allocator need not handle unhandled fixed intervals | |
| 1414 for (int n = 0; n < LinearScan::nof_regs; n++) { | |
| 1415 Interval* interval = interval_at(n); | |
| 1416 if (interval != NULL) { | |
| 1417 interval->add_range(0, 1); | |
| 1418 } | |
| 1419 } | |
| 1420 } | |
| 1421 | |
| 1422 | |
| 1423 // ********** Phase 5: actual register allocation | |
| 1424 | |
| 1425 int LinearScan::interval_cmp(Interval** a, Interval** b) { | |
| 1426 if (*a != NULL) { | |
| 1427 if (*b != NULL) { | |
| 1428 return (*a)->from() - (*b)->from(); | |
| 1429 } else { | |
| 1430 return -1; | |
| 1431 } | |
| 1432 } else { | |
| 1433 if (*b != NULL) { | |
| 1434 return 1; | |
| 1435 } else { | |
| 1436 return 0; | |
| 1437 } | |
| 1438 } | |
| 1439 } | |
| 1440 | |
| 1441 #ifndef PRODUCT | |
| 1442 bool LinearScan::is_sorted(IntervalArray* intervals) { | |
| 1443 int from = -1; | |
| 1444 int i, j; | |
| 1445 for (i = 0; i < intervals->length(); i ++) { | |
| 1446 Interval* it = intervals->at(i); | |
| 1447 if (it != NULL) { | |
| 1448 if (from > it->from()) { | |
| 1449 assert(false, ""); | |
| 1450 return false; | |
| 1451 } | |
| 1452 from = it->from(); | |
| 1453 } | |
| 1454 } | |
| 1455 | |
| 1456 // check in both directions if sorted list and unsorted list contain same intervals | |
| 1457 for (i = 0; i < interval_count(); i++) { | |
| 1458 if (interval_at(i) != NULL) { | |
| 1459 int num_found = 0; | |
| 1460 for (j = 0; j < intervals->length(); j++) { | |
| 1461 if (interval_at(i) == intervals->at(j)) { | |
| 1462 num_found++; | |
| 1463 } | |
| 1464 } | |
| 1465 assert(num_found == 1, "lists do not contain same intervals"); | |
| 1466 } | |
| 1467 } | |
| 1468 for (j = 0; j < intervals->length(); j++) { | |
| 1469 int num_found = 0; | |
| 1470 for (i = 0; i < interval_count(); i++) { | |
| 1471 if (interval_at(i) == intervals->at(j)) { | |
| 1472 num_found++; | |
| 1473 } | |
| 1474 } | |
| 1475 assert(num_found == 1, "lists do not contain same intervals"); | |
| 1476 } | |
| 1477 | |
| 1478 return true; | |
| 1479 } | |
| 1480 #endif | |
| 1481 | |
| 1482 void LinearScan::add_to_list(Interval** first, Interval** prev, Interval* interval) { | |
| 1483 if (*prev != NULL) { | |
| 1484 (*prev)->set_next(interval); | |
| 1485 } else { | |
| 1486 *first = interval; | |
| 1487 } | |
| 1488 *prev = interval; | |
| 1489 } | |
| 1490 | |
| 1491 void LinearScan::create_unhandled_lists(Interval** list1, Interval** list2, bool (is_list1)(const Interval* i), bool (is_list2)(const Interval* i)) { | |
| 1492 assert(is_sorted(_sorted_intervals), "interval list is not sorted"); | |
| 1493 | |
| 1494 *list1 = *list2 = Interval::end(); | |
| 1495 | |
| 1496 Interval* list1_prev = NULL; | |
| 1497 Interval* list2_prev = NULL; | |
| 1498 Interval* v; | |
| 1499 | |
| 1500 const int n = _sorted_intervals->length(); | |
| 1501 for (int i = 0; i < n; i++) { | |
| 1502 v = _sorted_intervals->at(i); | |
| 1503 if (v == NULL) continue; | |
| 1504 | |
| 1505 if (is_list1(v)) { | |
| 1506 add_to_list(list1, &list1_prev, v); | |
| 1507 } else if (is_list2 == NULL || is_list2(v)) { | |
| 1508 add_to_list(list2, &list2_prev, v); | |
| 1509 } | |
| 1510 } | |
| 1511 | |
| 1512 if (list1_prev != NULL) list1_prev->set_next(Interval::end()); | |
| 1513 if (list2_prev != NULL) list2_prev->set_next(Interval::end()); | |
| 1514 | |
| 1515 assert(list1_prev == NULL || list1_prev->next() == Interval::end(), "linear list ends not with sentinel"); | |
| 1516 assert(list2_prev == NULL || list2_prev->next() == Interval::end(), "linear list ends not with sentinel"); | |
| 1517 } | |
| 1518 | |
| 1519 | |
| 1520 void LinearScan::sort_intervals_before_allocation() { | |
| 1521 TIME_LINEAR_SCAN(timer_sort_intervals_before); | |
| 1522 | |
| 1523 IntervalList* unsorted_list = &_intervals; | |
| 1524 int unsorted_len = unsorted_list->length(); | |
| 1525 int sorted_len = 0; | |
| 1526 int unsorted_idx; | |
| 1527 int sorted_idx = 0; | |
| 1528 int sorted_from_max = -1; | |
| 1529 | |
| 1530 // calc number of items for sorted list (sorted list must not contain NULL values) | |
| 1531 for (unsorted_idx = 0; unsorted_idx < unsorted_len; unsorted_idx++) { | |
| 1532 if (unsorted_list->at(unsorted_idx) != NULL) { | |
| 1533 sorted_len++; | |
| 1534 } | |
| 1535 } | |
| 1536 IntervalArray* sorted_list = new IntervalArray(sorted_len); | |
| 1537 | |
| 1538 // special sorting algorithm: the original interval-list is almost sorted, | |
| 1539 // only some intervals are swapped. So this is much faster than a complete QuickSort | |
| 1540 for (unsorted_idx = 0; unsorted_idx < unsorted_len; unsorted_idx++) { | |
| 1541 Interval* cur_interval = unsorted_list->at(unsorted_idx); | |
| 1542 | |
| 1543 if (cur_interval != NULL) { | |
| 1544 int cur_from = cur_interval->from(); | |
| 1545 | |
| 1546 if (sorted_from_max <= cur_from) { | |
| 1547 sorted_list->at_put(sorted_idx++, cur_interval); | |
| 1548 sorted_from_max = cur_interval->from(); | |
| 1549 } else { | |
| 1550 // the asumption that the intervals are already sorted failed, | |
| 1551 // so this interval must be sorted in manually | |
| 1552 int j; | |
| 1553 for (j = sorted_idx - 1; j >= 0 && cur_from < sorted_list->at(j)->from(); j--) { | |
| 1554 sorted_list->at_put(j + 1, sorted_list->at(j)); | |
| 1555 } | |
| 1556 sorted_list->at_put(j + 1, cur_interval); | |
| 1557 sorted_idx++; | |
| 1558 } | |
| 1559 } | |
| 1560 } | |
| 1561 _sorted_intervals = sorted_list; | |
| 1562 } | |
| 1563 | |
| 1564 void LinearScan::sort_intervals_after_allocation() { | |
| 1565 TIME_LINEAR_SCAN(timer_sort_intervals_after); | |
| 1566 | |
| 1567 IntervalArray* old_list = _sorted_intervals; | |
| 1568 IntervalList* new_list = _new_intervals_from_allocation; | |
| 1569 int old_len = old_list->length(); | |
| 1570 int new_len = new_list->length(); | |
| 1571 | |
| 1572 if (new_len == 0) { | |
| 1573 // no intervals have been added during allocation, so sorted list is already up to date | |
| 1574 return; | |
| 1575 } | |
| 1576 | |
| 1577 // conventional sort-algorithm for new intervals | |
| 1578 new_list->sort(interval_cmp); | |
| 1579 | |
| 1580 // merge old and new list (both already sorted) into one combined list | |
| 1581 IntervalArray* combined_list = new IntervalArray(old_len + new_len); | |
| 1582 int old_idx = 0; | |
| 1583 int new_idx = 0; | |
| 1584 | |
| 1585 while (old_idx + new_idx < old_len + new_len) { | |
| 1586 if (new_idx >= new_len || (old_idx < old_len && old_list->at(old_idx)->from() <= new_list->at(new_idx)->from())) { | |
| 1587 combined_list->at_put(old_idx + new_idx, old_list->at(old_idx)); | |
| 1588 old_idx++; | |
| 1589 } else { | |
| 1590 combined_list->at_put(old_idx + new_idx, new_list->at(new_idx)); | |
| 1591 new_idx++; | |
| 1592 } | |
| 1593 } | |
| 1594 | |
| 1595 _sorted_intervals = combined_list; | |
| 1596 } | |
| 1597 | |
| 1598 | |
| 1599 void LinearScan::allocate_registers() { | |
| 1600 TIME_LINEAR_SCAN(timer_allocate_registers); | |
| 1601 | |
| 1602 Interval* precolored_cpu_intervals, *not_precolored_cpu_intervals; | |
| 1603 Interval* precolored_fpu_intervals, *not_precolored_fpu_intervals; | |
| 1604 | |
| 1605 create_unhandled_lists(&precolored_cpu_intervals, ¬_precolored_cpu_intervals, is_precolored_cpu_interval, is_virtual_cpu_interval); | |
| 1606 if (has_fpu_registers()) { | |
| 1607 create_unhandled_lists(&precolored_fpu_intervals, ¬_precolored_fpu_intervals, is_precolored_fpu_interval, is_virtual_fpu_interval); | |
| 1608 #ifdef ASSERT | |
| 1609 } else { | |
| 1610 // fpu register allocation is omitted because no virtual fpu registers are present | |
| 1611 // just check this again... | |
| 1612 create_unhandled_lists(&precolored_fpu_intervals, ¬_precolored_fpu_intervals, is_precolored_fpu_interval, is_virtual_fpu_interval); | |
| 1613 assert(not_precolored_fpu_intervals == Interval::end(), "missed an uncolored fpu interval"); | |
| 1614 #endif | |
| 1615 } | |
| 1616 | |
| 1617 // allocate cpu registers | |
| 1618 LinearScanWalker cpu_lsw(this, precolored_cpu_intervals, not_precolored_cpu_intervals); | |
| 1619 cpu_lsw.walk(); | |
| 1620 cpu_lsw.finish_allocation(); | |
| 1621 | |
| 1622 if (has_fpu_registers()) { | |
| 1623 // allocate fpu registers | |
| 1624 LinearScanWalker fpu_lsw(this, precolored_fpu_intervals, not_precolored_fpu_intervals); | |
| 1625 fpu_lsw.walk(); | |
| 1626 fpu_lsw.finish_allocation(); | |
| 1627 } | |
| 1628 } | |
| 1629 | |
| 1630 | |
| 1631 // ********** Phase 6: resolve data flow | |
| 1632 // (insert moves at edges between blocks if intervals have been split) | |
| 1633 | |
| 1634 // wrapper for Interval::split_child_at_op_id that performs a bailout in product mode | |
| 1635 // instead of returning NULL | |
| 1636 Interval* LinearScan::split_child_at_op_id(Interval* interval, int op_id, LIR_OpVisitState::OprMode mode) { | |
| 1637 Interval* result = interval->split_child_at_op_id(op_id, mode); | |
| 1638 if (result != NULL) { | |
| 1639 return result; | |
| 1640 } | |
| 1641 | |
| 1642 assert(false, "must find an interval, but do a clean bailout in product mode"); | |
| 1643 result = new Interval(LIR_OprDesc::vreg_base); | |
| 1644 result->assign_reg(0); | |
| 1645 result->set_type(T_INT); | |
| 1646 BAILOUT_("LinearScan: interval is NULL", result); | |
| 1647 } | |
| 1648 | |
| 1649 | |
| 1650 Interval* LinearScan::interval_at_block_begin(BlockBegin* block, int reg_num) { | |
| 1651 assert(LinearScan::nof_regs <= reg_num && reg_num < num_virtual_regs(), "register number out of bounds"); | |
| 1652 assert(interval_at(reg_num) != NULL, "no interval found"); | |
| 1653 | |
| 1654 return split_child_at_op_id(interval_at(reg_num), block->first_lir_instruction_id(), LIR_OpVisitState::outputMode); | |
| 1655 } | |
| 1656 | |
| 1657 Interval* LinearScan::interval_at_block_end(BlockBegin* block, int reg_num) { | |
| 1658 assert(LinearScan::nof_regs <= reg_num && reg_num < num_virtual_regs(), "register number out of bounds"); | |
| 1659 assert(interval_at(reg_num) != NULL, "no interval found"); | |
| 1660 | |
| 1661 return split_child_at_op_id(interval_at(reg_num), block->last_lir_instruction_id() + 1, LIR_OpVisitState::outputMode); | |
| 1662 } | |
| 1663 | |
| 1664 Interval* LinearScan::interval_at_op_id(int reg_num, int op_id) { | |
| 1665 assert(LinearScan::nof_regs <= reg_num && reg_num < num_virtual_regs(), "register number out of bounds"); | |
| 1666 assert(interval_at(reg_num) != NULL, "no interval found"); | |
| 1667 | |
| 1668 return split_child_at_op_id(interval_at(reg_num), op_id, LIR_OpVisitState::inputMode); | |
| 1669 } | |
| 1670 | |
| 1671 | |
| 1672 void LinearScan::resolve_collect_mappings(BlockBegin* from_block, BlockBegin* to_block, MoveResolver &move_resolver) { | |
| 1673 DEBUG_ONLY(move_resolver.check_empty()); | |
| 1674 | |
| 1675 const int num_regs = num_virtual_regs(); | |
| 1676 const int size = live_set_size(); | |
| 1677 const BitMap live_at_edge = to_block->live_in(); | |
| 1678 | |
| 1679 // visit all registers where the live_at_edge bit is set | |
| 304 | 1680 for (int r = (int)live_at_edge.get_next_one_offset(0, size); r < size; r = (int)live_at_edge.get_next_one_offset(r + 1, size)) { |
| 0 | 1681 assert(r < num_regs, "live information set for not exisiting interval"); |
| 1682 assert(from_block->live_out().at(r) && to_block->live_in().at(r), "interval not live at this edge"); | |
| 1683 | |
| 1684 Interval* from_interval = interval_at_block_end(from_block, r); | |
| 1685 Interval* to_interval = interval_at_block_begin(to_block, r); | |
| 1686 | |
| 1687 if (from_interval != to_interval && (from_interval->assigned_reg() != to_interval->assigned_reg() || from_interval->assigned_regHi() != to_interval->assigned_regHi())) { | |
| 1688 // need to insert move instruction | |
| 1689 move_resolver.add_mapping(from_interval, to_interval); | |
| 1690 } | |
| 1691 } | |
| 1692 } | |
| 1693 | |
| 1694 | |
| 1695 void LinearScan::resolve_find_insert_pos(BlockBegin* from_block, BlockBegin* to_block, MoveResolver &move_resolver) { | |
| 1696 if (from_block->number_of_sux() <= 1) { | |
| 1697 TRACE_LINEAR_SCAN(4, tty->print_cr("inserting moves at end of from_block B%d", from_block->block_id())); | |
| 1698 | |
| 1699 LIR_OpList* instructions = from_block->lir()->instructions_list(); | |
| 1700 LIR_OpBranch* branch = instructions->last()->as_OpBranch(); | |
| 1701 if (branch != NULL) { | |
| 1702 // insert moves before branch | |
| 1703 assert(branch->cond() == lir_cond_always, "block does not end with an unconditional jump"); | |
| 1704 move_resolver.set_insert_position(from_block->lir(), instructions->length() - 2); | |
| 1705 } else { | |
| 1706 move_resolver.set_insert_position(from_block->lir(), instructions->length() - 1); | |
| 1707 } | |
| 1708 | |
| 1709 } else { | |
| 1710 TRACE_LINEAR_SCAN(4, tty->print_cr("inserting moves at beginning of to_block B%d", to_block->block_id())); | |
| 1711 #ifdef ASSERT | |
| 1712 assert(from_block->lir()->instructions_list()->at(0)->as_OpLabel() != NULL, "block does not start with a label"); | |
| 1713 | |
| 1714 // because the number of predecessor edges matches the number of | |
| 1715 // successor edges, blocks which are reached by switch statements | |
| 1716 // may have be more than one predecessor but it will be guaranteed | |
| 1717 // that all predecessors will be the same. | |
| 1718 for (int i = 0; i < to_block->number_of_preds(); i++) { | |
| 1719 assert(from_block == to_block->pred_at(i), "all critical edges must be broken"); | |
| 1720 } | |
| 1721 #endif | |
| 1722 | |
| 1723 move_resolver.set_insert_position(to_block->lir(), 0); | |
| 1724 } | |
| 1725 } | |
| 1726 | |
| 1727 | |
| 1728 // insert necessary moves (spilling or reloading) at edges between blocks if interval has been split | |
| 1729 void LinearScan::resolve_data_flow() { | |
| 1730 TIME_LINEAR_SCAN(timer_resolve_data_flow); | |
| 1731 | |
| 1732 int num_blocks = block_count(); | |
| 1733 MoveResolver move_resolver(this); | |
| 1734 BitMap block_completed(num_blocks); block_completed.clear(); | |
| 1735 BitMap already_resolved(num_blocks); already_resolved.clear(); | |
| 1736 | |
| 1737 int i; | |
| 1738 for (i = 0; i < num_blocks; i++) { | |
| 1739 BlockBegin* block = block_at(i); | |
| 1740 | |
| 1741 // check if block has only one predecessor and only one successor | |
| 1742 if (block->number_of_preds() == 1 && block->number_of_sux() == 1 && block->number_of_exception_handlers() == 0) { | |
| 1743 LIR_OpList* instructions = block->lir()->instructions_list(); | |
| 1744 assert(instructions->at(0)->code() == lir_label, "block must start with label"); | |
| 1745 assert(instructions->last()->code() == lir_branch, "block with successors must end with branch"); | |
| 1746 assert(instructions->last()->as_OpBranch()->cond() == lir_cond_always, "block with successor must end with unconditional branch"); | |
| 1747 | |
| 1748 // check if block is empty (only label and branch) | |
| 1749 if (instructions->length() == 2) { | |
| 1750 BlockBegin* pred = block->pred_at(0); | |
| 1751 BlockBegin* sux = block->sux_at(0); | |
| 1752 | |
| 1753 // prevent optimization of two consecutive blocks | |
| 1754 if (!block_completed.at(pred->linear_scan_number()) && !block_completed.at(sux->linear_scan_number())) { | |
| 1755 TRACE_LINEAR_SCAN(3, tty->print_cr("**** optimizing empty block B%d (pred: B%d, sux: B%d)", block->block_id(), pred->block_id(), sux->block_id())); | |
| 1756 block_completed.set_bit(block->linear_scan_number()); | |
| 1757 | |
| 1758 // directly resolve between pred and sux (without looking at the empty block between) | |
| 1759 resolve_collect_mappings(pred, sux, move_resolver); | |
| 1760 if (move_resolver.has_mappings()) { | |
| 1761 move_resolver.set_insert_position(block->lir(), 0); | |
| 1762 move_resolver.resolve_and_append_moves(); | |
| 1763 } | |
| 1764 } | |
| 1765 } | |
| 1766 } | |
| 1767 } | |
| 1768 | |
| 1769 | |
| 1770 for (i = 0; i < num_blocks; i++) { | |
| 1771 if (!block_completed.at(i)) { | |
| 1772 BlockBegin* from_block = block_at(i); | |
| 1773 already_resolved.set_from(block_completed); | |
| 1774 | |
| 1775 int num_sux = from_block->number_of_sux(); | |
| 1776 for (int s = 0; s < num_sux; s++) { | |
| 1777 BlockBegin* to_block = from_block->sux_at(s); | |
| 1778 | |
| 1779 // check for duplicate edges between the same blocks (can happen with switch blocks) | |
| 1780 if (!already_resolved.at(to_block->linear_scan_number())) { | |
| 1781 TRACE_LINEAR_SCAN(3, tty->print_cr("**** processing edge between B%d and B%d", from_block->block_id(), to_block->block_id())); | |
| 1782 already_resolved.set_bit(to_block->linear_scan_number()); | |
| 1783 | |
| 1784 // collect all intervals that have been split between from_block and to_block | |
| 1785 resolve_collect_mappings(from_block, to_block, move_resolver); | |
| 1786 if (move_resolver.has_mappings()) { | |
| 1787 resolve_find_insert_pos(from_block, to_block, move_resolver); | |
| 1788 move_resolver.resolve_and_append_moves(); | |
| 1789 } | |
| 1790 } | |
| 1791 } | |
| 1792 } | |
| 1793 } | |
| 1794 } | |
| 1795 | |
| 1796 | |
| 1797 void LinearScan::resolve_exception_entry(BlockBegin* block, int reg_num, MoveResolver &move_resolver) { | |
| 1798 if (interval_at(reg_num) == NULL) { | |
| 1799 // if a phi function is never used, no interval is created -> ignore this | |
| 1800 return; | |
| 1801 } | |
| 1802 | |
| 1803 Interval* interval = interval_at_block_begin(block, reg_num); | |
| 1804 int reg = interval->assigned_reg(); | |
| 1805 int regHi = interval->assigned_regHi(); | |
| 1806 | |
| 1807 if ((reg < nof_regs && interval->always_in_memory()) || | |
| 1808 (use_fpu_stack_allocation() && reg >= pd_first_fpu_reg && reg <= pd_last_fpu_reg)) { | |
| 1809 // the interval is split to get a short range that is located on the stack | |
| 1810 // in the following two cases: | |
| 1811 // * the interval started in memory (e.g. method parameter), but is currently in a register | |
| 1812 // this is an optimization for exception handling that reduces the number of moves that | |
| 1813 // are necessary for resolving the states when an exception uses this exception handler | |
| 1814 // * the interval would be on the fpu stack at the begin of the exception handler | |
| 1815 // this is not allowed because of the complicated fpu stack handling on Intel | |
| 1816 | |
| 1817 // range that will be spilled to memory | |
| 1818 int from_op_id = block->first_lir_instruction_id(); | |
| 1819 int to_op_id = from_op_id + 1; // short live range of length 1 | |
| 1820 assert(interval->from() <= from_op_id && interval->to() >= to_op_id, | |
| 1821 "no split allowed between exception entry and first instruction"); | |
| 1822 | |
| 1823 if (interval->from() != from_op_id) { | |
| 1824 // the part before from_op_id is unchanged | |
| 1825 interval = interval->split(from_op_id); | |
| 1826 interval->assign_reg(reg, regHi); | |
| 1827 append_interval(interval); | |
| 1828 } | |
| 1829 assert(interval->from() == from_op_id, "must be true now"); | |
| 1830 | |
| 1831 Interval* spilled_part = interval; | |
| 1832 if (interval->to() != to_op_id) { | |
| 1833 // the part after to_op_id is unchanged | |
| 1834 spilled_part = interval->split_from_start(to_op_id); | |
| 1835 append_interval(spilled_part); | |
| 1836 move_resolver.add_mapping(spilled_part, interval); | |
| 1837 } | |
| 1838 assign_spill_slot(spilled_part); | |
| 1839 | |
| 1840 assert(spilled_part->from() == from_op_id && spilled_part->to() == to_op_id, "just checking"); | |
| 1841 } | |
| 1842 } | |
| 1843 | |
| 1844 void LinearScan::resolve_exception_entry(BlockBegin* block, MoveResolver &move_resolver) { | |
| 1845 assert(block->is_set(BlockBegin::exception_entry_flag), "should not call otherwise"); | |
| 1846 DEBUG_ONLY(move_resolver.check_empty()); | |
| 1847 | |
| 1848 // visit all registers where the live_in bit is set | |
| 1849 int size = live_set_size(); | |
| 304 | 1850 for (int r = (int)block->live_in().get_next_one_offset(0, size); r < size; r = (int)block->live_in().get_next_one_offset(r + 1, size)) { |
| 0 | 1851 resolve_exception_entry(block, r, move_resolver); |
| 1852 } | |
| 1853 | |
| 1854 // the live_in bits are not set for phi functions of the xhandler entry, so iterate them separately | |
| 1855 for_each_phi_fun(block, phi, | |
| 1856 resolve_exception_entry(block, phi->operand()->vreg_number(), move_resolver) | |
| 1857 ); | |
| 1858 | |
| 1859 if (move_resolver.has_mappings()) { | |
| 1860 // insert moves after first instruction | |
| 1861 move_resolver.set_insert_position(block->lir(), 1); | |
| 1862 move_resolver.resolve_and_append_moves(); | |
| 1863 } | |
| 1864 } | |
| 1865 | |
| 1866 | |
| 1867 void LinearScan::resolve_exception_edge(XHandler* handler, int throwing_op_id, int reg_num, Phi* phi, MoveResolver &move_resolver) { | |
| 1868 if (interval_at(reg_num) == NULL) { | |
| 1869 // if a phi function is never used, no interval is created -> ignore this | |
| 1870 return; | |
| 1871 } | |
| 1872 | |
| 1873 // the computation of to_interval is equal to resolve_collect_mappings, | |
| 1874 // but from_interval is more complicated because of phi functions | |
| 1875 BlockBegin* to_block = handler->entry_block(); | |
| 1876 Interval* to_interval = interval_at_block_begin(to_block, reg_num); | |
| 1877 | |
| 1878 if (phi != NULL) { | |
| 1879 // phi function of the exception entry block | |
| 1880 // no moves are created for this phi function in the LIR_Generator, so the | |
| 1881 // interval at the throwing instruction must be searched using the operands | |
| 1882 // of the phi function | |
| 1883 Value from_value = phi->operand_at(handler->phi_operand()); | |
| 1884 | |
| 1885 // with phi functions it can happen that the same from_value is used in | |
| 1886 // multiple mappings, so notify move-resolver that this is allowed | |
| 1887 move_resolver.set_multiple_reads_allowed(); | |
| 1888 | |
| 1889 Constant* con = from_value->as_Constant(); | |
| 1890 if (con != NULL && !con->is_pinned()) { | |
| 1891 // unpinned constants may have no register, so add mapping from constant to interval | |
| 1892 move_resolver.add_mapping(LIR_OprFact::value_type(con->type()), to_interval); | |
| 1893 } else { | |
| 1894 // search split child at the throwing op_id | |
| 1895 Interval* from_interval = interval_at_op_id(from_value->operand()->vreg_number(), throwing_op_id); | |
| 1896 move_resolver.add_mapping(from_interval, to_interval); | |
| 1897 } | |
| 1898 | |
| 1899 } else { | |
| 1900 // no phi function, so use reg_num also for from_interval | |
| 1901 // search split child at the throwing op_id | |
| 1902 Interval* from_interval = interval_at_op_id(reg_num, throwing_op_id); | |
| 1903 if (from_interval != to_interval) { | |
| 1904 // optimization to reduce number of moves: when to_interval is on stack and | |
| 1905 // the stack slot is known to be always correct, then no move is necessary | |
| 1906 if (!from_interval->always_in_memory() || from_interval->canonical_spill_slot() != to_interval->assigned_reg()) { | |
| 1907 move_resolver.add_mapping(from_interval, to_interval); | |
| 1908 } | |
| 1909 } | |
| 1910 } | |
| 1911 } | |
| 1912 | |
| 1913 void LinearScan::resolve_exception_edge(XHandler* handler, int throwing_op_id, MoveResolver &move_resolver) { | |
| 1914 TRACE_LINEAR_SCAN(4, tty->print_cr("resolving exception handler B%d: throwing_op_id=%d", handler->entry_block()->block_id(), throwing_op_id)); | |
| 1915 | |
| 1916 DEBUG_ONLY(move_resolver.check_empty()); | |
| 1917 assert(handler->lir_op_id() == -1, "already processed this xhandler"); | |
| 1918 DEBUG_ONLY(handler->set_lir_op_id(throwing_op_id)); | |
| 1919 assert(handler->entry_code() == NULL, "code already present"); | |
| 1920 | |
| 1921 // visit all registers where the live_in bit is set | |
| 1922 BlockBegin* block = handler->entry_block(); | |
| 1923 int size = live_set_size(); | |
| 304 | 1924 for (int r = (int)block->live_in().get_next_one_offset(0, size); r < size; r = (int)block->live_in().get_next_one_offset(r + 1, size)) { |
| 0 | 1925 resolve_exception_edge(handler, throwing_op_id, r, NULL, move_resolver); |
| 1926 } | |
| 1927 | |
| 1928 // the live_in bits are not set for phi functions of the xhandler entry, so iterate them separately | |
| 1929 for_each_phi_fun(block, phi, | |
| 1930 resolve_exception_edge(handler, throwing_op_id, phi->operand()->vreg_number(), phi, move_resolver) | |
| 1931 ); | |
| 1932 | |
| 1933 if (move_resolver.has_mappings()) { | |
| 1934 LIR_List* entry_code = new LIR_List(compilation()); | |
| 1935 move_resolver.set_insert_position(entry_code, 0); | |
| 1936 move_resolver.resolve_and_append_moves(); | |
| 1937 | |
| 1938 entry_code->jump(handler->entry_block()); | |
| 1939 handler->set_entry_code(entry_code); | |
| 1940 } | |
| 1941 } | |
| 1942 | |
| 1943 | |
| 1944 void LinearScan::resolve_exception_handlers() { | |
| 1945 MoveResolver move_resolver(this); | |
| 1946 LIR_OpVisitState visitor; | |
| 1947 int num_blocks = block_count(); | |
| 1948 | |
| 1949 int i; | |
| 1950 for (i = 0; i < num_blocks; i++) { | |
| 1951 BlockBegin* block = block_at(i); | |
| 1952 if (block->is_set(BlockBegin::exception_entry_flag)) { | |
| 1953 resolve_exception_entry(block, move_resolver); | |
| 1954 } | |
| 1955 } | |
| 1956 | |
| 1957 for (i = 0; i < num_blocks; i++) { | |
| 1958 BlockBegin* block = block_at(i); | |
| 1959 LIR_List* ops = block->lir(); | |
| 1960 int num_ops = ops->length(); | |
| 1961 | |
| 1962 // iterate all instructions of the block. skip the first because it is always a label | |
| 1963 assert(visitor.no_operands(ops->at(0)), "first operation must always be a label"); | |
| 1964 for (int j = 1; j < num_ops; j++) { | |
| 1965 LIR_Op* op = ops->at(j); | |
| 1966 int op_id = op->id(); | |
| 1967 | |
| 1968 if (op_id != -1 && has_info(op_id)) { | |
| 1969 // visit operation to collect all operands | |
| 1970 visitor.visit(op); | |
| 1971 assert(visitor.info_count() > 0, "should not visit otherwise"); | |
| 1972 | |
| 1973 XHandlers* xhandlers = visitor.all_xhandler(); | |
| 1974 int n = xhandlers->length(); | |
| 1975 for (int k = 0; k < n; k++) { | |
| 1976 resolve_exception_edge(xhandlers->handler_at(k), op_id, move_resolver); | |
| 1977 } | |
| 1978 | |
| 1979 #ifdef ASSERT | |
| 1980 } else { | |
| 1981 visitor.visit(op); | |
| 1982 assert(visitor.all_xhandler()->length() == 0, "missed exception handler"); | |
| 1983 #endif | |
| 1984 } | |
| 1985 } | |
| 1986 } | |
| 1987 } | |
| 1988 | |
| 1989 | |
| 1990 // ********** Phase 7: assign register numbers back to LIR | |
| 1991 // (includes computation of debug information and oop maps) | |
| 1992 | |
| 1993 VMReg LinearScan::vm_reg_for_interval(Interval* interval) { | |
| 1994 VMReg reg = interval->cached_vm_reg(); | |
| 1995 if (!reg->is_valid() ) { | |
| 1996 reg = vm_reg_for_operand(operand_for_interval(interval)); | |
| 1997 interval->set_cached_vm_reg(reg); | |
| 1998 } | |
| 1999 assert(reg == vm_reg_for_operand(operand_for_interval(interval)), "wrong cached value"); | |
| 2000 return reg; | |
| 2001 } | |
| 2002 | |
| 2003 VMReg LinearScan::vm_reg_for_operand(LIR_Opr opr) { | |
| 2004 assert(opr->is_oop(), "currently only implemented for oop operands"); | |
| 2005 return frame_map()->regname(opr); | |
| 2006 } | |
| 2007 | |
| 2008 | |
| 2009 LIR_Opr LinearScan::operand_for_interval(Interval* interval) { | |
| 2010 LIR_Opr opr = interval->cached_opr(); | |
| 2011 if (opr->is_illegal()) { | |
| 2012 opr = calc_operand_for_interval(interval); | |
| 2013 interval->set_cached_opr(opr); | |
| 2014 } | |
| 2015 | |
| 2016 assert(opr == calc_operand_for_interval(interval), "wrong cached value"); | |
| 2017 return opr; | |
| 2018 } | |
| 2019 | |
| 2020 LIR_Opr LinearScan::calc_operand_for_interval(const Interval* interval) { | |
| 2021 int assigned_reg = interval->assigned_reg(); | |
| 2022 BasicType type = interval->type(); | |
| 2023 | |
| 2024 if (assigned_reg >= nof_regs) { | |
| 2025 // stack slot | |
| 2026 assert(interval->assigned_regHi() == any_reg, "must not have hi register"); | |
| 2027 return LIR_OprFact::stack(assigned_reg - nof_regs, type); | |
| 2028 | |
| 2029 } else { | |
| 2030 // register | |
| 2031 switch (type) { | |
| 2032 case T_OBJECT: { | |
| 2033 assert(assigned_reg >= pd_first_cpu_reg && assigned_reg <= pd_last_cpu_reg, "no cpu register"); | |
| 2034 assert(interval->assigned_regHi() == any_reg, "must not have hi register"); | |
| 2035 return LIR_OprFact::single_cpu_oop(assigned_reg); | |
| 2036 } | |
| 2037 | |
|
1816
87b64980e2f1
6972540: sun/nio/ch/SocketChannelImpl compilation crashed when executing CompileTheWorld
never
parents:
1681
diff
changeset
|
2038 case T_ADDRESS: { |
|
87b64980e2f1
6972540: sun/nio/ch/SocketChannelImpl compilation crashed when executing CompileTheWorld
never
parents:
1681
diff
changeset
|
2039 assert(assigned_reg >= pd_first_cpu_reg && assigned_reg <= pd_last_cpu_reg, "no cpu register"); |
|
87b64980e2f1
6972540: sun/nio/ch/SocketChannelImpl compilation crashed when executing CompileTheWorld
never
parents:
1681
diff
changeset
|
2040 assert(interval->assigned_regHi() == any_reg, "must not have hi register"); |
|
87b64980e2f1
6972540: sun/nio/ch/SocketChannelImpl compilation crashed when executing CompileTheWorld
never
parents:
1681
diff
changeset
|
2041 return LIR_OprFact::single_cpu_address(assigned_reg); |
|
87b64980e2f1
6972540: sun/nio/ch/SocketChannelImpl compilation crashed when executing CompileTheWorld
never
parents:
1681
diff
changeset
|
2042 } |
|
87b64980e2f1
6972540: sun/nio/ch/SocketChannelImpl compilation crashed when executing CompileTheWorld
never
parents:
1681
diff
changeset
|
2043 |
|
1681
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
2044 #ifdef __SOFTFP__ |
|
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
2045 case T_FLOAT: // fall through |
|
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
2046 #endif // __SOFTFP__ |
| 0 | 2047 case T_INT: { |
| 2048 assert(assigned_reg >= pd_first_cpu_reg && assigned_reg <= pd_last_cpu_reg, "no cpu register"); | |
| 2049 assert(interval->assigned_regHi() == any_reg, "must not have hi register"); | |
| 2050 return LIR_OprFact::single_cpu(assigned_reg); | |
| 2051 } | |
| 2052 | |
|
1681
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
2053 #ifdef __SOFTFP__ |
|
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
2054 case T_DOUBLE: // fall through |
|
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
2055 #endif // __SOFTFP__ |
| 0 | 2056 case T_LONG: { |
| 2057 int assigned_regHi = interval->assigned_regHi(); | |
| 2058 assert(assigned_reg >= pd_first_cpu_reg && assigned_reg <= pd_last_cpu_reg, "no cpu register"); | |
| 2059 assert(num_physical_regs(T_LONG) == 1 || | |
| 2060 (assigned_regHi >= pd_first_cpu_reg && assigned_regHi <= pd_last_cpu_reg), "no cpu register"); | |
| 2061 | |
| 2062 assert(assigned_reg != assigned_regHi, "invalid allocation"); | |
| 2063 assert(num_physical_regs(T_LONG) == 1 || assigned_reg < assigned_regHi, | |
| 2064 "register numbers must be sorted (ensure that e.g. a move from eax,ebx to ebx,eax can not occur)"); | |
| 2065 assert((assigned_regHi != any_reg) ^ (num_physical_regs(T_LONG) == 1), "must be match"); | |
| 2066 if (requires_adjacent_regs(T_LONG)) { | |
| 2067 assert(assigned_reg % 2 == 0 && assigned_reg + 1 == assigned_regHi, "must be sequential and even"); | |
| 2068 } | |
| 2069 | |
| 2070 #ifdef _LP64 | |
| 2071 return LIR_OprFact::double_cpu(assigned_reg, assigned_reg); | |
| 2072 #else | |
|
1681
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
2073 #if defined(SPARC) || defined(PPC) |
| 0 | 2074 return LIR_OprFact::double_cpu(assigned_regHi, assigned_reg); |
| 2075 #else | |
| 2076 return LIR_OprFact::double_cpu(assigned_reg, assigned_regHi); | |
| 304 | 2077 #endif // SPARC |
| 2078 #endif // LP64 | |
| 0 | 2079 } |
| 2080 | |
|
1681
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
2081 #ifndef __SOFTFP__ |
| 0 | 2082 case T_FLOAT: { |
| 304 | 2083 #ifdef X86 |
| 0 | 2084 if (UseSSE >= 1) { |
| 2085 assert(assigned_reg >= pd_first_xmm_reg && assigned_reg <= pd_last_xmm_reg, "no xmm register"); | |
| 2086 assert(interval->assigned_regHi() == any_reg, "must not have hi register"); | |
| 2087 return LIR_OprFact::single_xmm(assigned_reg - pd_first_xmm_reg); | |
| 2088 } | |
| 2089 #endif | |
| 2090 | |
| 2091 assert(assigned_reg >= pd_first_fpu_reg && assigned_reg <= pd_last_fpu_reg, "no fpu register"); | |
| 2092 assert(interval->assigned_regHi() == any_reg, "must not have hi register"); | |
| 2093 return LIR_OprFact::single_fpu(assigned_reg - pd_first_fpu_reg); | |
| 2094 } | |
| 2095 | |
| 2096 case T_DOUBLE: { | |
| 304 | 2097 #ifdef X86 |
| 0 | 2098 if (UseSSE >= 2) { |
| 2099 assert(assigned_reg >= pd_first_xmm_reg && assigned_reg <= pd_last_xmm_reg, "no xmm register"); | |
| 2100 assert(interval->assigned_regHi() == any_reg, "must not have hi register (double xmm values are stored in one register)"); | |
| 2101 return LIR_OprFact::double_xmm(assigned_reg - pd_first_xmm_reg); | |
| 2102 } | |
| 2103 #endif | |
| 2104 | |
| 2105 #ifdef SPARC | |
| 2106 assert(assigned_reg >= pd_first_fpu_reg && assigned_reg <= pd_last_fpu_reg, "no fpu register"); | |
| 2107 assert(interval->assigned_regHi() >= pd_first_fpu_reg && interval->assigned_regHi() <= pd_last_fpu_reg, "no fpu register"); | |
| 2108 assert(assigned_reg % 2 == 0 && assigned_reg + 1 == interval->assigned_regHi(), "must be sequential and even"); | |
| 2109 LIR_Opr result = LIR_OprFact::double_fpu(interval->assigned_regHi() - pd_first_fpu_reg, assigned_reg - pd_first_fpu_reg); | |
|
1681
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
2110 #elif defined(ARM) |
|
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
2111 assert(assigned_reg >= pd_first_fpu_reg && assigned_reg <= pd_last_fpu_reg, "no fpu register"); |
|
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
2112 assert(interval->assigned_regHi() >= pd_first_fpu_reg && interval->assigned_regHi() <= pd_last_fpu_reg, "no fpu register"); |
|
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
2113 assert(assigned_reg % 2 == 0 && assigned_reg + 1 == interval->assigned_regHi(), "must be sequential and even"); |
|
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
2114 LIR_Opr result = LIR_OprFact::double_fpu(assigned_reg - pd_first_fpu_reg, interval->assigned_regHi() - pd_first_fpu_reg); |
| 0 | 2115 #else |
| 2116 assert(assigned_reg >= pd_first_fpu_reg && assigned_reg <= pd_last_fpu_reg, "no fpu register"); | |
| 2117 assert(interval->assigned_regHi() == any_reg, "must not have hi register (double fpu values are stored in one register on Intel)"); | |
| 2118 LIR_Opr result = LIR_OprFact::double_fpu(assigned_reg - pd_first_fpu_reg); | |
| 2119 #endif | |
| 2120 return result; | |
| 2121 } | |
|
1681
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
2122 #endif // __SOFTFP__ |
| 0 | 2123 |
| 2124 default: { | |
| 2125 ShouldNotReachHere(); | |
| 2126 return LIR_OprFact::illegalOpr; | |
| 2127 } | |
| 2128 } | |
| 2129 } | |
| 2130 } | |
| 2131 | |
| 2132 LIR_Opr LinearScan::canonical_spill_opr(Interval* interval) { | |
| 2133 assert(interval->canonical_spill_slot() >= nof_regs, "canonical spill slot not set"); | |
| 2134 return LIR_OprFact::stack(interval->canonical_spill_slot() - nof_regs, interval->type()); | |
| 2135 } | |
| 2136 | |
| 2137 LIR_Opr LinearScan::color_lir_opr(LIR_Opr opr, int op_id, LIR_OpVisitState::OprMode mode) { | |
| 2138 assert(opr->is_virtual(), "should not call this otherwise"); | |
| 2139 | |
| 2140 Interval* interval = interval_at(opr->vreg_number()); | |
| 2141 assert(interval != NULL, "interval must exist"); | |
| 2142 | |
| 2143 if (op_id != -1) { | |
| 2144 #ifdef ASSERT | |
| 2145 BlockBegin* block = block_of_op_with_id(op_id); | |
| 2146 if (block->number_of_sux() <= 1 && op_id == block->last_lir_instruction_id()) { | |
| 2147 // check if spill moves could have been appended at the end of this block, but | |
| 2148 // before the branch instruction. So the split child information for this branch would | |
| 2149 // be incorrect. | |
| 2150 LIR_OpBranch* branch = block->lir()->instructions_list()->last()->as_OpBranch(); | |
| 2151 if (branch != NULL) { | |
| 2152 if (block->live_out().at(opr->vreg_number())) { | |
| 2153 assert(branch->cond() == lir_cond_always, "block does not end with an unconditional jump"); | |
| 2154 assert(false, "can't get split child for the last branch of a block because the information would be incorrect (moves are inserted before the branch in resolve_data_flow)"); | |
| 2155 } | |
| 2156 } | |
| 2157 } | |
| 2158 #endif | |
| 2159 | |
| 2160 // operands are not changed when an interval is split during allocation, | |
| 2161 // so search the right interval here | |
| 2162 interval = split_child_at_op_id(interval, op_id, mode); | |
| 2163 } | |
| 2164 | |
| 2165 LIR_Opr res = operand_for_interval(interval); | |
| 2166 | |
| 304 | 2167 #ifdef X86 |
| 0 | 2168 // new semantic for is_last_use: not only set on definite end of interval, |
| 2169 // but also before hole | |
| 2170 // This may still miss some cases (e.g. for dead values), but it is not necessary that the | |
| 2171 // last use information is completely correct | |
| 2172 // information is only needed for fpu stack allocation | |
| 2173 if (res->is_fpu_register()) { | |
| 2174 if (opr->is_last_use() || op_id == interval->to() || (op_id != -1 && interval->has_hole_between(op_id, op_id + 1))) { | |
| 2175 assert(op_id == -1 || !is_block_begin(op_id), "holes at begin of block may also result from control flow"); | |
| 2176 res = res->make_last_use(); | |
| 2177 } | |
| 2178 } | |
| 2179 #endif | |
| 2180 | |
| 2181 assert(!gen()->is_vreg_flag_set(opr->vreg_number(), LIRGenerator::callee_saved) || !FrameMap::is_caller_save_register(res), "bad allocation"); | |
| 2182 | |
| 2183 return res; | |
| 2184 } | |
| 2185 | |
| 2186 | |
| 2187 #ifdef ASSERT | |
| 2188 // some methods used to check correctness of debug information | |
| 2189 | |
| 2190 void assert_no_register_values(GrowableArray<ScopeValue*>* values) { | |
| 2191 if (values == NULL) { | |
| 2192 return; | |
| 2193 } | |
| 2194 | |
| 2195 for (int i = 0; i < values->length(); i++) { | |
| 2196 ScopeValue* value = values->at(i); | |
| 2197 | |
| 2198 if (value->is_location()) { | |
| 2199 Location location = ((LocationValue*)value)->location(); | |
| 2200 assert(location.where() == Location::on_stack, "value is in register"); | |
| 2201 } | |
| 2202 } | |
| 2203 } | |
| 2204 | |
| 2205 void assert_no_register_values(GrowableArray<MonitorValue*>* values) { | |
| 2206 if (values == NULL) { | |
| 2207 return; | |
| 2208 } | |
| 2209 | |
| 2210 for (int i = 0; i < values->length(); i++) { | |
| 2211 MonitorValue* value = values->at(i); | |
| 2212 | |
| 2213 if (value->owner()->is_location()) { | |
| 2214 Location location = ((LocationValue*)value->owner())->location(); | |
| 2215 assert(location.where() == Location::on_stack, "owner is in register"); | |
| 2216 } | |
| 2217 assert(value->basic_lock().where() == Location::on_stack, "basic_lock is in register"); | |
| 2218 } | |
| 2219 } | |
| 2220 | |
| 2221 void assert_equal(Location l1, Location l2) { | |
| 2222 assert(l1.where() == l2.where() && l1.type() == l2.type() && l1.offset() == l2.offset(), ""); | |
| 2223 } | |
| 2224 | |
| 2225 void assert_equal(ScopeValue* v1, ScopeValue* v2) { | |
| 2226 if (v1->is_location()) { | |
| 2227 assert(v2->is_location(), ""); | |
| 2228 assert_equal(((LocationValue*)v1)->location(), ((LocationValue*)v2)->location()); | |
| 2229 } else if (v1->is_constant_int()) { | |
| 2230 assert(v2->is_constant_int(), ""); | |
| 2231 assert(((ConstantIntValue*)v1)->value() == ((ConstantIntValue*)v2)->value(), ""); | |
| 2232 } else if (v1->is_constant_double()) { | |
| 2233 assert(v2->is_constant_double(), ""); | |
| 2234 assert(((ConstantDoubleValue*)v1)->value() == ((ConstantDoubleValue*)v2)->value(), ""); | |
| 2235 } else if (v1->is_constant_long()) { | |
| 2236 assert(v2->is_constant_long(), ""); | |
| 2237 assert(((ConstantLongValue*)v1)->value() == ((ConstantLongValue*)v2)->value(), ""); | |
| 2238 } else if (v1->is_constant_oop()) { | |
| 2239 assert(v2->is_constant_oop(), ""); | |
| 2240 assert(((ConstantOopWriteValue*)v1)->value() == ((ConstantOopWriteValue*)v2)->value(), ""); | |
| 2241 } else { | |
| 2242 ShouldNotReachHere(); | |
| 2243 } | |
| 2244 } | |
| 2245 | |
| 2246 void assert_equal(MonitorValue* m1, MonitorValue* m2) { | |
| 2247 assert_equal(m1->owner(), m2->owner()); | |
| 2248 assert_equal(m1->basic_lock(), m2->basic_lock()); | |
| 2249 } | |
| 2250 | |
| 2251 void assert_equal(IRScopeDebugInfo* d1, IRScopeDebugInfo* d2) { | |
| 2252 assert(d1->scope() == d2->scope(), "not equal"); | |
| 2253 assert(d1->bci() == d2->bci(), "not equal"); | |
| 2254 | |
| 2255 if (d1->locals() != NULL) { | |
| 2256 assert(d1->locals() != NULL && d2->locals() != NULL, "not equal"); | |
| 2257 assert(d1->locals()->length() == d2->locals()->length(), "not equal"); | |
| 2258 for (int i = 0; i < d1->locals()->length(); i++) { | |
| 2259 assert_equal(d1->locals()->at(i), d2->locals()->at(i)); | |
| 2260 } | |
| 2261 } else { | |
| 2262 assert(d1->locals() == NULL && d2->locals() == NULL, "not equal"); | |
| 2263 } | |
| 2264 | |
| 2265 if (d1->expressions() != NULL) { | |
| 2266 assert(d1->expressions() != NULL && d2->expressions() != NULL, "not equal"); | |
| 2267 assert(d1->expressions()->length() == d2->expressions()->length(), "not equal"); | |
| 2268 for (int i = 0; i < d1->expressions()->length(); i++) { | |
| 2269 assert_equal(d1->expressions()->at(i), d2->expressions()->at(i)); | |
| 2270 } | |
| 2271 } else { | |
| 2272 assert(d1->expressions() == NULL && d2->expressions() == NULL, "not equal"); | |
| 2273 } | |
| 2274 | |
| 2275 if (d1->monitors() != NULL) { | |
| 2276 assert(d1->monitors() != NULL && d2->monitors() != NULL, "not equal"); | |
| 2277 assert(d1->monitors()->length() == d2->monitors()->length(), "not equal"); | |
| 2278 for (int i = 0; i < d1->monitors()->length(); i++) { | |
| 2279 assert_equal(d1->monitors()->at(i), d2->monitors()->at(i)); | |
| 2280 } | |
| 2281 } else { | |
| 2282 assert(d1->monitors() == NULL && d2->monitors() == NULL, "not equal"); | |
| 2283 } | |
| 2284 | |
| 2285 if (d1->caller() != NULL) { | |
| 2286 assert(d1->caller() != NULL && d2->caller() != NULL, "not equal"); | |
| 2287 assert_equal(d1->caller(), d2->caller()); | |
| 2288 } else { | |
| 2289 assert(d1->caller() == NULL && d2->caller() == NULL, "not equal"); | |
| 2290 } | |
| 2291 } | |
| 2292 | |
| 2293 void check_stack_depth(CodeEmitInfo* info, int stack_end) { | |
| 1819 | 2294 if (info->stack()->bci() != SynchronizationEntryBCI && !info->scope()->method()->is_native()) { |
| 2295 Bytecodes::Code code = info->scope()->method()->java_code_at_bci(info->stack()->bci()); | |
| 0 | 2296 switch (code) { |
| 2297 case Bytecodes::_ifnull : // fall through | |
| 2298 case Bytecodes::_ifnonnull : // fall through | |
| 2299 case Bytecodes::_ifeq : // fall through | |
| 2300 case Bytecodes::_ifne : // fall through | |
| 2301 case Bytecodes::_iflt : // fall through | |
| 2302 case Bytecodes::_ifge : // fall through | |
| 2303 case Bytecodes::_ifgt : // fall through | |
| 2304 case Bytecodes::_ifle : // fall through | |
| 2305 case Bytecodes::_if_icmpeq : // fall through | |
| 2306 case Bytecodes::_if_icmpne : // fall through | |
| 2307 case Bytecodes::_if_icmplt : // fall through | |
| 2308 case Bytecodes::_if_icmpge : // fall through | |
| 2309 case Bytecodes::_if_icmpgt : // fall through | |
| 2310 case Bytecodes::_if_icmple : // fall through | |
| 2311 case Bytecodes::_if_acmpeq : // fall through | |
| 2312 case Bytecodes::_if_acmpne : | |
| 2313 assert(stack_end >= -Bytecodes::depth(code), "must have non-empty expression stack at if bytecode"); | |
| 2314 break; | |
| 2315 } | |
| 2316 } | |
| 2317 } | |
| 2318 | |
| 2319 #endif // ASSERT | |
| 2320 | |
| 2321 | |
| 2322 IntervalWalker* LinearScan::init_compute_oop_maps() { | |
| 2323 // setup lists of potential oops for walking | |
| 2324 Interval* oop_intervals; | |
| 2325 Interval* non_oop_intervals; | |
| 2326 | |
| 2327 create_unhandled_lists(&oop_intervals, &non_oop_intervals, is_oop_interval, NULL); | |
| 2328 | |
| 2329 // intervals that have no oops inside need not to be processed | |
| 2330 // to ensure a walking until the last instruction id, add a dummy interval | |
| 2331 // with a high operation id | |
| 2332 non_oop_intervals = new Interval(any_reg); | |
| 2333 non_oop_intervals->add_range(max_jint - 2, max_jint - 1); | |
| 2334 | |
| 2335 return new IntervalWalker(this, oop_intervals, non_oop_intervals); | |
| 2336 } | |
| 2337 | |
| 2338 | |
| 2339 OopMap* LinearScan::compute_oop_map(IntervalWalker* iw, LIR_Op* op, CodeEmitInfo* info, bool is_call_site) { | |
| 2340 TRACE_LINEAR_SCAN(3, tty->print_cr("creating oop map at op_id %d", op->id())); | |
| 2341 | |
| 2342 // walk before the current operation -> intervals that start at | |
| 2343 // the operation (= output operands of the operation) are not | |
| 2344 // included in the oop map | |
| 2345 iw->walk_before(op->id()); | |
| 2346 | |
| 2347 int frame_size = frame_map()->framesize(); | |
| 2348 int arg_count = frame_map()->oop_map_arg_count(); | |
| 2349 OopMap* map = new OopMap(frame_size, arg_count); | |
| 2350 | |
| 2351 // Check if this is a patch site. | |
| 2352 bool is_patch_info = false; | |
| 2353 if (op->code() == lir_move) { | |
| 2354 assert(!is_call_site, "move must not be a call site"); | |
| 2355 assert(op->as_Op1() != NULL, "move must be LIR_Op1"); | |
| 2356 LIR_Op1* move = (LIR_Op1*)op; | |
| 2357 | |
| 2358 is_patch_info = move->patch_code() != lir_patch_none; | |
| 2359 } | |
| 2360 | |
| 2361 // Iterate through active intervals | |
| 2362 for (Interval* interval = iw->active_first(fixedKind); interval != Interval::end(); interval = interval->next()) { | |
| 2363 int assigned_reg = interval->assigned_reg(); | |
| 2364 | |
| 2365 assert(interval->current_from() <= op->id() && op->id() <= interval->current_to(), "interval should not be active otherwise"); | |
| 2366 assert(interval->assigned_regHi() == any_reg, "oop must be single word"); | |
| 2367 assert(interval->reg_num() >= LIR_OprDesc::vreg_base, "fixed interval found"); | |
| 2368 | |
| 2369 // Check if this range covers the instruction. Intervals that | |
| 2370 // start or end at the current operation are not included in the | |
| 2371 // oop map, except in the case of patching moves. For patching | |
| 2372 // moves, any intervals which end at this instruction are included | |
| 2373 // in the oop map since we may safepoint while doing the patch | |
| 2374 // before we've consumed the inputs. | |
| 2375 if (is_patch_info || op->id() < interval->current_to()) { | |
| 2376 | |
| 2377 // caller-save registers must not be included into oop-maps at calls | |
| 2378 assert(!is_call_site || assigned_reg >= nof_regs || !is_caller_save(assigned_reg), "interval is in a caller-save register at a call -> register will be overwritten"); | |
| 2379 | |
| 2380 VMReg name = vm_reg_for_interval(interval); | |
| 2381 map->set_oop(name); | |
| 2382 | |
| 2383 // Spill optimization: when the stack value is guaranteed to be always correct, | |
| 2384 // then it must be added to the oop map even if the interval is currently in a register | |
| 2385 if (interval->always_in_memory() && | |
| 2386 op->id() > interval->spill_definition_pos() && | |
| 2387 interval->assigned_reg() != interval->canonical_spill_slot()) { | |
| 2388 assert(interval->spill_definition_pos() > 0, "position not set correctly"); | |
| 2389 assert(interval->canonical_spill_slot() >= LinearScan::nof_regs, "no spill slot assigned"); | |
| 2390 assert(interval->assigned_reg() < LinearScan::nof_regs, "interval is on stack, so stack slot is registered twice"); | |
| 2391 | |
| 2392 map->set_oop(frame_map()->slot_regname(interval->canonical_spill_slot() - LinearScan::nof_regs)); | |
| 2393 } | |
| 2394 } | |
| 2395 } | |
| 2396 | |
| 2397 // add oops from lock stack | |
| 2398 assert(info->stack() != NULL, "CodeEmitInfo must always have a stack"); | |
| 1819 | 2399 int locks_count = info->stack()->total_locks_size(); |
| 0 | 2400 for (int i = 0; i < locks_count; i++) { |
| 2401 map->set_oop(frame_map()->monitor_object_regname(i)); | |
| 2402 } | |
| 2403 | |
| 2404 return map; | |
| 2405 } | |
| 2406 | |
| 2407 | |
| 2408 void LinearScan::compute_oop_map(IntervalWalker* iw, const LIR_OpVisitState &visitor, LIR_Op* op) { | |
| 2409 assert(visitor.info_count() > 0, "no oop map needed"); | |
| 2410 | |
| 2411 // compute oop_map only for first CodeEmitInfo | |
| 2412 // because it is (in most cases) equal for all other infos of the same operation | |
| 2413 CodeEmitInfo* first_info = visitor.info_at(0); | |
| 2414 OopMap* first_oop_map = compute_oop_map(iw, op, first_info, visitor.has_call()); | |
| 2415 | |
| 2416 for (int i = 0; i < visitor.info_count(); i++) { | |
| 2417 CodeEmitInfo* info = visitor.info_at(i); | |
| 2418 OopMap* oop_map = first_oop_map; | |
| 2419 | |
| 2420 if (info->stack()->locks_size() != first_info->stack()->locks_size()) { | |
| 2421 // this info has a different number of locks then the precomputed oop map | |
| 2422 // (possible for lock and unlock instructions) -> compute oop map with | |
| 2423 // correct lock information | |
| 2424 oop_map = compute_oop_map(iw, op, info, visitor.has_call()); | |
| 2425 } | |
| 2426 | |
| 2427 if (info->_oop_map == NULL) { | |
| 2428 info->_oop_map = oop_map; | |
| 2429 } else { | |
| 2430 // a CodeEmitInfo can not be shared between different LIR-instructions | |
| 2431 // because interval splitting can occur anywhere between two instructions | |
| 2432 // and so the oop maps must be different | |
| 2433 // -> check if the already set oop_map is exactly the one calculated for this operation | |
| 2434 assert(info->_oop_map == oop_map, "same CodeEmitInfo used for multiple LIR instructions"); | |
| 2435 } | |
| 2436 } | |
| 2437 } | |
| 2438 | |
| 2439 | |
| 2440 // frequently used constants | |
| 2441 ConstantOopWriteValue LinearScan::_oop_null_scope_value = ConstantOopWriteValue(NULL); | |
| 2442 ConstantIntValue LinearScan::_int_m1_scope_value = ConstantIntValue(-1); | |
| 2443 ConstantIntValue LinearScan::_int_0_scope_value = ConstantIntValue(0); | |
| 2444 ConstantIntValue LinearScan::_int_1_scope_value = ConstantIntValue(1); | |
| 2445 ConstantIntValue LinearScan::_int_2_scope_value = ConstantIntValue(2); | |
| 2446 LocationValue _illegal_value = LocationValue(Location()); | |
| 2447 | |
| 2448 void LinearScan::init_compute_debug_info() { | |
| 2449 // cache for frequently used scope values | |
| 2450 // (cpu registers and stack slots) | |
| 2451 _scope_value_cache = ScopeValueArray((LinearScan::nof_cpu_regs + frame_map()->argcount() + max_spills()) * 2, NULL); | |
| 2452 } | |
| 2453 | |
| 2454 MonitorValue* LinearScan::location_for_monitor_index(int monitor_index) { | |
| 2455 Location loc; | |
| 2456 if (!frame_map()->location_for_monitor_object(monitor_index, &loc)) { | |
| 2457 bailout("too large frame"); | |
| 2458 } | |
| 2459 ScopeValue* object_scope_value = new LocationValue(loc); | |
| 2460 | |
| 2461 if (!frame_map()->location_for_monitor_lock(monitor_index, &loc)) { | |
| 2462 bailout("too large frame"); | |
| 2463 } | |
| 2464 return new MonitorValue(object_scope_value, loc); | |
| 2465 } | |
| 2466 | |
| 2467 LocationValue* LinearScan::location_for_name(int name, Location::Type loc_type) { | |
| 2468 Location loc; | |
| 2469 if (!frame_map()->locations_for_slot(name, loc_type, &loc)) { | |
| 2470 bailout("too large frame"); | |
| 2471 } | |
| 2472 return new LocationValue(loc); | |
| 2473 } | |
| 2474 | |
| 2475 | |
| 2476 int LinearScan::append_scope_value_for_constant(LIR_Opr opr, GrowableArray<ScopeValue*>* scope_values) { | |
| 2477 assert(opr->is_constant(), "should not be called otherwise"); | |
| 2478 | |
| 2479 LIR_Const* c = opr->as_constant_ptr(); | |
| 2480 BasicType t = c->type(); | |
| 2481 switch (t) { | |
| 2482 case T_OBJECT: { | |
| 2483 jobject value = c->as_jobject(); | |
| 2484 if (value == NULL) { | |
| 2485 scope_values->append(&_oop_null_scope_value); | |
| 2486 } else { | |
| 2487 scope_values->append(new ConstantOopWriteValue(c->as_jobject())); | |
| 2488 } | |
| 2489 return 1; | |
| 2490 } | |
| 2491 | |
| 2492 case T_INT: // fall through | |
| 2493 case T_FLOAT: { | |
| 2494 int value = c->as_jint_bits(); | |
| 2495 switch (value) { | |
| 2496 case -1: scope_values->append(&_int_m1_scope_value); break; | |
| 2497 case 0: scope_values->append(&_int_0_scope_value); break; | |
| 2498 case 1: scope_values->append(&_int_1_scope_value); break; | |
| 2499 case 2: scope_values->append(&_int_2_scope_value); break; | |
| 2500 default: scope_values->append(new ConstantIntValue(c->as_jint_bits())); break; | |
| 2501 } | |
| 2502 return 1; | |
| 2503 } | |
| 2504 | |
| 2505 case T_LONG: // fall through | |
| 2506 case T_DOUBLE: { | |
| 1060 | 2507 #ifdef _LP64 |
| 2508 scope_values->append(&_int_0_scope_value); | |
| 2509 scope_values->append(new ConstantLongValue(c->as_jlong_bits())); | |
| 2510 #else | |
| 0 | 2511 if (hi_word_offset_in_bytes > lo_word_offset_in_bytes) { |
| 2512 scope_values->append(new ConstantIntValue(c->as_jint_hi_bits())); | |
| 2513 scope_values->append(new ConstantIntValue(c->as_jint_lo_bits())); | |
| 2514 } else { | |
| 2515 scope_values->append(new ConstantIntValue(c->as_jint_lo_bits())); | |
| 2516 scope_values->append(new ConstantIntValue(c->as_jint_hi_bits())); | |
| 2517 } | |
| 1060 | 2518 #endif |
| 0 | 2519 return 2; |
| 2520 } | |
| 2521 | |
|
1297
c466efa608d5
6932496: c1: deoptimization of jsr subroutine fails on sparcv9
roland
parents:
1060
diff
changeset
|
2522 case T_ADDRESS: { |
|
c466efa608d5
6932496: c1: deoptimization of jsr subroutine fails on sparcv9
roland
parents:
1060
diff
changeset
|
2523 #ifdef _LP64 |
|
c466efa608d5
6932496: c1: deoptimization of jsr subroutine fails on sparcv9
roland
parents:
1060
diff
changeset
|
2524 scope_values->append(new ConstantLongValue(c->as_jint())); |
|
c466efa608d5
6932496: c1: deoptimization of jsr subroutine fails on sparcv9
roland
parents:
1060
diff
changeset
|
2525 #else |
|
c466efa608d5
6932496: c1: deoptimization of jsr subroutine fails on sparcv9
roland
parents:
1060
diff
changeset
|
2526 scope_values->append(new ConstantIntValue(c->as_jint())); |
|
c466efa608d5
6932496: c1: deoptimization of jsr subroutine fails on sparcv9
roland
parents:
1060
diff
changeset
|
2527 #endif |
|
c466efa608d5
6932496: c1: deoptimization of jsr subroutine fails on sparcv9
roland
parents:
1060
diff
changeset
|
2528 return 1; |
|
c466efa608d5
6932496: c1: deoptimization of jsr subroutine fails on sparcv9
roland
parents:
1060
diff
changeset
|
2529 } |
|
c466efa608d5
6932496: c1: deoptimization of jsr subroutine fails on sparcv9
roland
parents:
1060
diff
changeset
|
2530 |
| 0 | 2531 default: |
| 2532 ShouldNotReachHere(); | |
| 304 | 2533 return -1; |
| 0 | 2534 } |
| 2535 } | |
| 2536 | |
| 2537 int LinearScan::append_scope_value_for_operand(LIR_Opr opr, GrowableArray<ScopeValue*>* scope_values) { | |
| 2538 if (opr->is_single_stack()) { | |
| 2539 int stack_idx = opr->single_stack_ix(); | |
| 2540 bool is_oop = opr->is_oop_register(); | |
| 2541 int cache_idx = (stack_idx + LinearScan::nof_cpu_regs) * 2 + (is_oop ? 1 : 0); | |
| 2542 | |
| 2543 ScopeValue* sv = _scope_value_cache.at(cache_idx); | |
| 2544 if (sv == NULL) { | |
| 2545 Location::Type loc_type = is_oop ? Location::oop : Location::normal; | |
| 2546 sv = location_for_name(stack_idx, loc_type); | |
| 2547 _scope_value_cache.at_put(cache_idx, sv); | |
| 2548 } | |
| 2549 | |
| 2550 // check if cached value is correct | |
| 2551 DEBUG_ONLY(assert_equal(sv, location_for_name(stack_idx, is_oop ? Location::oop : Location::normal))); | |
| 2552 | |
| 2553 scope_values->append(sv); | |
| 2554 return 1; | |
| 2555 | |
| 2556 } else if (opr->is_single_cpu()) { | |
| 2557 bool is_oop = opr->is_oop_register(); | |
| 2558 int cache_idx = opr->cpu_regnr() * 2 + (is_oop ? 1 : 0); | |
| 1060 | 2559 Location::Type int_loc_type = NOT_LP64(Location::normal) LP64_ONLY(Location::int_in_long); |
| 0 | 2560 |
| 2561 ScopeValue* sv = _scope_value_cache.at(cache_idx); | |
| 2562 if (sv == NULL) { | |
| 1060 | 2563 Location::Type loc_type = is_oop ? Location::oop : int_loc_type; |
| 0 | 2564 VMReg rname = frame_map()->regname(opr); |
| 2565 sv = new LocationValue(Location::new_reg_loc(loc_type, rname)); | |
| 2566 _scope_value_cache.at_put(cache_idx, sv); | |
| 2567 } | |
| 2568 | |
| 2569 // check if cached value is correct | |
| 1060 | 2570 DEBUG_ONLY(assert_equal(sv, new LocationValue(Location::new_reg_loc(is_oop ? Location::oop : int_loc_type, frame_map()->regname(opr))))); |
| 0 | 2571 |
| 2572 scope_values->append(sv); | |
| 2573 return 1; | |
| 2574 | |
| 304 | 2575 #ifdef X86 |
| 0 | 2576 } else if (opr->is_single_xmm()) { |
| 2577 VMReg rname = opr->as_xmm_float_reg()->as_VMReg(); | |
| 2578 LocationValue* sv = new LocationValue(Location::new_reg_loc(Location::normal, rname)); | |
| 2579 | |
| 2580 scope_values->append(sv); | |
| 2581 return 1; | |
| 2582 #endif | |
| 2583 | |
| 2584 } else if (opr->is_single_fpu()) { | |
| 304 | 2585 #ifdef X86 |
| 0 | 2586 // the exact location of fpu stack values is only known |
| 2587 // during fpu stack allocation, so the stack allocator object | |
| 2588 // must be present | |
| 2589 assert(use_fpu_stack_allocation(), "should not have float stack values without fpu stack allocation (all floats must be SSE2)"); | |
| 2590 assert(_fpu_stack_allocator != NULL, "must be present"); | |
| 2591 opr = _fpu_stack_allocator->to_fpu_stack(opr); | |
| 2592 #endif | |
| 2593 | |
| 2594 Location::Type loc_type = float_saved_as_double ? Location::float_in_dbl : Location::normal; | |
| 2595 VMReg rname = frame_map()->fpu_regname(opr->fpu_regnr()); | |
| 2596 LocationValue* sv = new LocationValue(Location::new_reg_loc(loc_type, rname)); | |
| 2597 | |
| 2598 scope_values->append(sv); | |
| 2599 return 1; | |
| 2600 | |
| 2601 } else { | |
| 2602 // double-size operands | |
| 2603 | |
| 2604 ScopeValue* first; | |
| 2605 ScopeValue* second; | |
| 2606 | |
| 2607 if (opr->is_double_stack()) { | |
| 304 | 2608 #ifdef _LP64 |
| 2609 Location loc1; | |
| 2610 Location::Type loc_type = opr->type() == T_LONG ? Location::lng : Location::dbl; | |
| 2611 if (!frame_map()->locations_for_slot(opr->double_stack_ix(), loc_type, &loc1, NULL)) { | |
| 2612 bailout("too large frame"); | |
| 2613 } | |
| 2614 // Does this reverse on x86 vs. sparc? | |
| 2615 first = new LocationValue(loc1); | |
| 2616 second = &_int_0_scope_value; | |
| 2617 #else | |
| 0 | 2618 Location loc1, loc2; |
| 2619 if (!frame_map()->locations_for_slot(opr->double_stack_ix(), Location::normal, &loc1, &loc2)) { | |
| 2620 bailout("too large frame"); | |
| 2621 } | |
| 2622 first = new LocationValue(loc1); | |
| 2623 second = new LocationValue(loc2); | |
| 304 | 2624 #endif // _LP64 |
| 0 | 2625 |
| 2626 } else if (opr->is_double_cpu()) { | |
| 2627 #ifdef _LP64 | |
| 2628 VMReg rname_first = opr->as_register_lo()->as_VMReg(); | |
| 2629 first = new LocationValue(Location::new_reg_loc(Location::lng, rname_first)); | |
| 2630 second = &_int_0_scope_value; | |
| 2631 #else | |
| 2632 VMReg rname_first = opr->as_register_lo()->as_VMReg(); | |
| 2633 VMReg rname_second = opr->as_register_hi()->as_VMReg(); | |
| 2634 | |
| 2635 if (hi_word_offset_in_bytes < lo_word_offset_in_bytes) { | |
| 2636 // lo/hi and swapped relative to first and second, so swap them | |
| 2637 VMReg tmp = rname_first; | |
| 2638 rname_first = rname_second; | |
| 2639 rname_second = tmp; | |
| 2640 } | |
| 2641 | |
| 2642 first = new LocationValue(Location::new_reg_loc(Location::normal, rname_first)); | |
| 2643 second = new LocationValue(Location::new_reg_loc(Location::normal, rname_second)); | |
| 304 | 2644 #endif //_LP64 |
| 2645 | |
| 2646 | |
| 2647 #ifdef X86 | |
| 0 | 2648 } else if (opr->is_double_xmm()) { |
| 2649 assert(opr->fpu_regnrLo() == opr->fpu_regnrHi(), "assumed in calculation"); | |
| 2650 VMReg rname_first = opr->as_xmm_double_reg()->as_VMReg(); | |
| 1369 | 2651 # ifdef _LP64 |
| 2652 first = new LocationValue(Location::new_reg_loc(Location::dbl, rname_first)); | |
| 2653 second = &_int_0_scope_value; | |
| 2654 # else | |
| 0 | 2655 first = new LocationValue(Location::new_reg_loc(Location::normal, rname_first)); |
| 2656 // %%% This is probably a waste but we'll keep things as they were for now | |
| 2657 if (true) { | |
| 2658 VMReg rname_second = rname_first->next(); | |
| 2659 second = new LocationValue(Location::new_reg_loc(Location::normal, rname_second)); | |
| 2660 } | |
| 1369 | 2661 # endif |
| 0 | 2662 #endif |
| 2663 | |
| 2664 } else if (opr->is_double_fpu()) { | |
| 2665 // On SPARC, fpu_regnrLo/fpu_regnrHi represents the two halves of | |
| 304 | 2666 // the double as float registers in the native ordering. On X86, |
| 0 | 2667 // fpu_regnrLo is a FPU stack slot whose VMReg represents |
| 2668 // the low-order word of the double and fpu_regnrLo + 1 is the | |
| 2669 // name for the other half. *first and *second must represent the | |
| 2670 // least and most significant words, respectively. | |
| 2671 | |
| 304 | 2672 #ifdef X86 |
| 0 | 2673 // the exact location of fpu stack values is only known |
| 2674 // during fpu stack allocation, so the stack allocator object | |
| 2675 // must be present | |
| 2676 assert(use_fpu_stack_allocation(), "should not have float stack values without fpu stack allocation (all floats must be SSE2)"); | |
| 2677 assert(_fpu_stack_allocator != NULL, "must be present"); | |
| 2678 opr = _fpu_stack_allocator->to_fpu_stack(opr); | |
| 2679 | |
| 2680 assert(opr->fpu_regnrLo() == opr->fpu_regnrHi(), "assumed in calculation (only fpu_regnrHi is used)"); | |
| 2681 #endif | |
| 2682 #ifdef SPARC | |
| 2683 assert(opr->fpu_regnrLo() == opr->fpu_regnrHi() + 1, "assumed in calculation (only fpu_regnrHi is used)"); | |
| 2684 #endif | |
|
1681
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
2685 #ifdef ARM |
|
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
2686 assert(opr->fpu_regnrHi() == opr->fpu_regnrLo() + 1, "assumed in calculation (only fpu_regnrLo is used)"); |
|
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
2687 #endif |
|
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
2688 #ifdef PPC |
|
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
2689 assert(opr->fpu_regnrLo() == opr->fpu_regnrHi(), "assumed in calculation (only fpu_regnrHi is used)"); |
|
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
2690 #endif |
| 0 | 2691 |
| 2692 VMReg rname_first = frame_map()->fpu_regname(opr->fpu_regnrHi()); | |
| 1369 | 2693 #ifdef _LP64 |
| 2694 first = new LocationValue(Location::new_reg_loc(Location::dbl, rname_first)); | |
| 2695 second = &_int_0_scope_value; | |
| 2696 #else | |
| 0 | 2697 first = new LocationValue(Location::new_reg_loc(Location::normal, rname_first)); |
| 2698 // %%% This is probably a waste but we'll keep things as they were for now | |
| 2699 if (true) { | |
| 2700 VMReg rname_second = rname_first->next(); | |
| 2701 second = new LocationValue(Location::new_reg_loc(Location::normal, rname_second)); | |
| 2702 } | |
| 1369 | 2703 #endif |
| 0 | 2704 |
| 2705 } else { | |
| 2706 ShouldNotReachHere(); | |
| 2707 first = NULL; | |
| 2708 second = NULL; | |
| 2709 } | |
| 2710 | |
| 2711 assert(first != NULL && second != NULL, "must be set"); | |
| 2712 // The convention the interpreter uses is that the second local | |
| 2713 // holds the first raw word of the native double representation. | |
| 2714 // This is actually reasonable, since locals and stack arrays | |
| 2715 // grow downwards in all implementations. | |
| 2716 // (If, on some machine, the interpreter's Java locals or stack | |
| 2717 // were to grow upwards, the embedded doubles would be word-swapped.) | |
| 2718 scope_values->append(second); | |
| 2719 scope_values->append(first); | |
| 2720 return 2; | |
| 2721 } | |
| 2722 } | |
| 2723 | |
| 2724 | |
| 2725 int LinearScan::append_scope_value(int op_id, Value value, GrowableArray<ScopeValue*>* scope_values) { | |
| 2726 if (value != NULL) { | |
| 2727 LIR_Opr opr = value->operand(); | |
| 2728 Constant* con = value->as_Constant(); | |
| 2729 | |
| 2730 assert(con == NULL || opr->is_virtual() || opr->is_constant() || opr->is_illegal(), "asumption: Constant instructions have only constant operands (or illegal if constant is optimized away)"); | |
| 2731 assert(con != NULL || opr->is_virtual(), "asumption: non-Constant instructions have only virtual operands"); | |
| 2732 | |
| 2733 if (con != NULL && !con->is_pinned() && !opr->is_constant()) { | |
| 2734 // Unpinned constants may have a virtual operand for a part of the lifetime | |
| 2735 // or may be illegal when it was optimized away, | |
| 2736 // so always use a constant operand | |
| 2737 opr = LIR_OprFact::value_type(con->type()); | |
| 2738 } | |
| 2739 assert(opr->is_virtual() || opr->is_constant(), "other cases not allowed here"); | |
| 2740 | |
| 2741 if (opr->is_virtual()) { | |
| 2742 LIR_OpVisitState::OprMode mode = LIR_OpVisitState::inputMode; | |
| 2743 | |
| 2744 BlockBegin* block = block_of_op_with_id(op_id); | |
| 2745 if (block->number_of_sux() == 1 && op_id == block->last_lir_instruction_id()) { | |
| 2746 // generating debug information for the last instruction of a block. | |
| 2747 // if this instruction is a branch, spill moves are inserted before this branch | |
| 2748 // and so the wrong operand would be returned (spill moves at block boundaries are not | |
| 2749 // considered in the live ranges of intervals) | |
| 2750 // Solution: use the first op_id of the branch target block instead. | |
| 2751 if (block->lir()->instructions_list()->last()->as_OpBranch() != NULL) { | |
| 2752 if (block->live_out().at(opr->vreg_number())) { | |
| 2753 op_id = block->sux_at(0)->first_lir_instruction_id(); | |
| 2754 mode = LIR_OpVisitState::outputMode; | |
| 2755 } | |
| 2756 } | |
| 2757 } | |
| 2758 | |
| 2759 // Get current location of operand | |
| 2760 // The operand must be live because debug information is considered when building the intervals | |
| 2761 // if the interval is not live, color_lir_opr will cause an assertion failure | |
| 2762 opr = color_lir_opr(opr, op_id, mode); | |
| 2763 assert(!has_call(op_id) || opr->is_stack() || !is_caller_save(reg_num(opr)), "can not have caller-save register operands at calls"); | |
| 2764 | |
| 2765 // Append to ScopeValue array | |
| 2766 return append_scope_value_for_operand(opr, scope_values); | |
| 2767 | |
| 2768 } else { | |
| 2769 assert(value->as_Constant() != NULL, "all other instructions have only virtual operands"); | |
| 2770 assert(opr->is_constant(), "operand must be constant"); | |
| 2771 | |
| 2772 return append_scope_value_for_constant(opr, scope_values); | |
| 2773 } | |
| 2774 } else { | |
| 2775 // append a dummy value because real value not needed | |
| 2776 scope_values->append(&_illegal_value); | |
| 2777 return 1; | |
| 2778 } | |
| 2779 } | |
| 2780 | |
| 2781 | |
| 1819 | 2782 IRScopeDebugInfo* LinearScan::compute_debug_info_for_scope(int op_id, IRScope* cur_scope, ValueStack* cur_state, ValueStack* innermost_state) { |
| 0 | 2783 IRScopeDebugInfo* caller_debug_info = NULL; |
| 1819 | 2784 |
| 2785 ValueStack* caller_state = cur_state->caller_state(); | |
| 0 | 2786 if (caller_state != NULL) { |
| 2787 // process recursively to compute outermost scope first | |
| 1819 | 2788 caller_debug_info = compute_debug_info_for_scope(op_id, cur_scope->caller(), caller_state, innermost_state); |
| 0 | 2789 } |
| 2790 | |
| 2791 // initialize these to null. | |
| 2792 // If we don't need deopt info or there are no locals, expressions or monitors, | |
| 2793 // then these get recorded as no information and avoids the allocation of 0 length arrays. | |
| 2794 GrowableArray<ScopeValue*>* locals = NULL; | |
| 2795 GrowableArray<ScopeValue*>* expressions = NULL; | |
| 2796 GrowableArray<MonitorValue*>* monitors = NULL; | |
| 2797 | |
| 2798 // describe local variable values | |
| 1819 | 2799 int nof_locals = cur_state->locals_size(); |
| 0 | 2800 if (nof_locals > 0) { |
| 2801 locals = new GrowableArray<ScopeValue*>(nof_locals); | |
| 2802 | |
| 2803 int pos = 0; | |
| 2804 while (pos < nof_locals) { | |
| 2805 assert(pos < cur_state->locals_size(), "why not?"); | |
| 2806 | |
| 2807 Value local = cur_state->local_at(pos); | |
| 2808 pos += append_scope_value(op_id, local, locals); | |
| 2809 | |
| 2810 assert(locals->length() == pos, "must match"); | |
| 2811 } | |
| 2812 assert(locals->length() == cur_scope->method()->max_locals(), "wrong number of locals"); | |
| 2813 assert(locals->length() == cur_state->locals_size(), "wrong number of locals"); | |
| 1819 | 2814 } else if (cur_scope->method()->max_locals() > 0) { |
| 2815 assert(cur_state->kind() == ValueStack::EmptyExceptionState, "should be"); | |
| 2816 nof_locals = cur_scope->method()->max_locals(); | |
| 2817 locals = new GrowableArray<ScopeValue*>(nof_locals); | |
| 2818 for(int i = 0; i < nof_locals; i++) { | |
| 2819 locals->append(&_illegal_value); | |
| 2820 } | |
| 2821 } | |
| 0 | 2822 |
| 2823 // describe expression stack | |
| 1819 | 2824 int nof_stack = cur_state->stack_size(); |
| 0 | 2825 if (nof_stack > 0) { |
| 2826 expressions = new GrowableArray<ScopeValue*>(nof_stack); | |
| 2827 | |
| 1819 | 2828 int pos = 0; |
| 2829 while (pos < nof_stack) { | |
| 2830 Value expression = cur_state->stack_at_inc(pos); | |
| 0 | 2831 append_scope_value(op_id, expression, expressions); |
| 2832 | |
| 1819 | 2833 assert(expressions->length() == pos, "must match"); |
| 2834 } | |
| 2835 assert(expressions->length() == cur_state->stack_size(), "wrong number of stack entries"); | |
| 0 | 2836 } |
| 2837 | |
| 2838 // describe monitors | |
| 1819 | 2839 int nof_locks = cur_state->locks_size(); |
| 0 | 2840 if (nof_locks > 0) { |
| 1819 | 2841 int lock_offset = cur_state->caller_state() != NULL ? cur_state->caller_state()->total_locks_size() : 0; |
| 0 | 2842 monitors = new GrowableArray<MonitorValue*>(nof_locks); |
| 1819 | 2843 for (int i = 0; i < nof_locks; i++) { |
| 2844 monitors->append(location_for_monitor_index(lock_offset + i)); | |
| 2845 } | |
| 2846 } | |
| 2847 | |
| 2848 return new IRScopeDebugInfo(cur_scope, cur_state->bci(), locals, expressions, monitors, caller_debug_info); | |
| 0 | 2849 } |
| 2850 | |
| 2851 | |
| 2852 void LinearScan::compute_debug_info(CodeEmitInfo* info, int op_id) { | |
| 2853 TRACE_LINEAR_SCAN(3, tty->print_cr("creating debug information at op_id %d", op_id)); | |
| 2854 | |
| 2855 IRScope* innermost_scope = info->scope(); | |
| 2856 ValueStack* innermost_state = info->stack(); | |
| 2857 | |
| 2858 assert(innermost_scope != NULL && innermost_state != NULL, "why is it missing?"); | |
| 2859 | |
| 1819 | 2860 DEBUG_ONLY(check_stack_depth(info, innermost_state->stack_size())); |
| 0 | 2861 |
| 2862 if (info->_scope_debug_info == NULL) { | |
| 2863 // compute debug information | |
| 1819 | 2864 info->_scope_debug_info = compute_debug_info_for_scope(op_id, innermost_scope, innermost_state, innermost_state); |
| 0 | 2865 } else { |
| 2866 // debug information already set. Check that it is correct from the current point of view | |
| 1819 | 2867 DEBUG_ONLY(assert_equal(info->_scope_debug_info, compute_debug_info_for_scope(op_id, innermost_scope, innermost_state, innermost_state))); |
| 0 | 2868 } |
| 2869 } | |
| 2870 | |
| 2871 | |
| 2872 void LinearScan::assign_reg_num(LIR_OpList* instructions, IntervalWalker* iw) { | |
| 2873 LIR_OpVisitState visitor; | |
| 2874 int num_inst = instructions->length(); | |
| 2875 bool has_dead = false; | |
| 2876 | |
| 2877 for (int j = 0; j < num_inst; j++) { | |
| 2878 LIR_Op* op = instructions->at(j); | |
| 2879 if (op == NULL) { // this can happen when spill-moves are removed in eliminate_spill_moves | |
| 2880 has_dead = true; | |
| 2881 continue; | |
| 2882 } | |
| 2883 int op_id = op->id(); | |
| 2884 | |
| 2885 // visit instruction to get list of operands | |
| 2886 visitor.visit(op); | |
| 2887 | |
| 2888 // iterate all modes of the visitor and process all virtual operands | |
| 2889 for_each_visitor_mode(mode) { | |
| 2890 int n = visitor.opr_count(mode); | |
| 2891 for (int k = 0; k < n; k++) { | |
| 2892 LIR_Opr opr = visitor.opr_at(mode, k); | |
| 2893 if (opr->is_virtual_register()) { | |
| 2894 visitor.set_opr_at(mode, k, color_lir_opr(opr, op_id, mode)); | |
| 2895 } | |
| 2896 } | |
| 2897 } | |
| 2898 | |
| 2899 if (visitor.info_count() > 0) { | |
| 2900 // exception handling | |
| 2901 if (compilation()->has_exception_handlers()) { | |
| 2902 XHandlers* xhandlers = visitor.all_xhandler(); | |
| 2903 int n = xhandlers->length(); | |
| 2904 for (int k = 0; k < n; k++) { | |
| 2905 XHandler* handler = xhandlers->handler_at(k); | |
| 2906 if (handler->entry_code() != NULL) { | |
| 2907 assign_reg_num(handler->entry_code()->instructions_list(), NULL); | |
| 2908 } | |
| 2909 } | |
| 2910 } else { | |
| 2911 assert(visitor.all_xhandler()->length() == 0, "missed exception handler"); | |
| 2912 } | |
| 2913 | |
| 2914 // compute oop map | |
| 2915 assert(iw != NULL, "needed for compute_oop_map"); | |
| 2916 compute_oop_map(iw, visitor, op); | |
| 2917 | |
| 2918 // compute debug information | |
| 2919 if (!use_fpu_stack_allocation()) { | |
| 2920 // compute debug information if fpu stack allocation is not needed. | |
| 2921 // when fpu stack allocation is needed, the debug information can not | |
| 2922 // be computed here because the exact location of fpu operands is not known | |
| 2923 // -> debug information is created inside the fpu stack allocator | |
| 2924 int n = visitor.info_count(); | |
| 2925 for (int k = 0; k < n; k++) { | |
| 2926 compute_debug_info(visitor.info_at(k), op_id); | |
| 2927 } | |
| 2928 } | |
| 2929 } | |
| 2930 | |
| 2931 #ifdef ASSERT | |
| 2932 // make sure we haven't made the op invalid. | |
| 2933 op->verify(); | |
| 2934 #endif | |
| 2935 | |
| 2936 // remove useless moves | |
| 2937 if (op->code() == lir_move) { | |
| 2938 assert(op->as_Op1() != NULL, "move must be LIR_Op1"); | |
| 2939 LIR_Op1* move = (LIR_Op1*)op; | |
| 2940 LIR_Opr src = move->in_opr(); | |
| 2941 LIR_Opr dst = move->result_opr(); | |
| 2942 if (dst == src || | |
| 2943 !dst->is_pointer() && !src->is_pointer() && | |
| 2944 src->is_same_register(dst)) { | |
| 2945 instructions->at_put(j, NULL); | |
| 2946 has_dead = true; | |
| 2947 } | |
| 2948 } | |
| 2949 } | |
| 2950 | |
| 2951 if (has_dead) { | |
| 2952 // iterate all instructions of the block and remove all null-values. | |
| 2953 int insert_point = 0; | |
| 2954 for (int j = 0; j < num_inst; j++) { | |
| 2955 LIR_Op* op = instructions->at(j); | |
| 2956 if (op != NULL) { | |
| 2957 if (insert_point != j) { | |
| 2958 instructions->at_put(insert_point, op); | |
| 2959 } | |
| 2960 insert_point++; | |
| 2961 } | |
| 2962 } | |
| 2963 instructions->truncate(insert_point); | |
| 2964 } | |
| 2965 } | |
| 2966 | |
| 2967 void LinearScan::assign_reg_num() { | |
| 2968 TIME_LINEAR_SCAN(timer_assign_reg_num); | |
| 2969 | |
| 2970 init_compute_debug_info(); | |
| 2971 IntervalWalker* iw = init_compute_oop_maps(); | |
| 2972 | |
| 2973 int num_blocks = block_count(); | |
| 2974 for (int i = 0; i < num_blocks; i++) { | |
| 2975 BlockBegin* block = block_at(i); | |
| 2976 assign_reg_num(block->lir()->instructions_list(), iw); | |
| 2977 } | |
| 2978 } | |
| 2979 | |
| 2980 | |
| 2981 void LinearScan::do_linear_scan() { | |
| 2982 NOT_PRODUCT(_total_timer.begin_method()); | |
| 2983 | |
| 2984 number_instructions(); | |
| 2985 | |
| 2986 NOT_PRODUCT(print_lir(1, "Before Register Allocation")); | |
| 2987 | |
| 2988 compute_local_live_sets(); | |
| 2989 compute_global_live_sets(); | |
| 2990 CHECK_BAILOUT(); | |
| 2991 | |
| 2992 build_intervals(); | |
| 2993 CHECK_BAILOUT(); | |
| 2994 sort_intervals_before_allocation(); | |
| 2995 | |
| 2996 NOT_PRODUCT(print_intervals("Before Register Allocation")); | |
| 2997 NOT_PRODUCT(LinearScanStatistic::compute(this, _stat_before_alloc)); | |
| 2998 | |
| 2999 allocate_registers(); | |
| 3000 CHECK_BAILOUT(); | |
| 3001 | |
| 3002 resolve_data_flow(); | |
| 3003 if (compilation()->has_exception_handlers()) { | |
| 3004 resolve_exception_handlers(); | |
| 3005 } | |
| 3006 // fill in number of spill slots into frame_map | |
| 3007 propagate_spill_slots(); | |
| 3008 CHECK_BAILOUT(); | |
| 3009 | |
| 3010 NOT_PRODUCT(print_intervals("After Register Allocation")); | |
| 3011 NOT_PRODUCT(print_lir(2, "LIR after register allocation:")); | |
| 3012 | |
| 3013 sort_intervals_after_allocation(); | |
|
722
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3014 |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3015 DEBUG_ONLY(verify()); |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3016 |
| 0 | 3017 eliminate_spill_moves(); |
| 3018 assign_reg_num(); | |
| 3019 CHECK_BAILOUT(); | |
| 3020 | |
| 3021 NOT_PRODUCT(print_lir(2, "LIR after assignment of register numbers:")); | |
| 3022 NOT_PRODUCT(LinearScanStatistic::compute(this, _stat_after_asign)); | |
| 3023 | |
| 3024 { TIME_LINEAR_SCAN(timer_allocate_fpu_stack); | |
| 3025 | |
| 3026 if (use_fpu_stack_allocation()) { | |
| 3027 allocate_fpu_stack(); // Only has effect on Intel | |
| 3028 NOT_PRODUCT(print_lir(2, "LIR after FPU stack allocation:")); | |
| 3029 } | |
| 3030 } | |
| 3031 | |
| 3032 { TIME_LINEAR_SCAN(timer_optimize_lir); | |
| 3033 | |
| 3034 EdgeMoveOptimizer::optimize(ir()->code()); | |
| 3035 ControlFlowOptimizer::optimize(ir()->code()); | |
| 3036 // check that cfg is still correct after optimizations | |
| 3037 ir()->verify(); | |
| 3038 } | |
| 3039 | |
| 3040 NOT_PRODUCT(print_lir(1, "Before Code Generation", false)); | |
| 3041 NOT_PRODUCT(LinearScanStatistic::compute(this, _stat_final)); | |
| 3042 NOT_PRODUCT(_total_timer.end_method(this)); | |
| 3043 } | |
| 3044 | |
| 3045 | |
| 3046 // ********** Printing functions | |
| 3047 | |
| 3048 #ifndef PRODUCT | |
| 3049 | |
| 3050 void LinearScan::print_timers(double total) { | |
| 3051 _total_timer.print(total); | |
| 3052 } | |
| 3053 | |
| 3054 void LinearScan::print_statistics() { | |
| 3055 _stat_before_alloc.print("before allocation"); | |
| 3056 _stat_after_asign.print("after assignment of register"); | |
| 3057 _stat_final.print("after optimization"); | |
| 3058 } | |
| 3059 | |
| 3060 void LinearScan::print_bitmap(BitMap& b) { | |
| 3061 for (unsigned int i = 0; i < b.size(); i++) { | |
| 3062 if (b.at(i)) tty->print("%d ", i); | |
| 3063 } | |
| 3064 tty->cr(); | |
| 3065 } | |
| 3066 | |
| 3067 void LinearScan::print_intervals(const char* label) { | |
| 3068 if (TraceLinearScanLevel >= 1) { | |
| 3069 int i; | |
| 3070 tty->cr(); | |
| 3071 tty->print_cr("%s", label); | |
| 3072 | |
| 3073 for (i = 0; i < interval_count(); i++) { | |
| 3074 Interval* interval = interval_at(i); | |
| 3075 if (interval != NULL) { | |
| 3076 interval->print(); | |
| 3077 } | |
| 3078 } | |
| 3079 | |
| 3080 tty->cr(); | |
| 3081 tty->print_cr("--- Basic Blocks ---"); | |
| 3082 for (i = 0; i < block_count(); i++) { | |
| 3083 BlockBegin* block = block_at(i); | |
| 3084 tty->print("B%d [%d, %d, %d, %d] ", block->block_id(), block->first_lir_instruction_id(), block->last_lir_instruction_id(), block->loop_index(), block->loop_depth()); | |
| 3085 } | |
| 3086 tty->cr(); | |
| 3087 tty->cr(); | |
| 3088 } | |
| 3089 | |
| 3090 if (PrintCFGToFile) { | |
| 3091 CFGPrinter::print_intervals(&_intervals, label); | |
| 3092 } | |
| 3093 } | |
| 3094 | |
| 3095 void LinearScan::print_lir(int level, const char* label, bool hir_valid) { | |
| 3096 if (TraceLinearScanLevel >= level) { | |
| 3097 tty->cr(); | |
| 3098 tty->print_cr("%s", label); | |
| 3099 print_LIR(ir()->linear_scan_order()); | |
| 3100 tty->cr(); | |
| 3101 } | |
| 3102 | |
| 3103 if (level == 1 && PrintCFGToFile) { | |
| 3104 CFGPrinter::print_cfg(ir()->linear_scan_order(), label, hir_valid, true); | |
| 3105 } | |
| 3106 } | |
| 3107 | |
| 3108 #endif //PRODUCT | |
| 3109 | |
| 3110 | |
| 3111 // ********** verification functions for allocation | |
| 3112 // (check that all intervals have a correct register and that no registers are overwritten) | |
| 3113 #ifdef ASSERT | |
| 3114 | |
| 3115 void LinearScan::verify() { | |
| 3116 TRACE_LINEAR_SCAN(2, tty->print_cr("********* verifying intervals ******************************************")); | |
| 3117 verify_intervals(); | |
| 3118 | |
| 3119 TRACE_LINEAR_SCAN(2, tty->print_cr("********* verifying that no oops are in fixed intervals ****************")); | |
| 3120 verify_no_oops_in_fixed_intervals(); | |
| 3121 | |
| 3122 TRACE_LINEAR_SCAN(2, tty->print_cr("********* verifying that unpinned constants are not alive across block boundaries")); | |
| 3123 verify_constants(); | |
| 3124 | |
| 3125 TRACE_LINEAR_SCAN(2, tty->print_cr("********* verifying register allocation ********************************")); | |
| 3126 verify_registers(); | |
| 3127 | |
| 3128 TRACE_LINEAR_SCAN(2, tty->print_cr("********* no errors found **********************************************")); | |
| 3129 } | |
| 3130 | |
| 3131 void LinearScan::verify_intervals() { | |
| 3132 int len = interval_count(); | |
| 3133 bool has_error = false; | |
| 3134 | |
| 3135 for (int i = 0; i < len; i++) { | |
| 3136 Interval* i1 = interval_at(i); | |
| 3137 if (i1 == NULL) continue; | |
| 3138 | |
| 3139 i1->check_split_children(); | |
| 3140 | |
| 3141 if (i1->reg_num() != i) { | |
| 3142 tty->print_cr("Interval %d is on position %d in list", i1->reg_num(), i); i1->print(); tty->cr(); | |
| 3143 has_error = true; | |
| 3144 } | |
| 3145 | |
| 3146 if (i1->reg_num() >= LIR_OprDesc::vreg_base && i1->type() == T_ILLEGAL) { | |
| 3147 tty->print_cr("Interval %d has no type assigned", i1->reg_num()); i1->print(); tty->cr(); | |
| 3148 has_error = true; | |
| 3149 } | |
| 3150 | |
| 3151 if (i1->assigned_reg() == any_reg) { | |
| 3152 tty->print_cr("Interval %d has no register assigned", i1->reg_num()); i1->print(); tty->cr(); | |
| 3153 has_error = true; | |
| 3154 } | |
| 3155 | |
| 3156 if (i1->assigned_reg() == i1->assigned_regHi()) { | |
| 3157 tty->print_cr("Interval %d: low and high register equal", i1->reg_num()); i1->print(); tty->cr(); | |
| 3158 has_error = true; | |
| 3159 } | |
| 3160 | |
| 3161 if (!is_processed_reg_num(i1->assigned_reg())) { | |
| 3162 tty->print_cr("Can not have an Interval for an ignored register"); i1->print(); tty->cr(); | |
| 3163 has_error = true; | |
| 3164 } | |
| 3165 | |
| 3166 if (i1->first() == Range::end()) { | |
| 3167 tty->print_cr("Interval %d has no Range", i1->reg_num()); i1->print(); tty->cr(); | |
| 3168 has_error = true; | |
| 3169 } | |
| 3170 | |
| 3171 for (Range* r = i1->first(); r != Range::end(); r = r->next()) { | |
| 3172 if (r->from() >= r->to()) { | |
| 3173 tty->print_cr("Interval %d has zero length range", i1->reg_num()); i1->print(); tty->cr(); | |
| 3174 has_error = true; | |
| 3175 } | |
| 3176 } | |
| 3177 | |
| 3178 for (int j = i + 1; j < len; j++) { | |
| 3179 Interval* i2 = interval_at(j); | |
| 3180 if (i2 == NULL) continue; | |
| 3181 | |
| 3182 // special intervals that are created in MoveResolver | |
| 3183 // -> ignore them because the range information has no meaning there | |
| 3184 if (i1->from() == 1 && i1->to() == 2) continue; | |
| 3185 if (i2->from() == 1 && i2->to() == 2) continue; | |
| 3186 | |
| 3187 int r1 = i1->assigned_reg(); | |
| 3188 int r1Hi = i1->assigned_regHi(); | |
| 3189 int r2 = i2->assigned_reg(); | |
| 3190 int r2Hi = i2->assigned_regHi(); | |
| 3191 if (i1->intersects(i2) && (r1 == r2 || r1 == r2Hi || (r1Hi != any_reg && (r1Hi == r2 || r1Hi == r2Hi)))) { | |
| 3192 tty->print_cr("Intervals %d and %d overlap and have the same register assigned", i1->reg_num(), i2->reg_num()); | |
| 3193 i1->print(); tty->cr(); | |
| 3194 i2->print(); tty->cr(); | |
| 3195 has_error = true; | |
| 3196 } | |
| 3197 } | |
| 3198 } | |
| 3199 | |
| 3200 assert(has_error == false, "register allocation invalid"); | |
| 3201 } | |
| 3202 | |
| 3203 | |
| 3204 void LinearScan::verify_no_oops_in_fixed_intervals() { | |
|
722
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3205 Interval* fixed_intervals; |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3206 Interval* other_intervals; |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3207 create_unhandled_lists(&fixed_intervals, &other_intervals, is_precolored_cpu_interval, NULL); |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3208 |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3209 // to ensure a walking until the last instruction id, add a dummy interval |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3210 // with a high operation id |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3211 other_intervals = new Interval(any_reg); |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3212 other_intervals->add_range(max_jint - 2, max_jint - 1); |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3213 IntervalWalker* iw = new IntervalWalker(this, fixed_intervals, other_intervals); |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3214 |
| 0 | 3215 LIR_OpVisitState visitor; |
| 3216 for (int i = 0; i < block_count(); i++) { | |
| 3217 BlockBegin* block = block_at(i); | |
| 3218 | |
| 3219 LIR_OpList* instructions = block->lir()->instructions_list(); | |
| 3220 | |
| 3221 for (int j = 0; j < instructions->length(); j++) { | |
| 3222 LIR_Op* op = instructions->at(j); | |
| 3223 int op_id = op->id(); | |
| 3224 | |
| 3225 visitor.visit(op); | |
| 3226 | |
|
722
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3227 if (visitor.info_count() > 0) { |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3228 iw->walk_before(op->id()); |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3229 bool check_live = true; |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3230 if (op->code() == lir_move) { |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3231 LIR_Op1* move = (LIR_Op1*)op; |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3232 check_live = (move->patch_code() == lir_patch_none); |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3233 } |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3234 LIR_OpBranch* branch = op->as_OpBranch(); |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3235 if (branch != NULL && branch->stub() != NULL && branch->stub()->is_exception_throw_stub()) { |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3236 // Don't bother checking the stub in this case since the |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3237 // exception stub will never return to normal control flow. |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3238 check_live = false; |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3239 } |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3240 |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3241 // Make sure none of the fixed registers is live across an |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3242 // oopmap since we can't handle that correctly. |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3243 if (check_live) { |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3244 for (Interval* interval = iw->active_first(fixedKind); |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3245 interval != Interval::end(); |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3246 interval = interval->next()) { |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3247 if (interval->current_to() > op->id() + 1) { |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3248 // This interval is live out of this op so make sure |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3249 // that this interval represents some value that's |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3250 // referenced by this op either as an input or output. |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3251 bool ok = false; |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3252 for_each_visitor_mode(mode) { |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3253 int n = visitor.opr_count(mode); |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3254 for (int k = 0; k < n; k++) { |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3255 LIR_Opr opr = visitor.opr_at(mode, k); |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3256 if (opr->is_fixed_cpu()) { |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3257 if (interval_at(reg_num(opr)) == interval) { |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3258 ok = true; |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3259 break; |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3260 } |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3261 int hi = reg_numHi(opr); |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3262 if (hi != -1 && interval_at(hi) == interval) { |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3263 ok = true; |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3264 break; |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3265 } |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3266 } |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3267 } |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3268 } |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3269 assert(ok, "fixed intervals should never be live across an oopmap point"); |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3270 } |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3271 } |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3272 } |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3273 } |
|
a134d9824964
6828024: verification of fixed interval usage is too weak
never
parents:
337
diff
changeset
|
3274 |
| 0 | 3275 // oop-maps at calls do not contain registers, so check is not needed |
| 3276 if (!visitor.has_call()) { | |
| 3277 | |
| 3278 for_each_visitor_mode(mode) { | |
| 3279 int n = visitor.opr_count(mode); | |
| 3280 for (int k = 0; k < n; k++) { | |
| 3281 LIR_Opr opr = visitor.opr_at(mode, k); | |
| 3282 | |
| 3283 if (opr->is_fixed_cpu() && opr->is_oop()) { | |
| 3284 // operand is a non-virtual cpu register and contains an oop | |
| 3285 TRACE_LINEAR_SCAN(4, op->print_on(tty); tty->print("checking operand "); opr->print(); tty->cr()); | |
| 3286 | |
| 3287 Interval* interval = interval_at(reg_num(opr)); | |
| 3288 assert(interval != NULL, "no interval"); | |
| 3289 | |
| 3290 if (mode == LIR_OpVisitState::inputMode) { | |
| 3291 if (interval->to() >= op_id + 1) { | |
| 3292 assert(interval->to() < op_id + 2 || | |
| 3293 interval->has_hole_between(op_id, op_id + 2), | |
| 3294 "oop input operand live after instruction"); | |
| 3295 } | |
| 3296 } else if (mode == LIR_OpVisitState::outputMode) { | |
| 3297 if (interval->from() <= op_id - 1) { | |
| 3298 assert(interval->has_hole_between(op_id - 1, op_id), | |
| 3299 "oop input operand live after instruction"); | |
| 3300 } | |
| 3301 } | |
| 3302 } | |
| 3303 } | |
| 3304 } | |
| 3305 } | |
| 3306 } | |
| 3307 } | |
| 3308 } | |
| 3309 | |
| 3310 | |
| 3311 void LinearScan::verify_constants() { | |
| 3312 int num_regs = num_virtual_regs(); | |
| 3313 int size = live_set_size(); | |
| 3314 int num_blocks = block_count(); | |
| 3315 | |
| 3316 for (int i = 0; i < num_blocks; i++) { | |
| 3317 BlockBegin* block = block_at(i); | |
| 3318 BitMap live_at_edge = block->live_in(); | |
| 3319 | |
| 3320 // visit all registers where the live_at_edge bit is set | |
| 304 | 3321 for (int r = (int)live_at_edge.get_next_one_offset(0, size); r < size; r = (int)live_at_edge.get_next_one_offset(r + 1, size)) { |
| 0 | 3322 TRACE_LINEAR_SCAN(4, tty->print("checking interval %d of block B%d", r, block->block_id())); |
| 3323 | |
| 3324 Value value = gen()->instruction_for_vreg(r); | |
| 3325 | |
| 3326 assert(value != NULL, "all intervals live across block boundaries must have Value"); | |
| 3327 assert(value->operand()->is_register() && value->operand()->is_virtual(), "value must have virtual operand"); | |
| 3328 assert(value->operand()->vreg_number() == r, "register number must match"); | |
| 3329 // TKR assert(value->as_Constant() == NULL || value->is_pinned(), "only pinned constants can be alive accross block boundaries"); | |
| 3330 } | |
| 3331 } | |
| 3332 } | |
| 3333 | |
| 3334 | |
| 3335 class RegisterVerifier: public StackObj { | |
| 3336 private: | |
| 3337 LinearScan* _allocator; | |
| 3338 BlockList _work_list; // all blocks that must be processed | |
| 3339 IntervalsList _saved_states; // saved information of previous check | |
| 3340 | |
| 3341 // simplified access to methods of LinearScan | |
| 3342 Compilation* compilation() const { return _allocator->compilation(); } | |
| 3343 Interval* interval_at(int reg_num) const { return _allocator->interval_at(reg_num); } | |
| 3344 int reg_num(LIR_Opr opr) const { return _allocator->reg_num(opr); } | |
| 3345 | |
| 3346 // currently, only registers are processed | |
| 3347 int state_size() { return LinearScan::nof_regs; } | |
| 3348 | |
| 3349 // accessors | |
| 3350 IntervalList* state_for_block(BlockBegin* block) { return _saved_states.at(block->block_id()); } | |
| 3351 void set_state_for_block(BlockBegin* block, IntervalList* saved_state) { _saved_states.at_put(block->block_id(), saved_state); } | |
| 3352 void add_to_work_list(BlockBegin* block) { if (!_work_list.contains(block)) _work_list.append(block); } | |
| 3353 | |
| 3354 // helper functions | |
| 3355 IntervalList* copy(IntervalList* input_state); | |
| 3356 void state_put(IntervalList* input_state, int reg, Interval* interval); | |
| 3357 bool check_state(IntervalList* input_state, int reg, Interval* interval); | |
| 3358 | |
| 3359 void process_block(BlockBegin* block); | |
| 3360 void process_xhandler(XHandler* xhandler, IntervalList* input_state); | |
| 3361 void process_successor(BlockBegin* block, IntervalList* input_state); | |
| 3362 void process_operations(LIR_List* ops, IntervalList* input_state); | |
| 3363 | |
| 3364 public: | |
| 3365 RegisterVerifier(LinearScan* allocator) | |
| 3366 : _allocator(allocator) | |
| 3367 , _work_list(16) | |
| 3368 , _saved_states(BlockBegin::number_of_blocks(), NULL) | |
| 3369 { } | |
| 3370 | |
| 3371 void verify(BlockBegin* start); | |
| 3372 }; | |
| 3373 | |
| 3374 | |
| 3375 // entry function from LinearScan that starts the verification | |
| 3376 void LinearScan::verify_registers() { | |
| 3377 RegisterVerifier verifier(this); | |
| 3378 verifier.verify(block_at(0)); | |
| 3379 } | |
| 3380 | |
| 3381 | |
| 3382 void RegisterVerifier::verify(BlockBegin* start) { | |
| 3383 // setup input registers (method arguments) for first block | |
| 3384 IntervalList* input_state = new IntervalList(state_size(), NULL); | |
| 3385 CallingConvention* args = compilation()->frame_map()->incoming_arguments(); | |
| 3386 for (int n = 0; n < args->length(); n++) { | |
| 3387 LIR_Opr opr = args->at(n); | |
| 3388 if (opr->is_register()) { | |
| 3389 Interval* interval = interval_at(reg_num(opr)); | |
| 3390 | |
| 3391 if (interval->assigned_reg() < state_size()) { | |
| 3392 input_state->at_put(interval->assigned_reg(), interval); | |
| 3393 } | |
| 3394 if (interval->assigned_regHi() != LinearScan::any_reg && interval->assigned_regHi() < state_size()) { | |
| 3395 input_state->at_put(interval->assigned_regHi(), interval); | |
| 3396 } | |
| 3397 } | |
| 3398 } | |
| 3399 | |
| 3400 set_state_for_block(start, input_state); | |
| 3401 add_to_work_list(start); | |
| 3402 | |
| 3403 // main loop for verification | |
| 3404 do { | |
| 3405 BlockBegin* block = _work_list.at(0); | |
| 3406 _work_list.remove_at(0); | |
| 3407 | |
| 3408 process_block(block); | |
| 3409 } while (!_work_list.is_empty()); | |
| 3410 } | |
| 3411 | |
| 3412 void RegisterVerifier::process_block(BlockBegin* block) { | |
| 3413 TRACE_LINEAR_SCAN(2, tty->cr(); tty->print_cr("process_block B%d", block->block_id())); | |
| 3414 | |
| 3415 // must copy state because it is modified | |
| 3416 IntervalList* input_state = copy(state_for_block(block)); | |
| 3417 | |
| 3418 if (TraceLinearScanLevel >= 4) { | |
| 3419 tty->print_cr("Input-State of intervals:"); | |
| 3420 tty->print(" "); | |
| 3421 for (int i = 0; i < state_size(); i++) { | |
| 3422 if (input_state->at(i) != NULL) { | |
| 3423 tty->print(" %4d", input_state->at(i)->reg_num()); | |
| 3424 } else { | |
| 3425 tty->print(" __"); | |
| 3426 } | |
| 3427 } | |
| 3428 tty->cr(); | |
| 3429 tty->cr(); | |
| 3430 } | |
| 3431 | |
| 3432 // process all operations of the block | |
| 3433 process_operations(block->lir(), input_state); | |
| 3434 | |
| 3435 // iterate all successors | |
| 3436 for (int i = 0; i < block->number_of_sux(); i++) { | |
| 3437 process_successor(block->sux_at(i), input_state); | |
| 3438 } | |
| 3439 } | |
| 3440 | |
| 3441 void RegisterVerifier::process_xhandler(XHandler* xhandler, IntervalList* input_state) { | |
| 3442 TRACE_LINEAR_SCAN(2, tty->print_cr("process_xhandler B%d", xhandler->entry_block()->block_id())); | |
| 3443 | |
| 3444 // must copy state because it is modified | |
| 3445 input_state = copy(input_state); | |
| 3446 | |
| 3447 if (xhandler->entry_code() != NULL) { | |
| 3448 process_operations(xhandler->entry_code(), input_state); | |
| 3449 } | |
| 3450 process_successor(xhandler->entry_block(), input_state); | |
| 3451 } | |
| 3452 | |
| 3453 void RegisterVerifier::process_successor(BlockBegin* block, IntervalList* input_state) { | |
| 3454 IntervalList* saved_state = state_for_block(block); | |
| 3455 | |
| 3456 if (saved_state != NULL) { | |
| 3457 // this block was already processed before. | |
| 3458 // check if new input_state is consistent with saved_state | |
| 3459 | |
| 3460 bool saved_state_correct = true; | |
| 3461 for (int i = 0; i < state_size(); i++) { | |
| 3462 if (input_state->at(i) != saved_state->at(i)) { | |
| 3463 // current input_state and previous saved_state assume a different | |
| 3464 // interval in this register -> assume that this register is invalid | |
| 3465 if (saved_state->at(i) != NULL) { | |
| 3466 // invalidate old calculation only if it assumed that | |
| 3467 // register was valid. when the register was already invalid, | |
| 3468 // then the old calculation was correct. | |
| 3469 saved_state_correct = false; | |
| 3470 saved_state->at_put(i, NULL); | |
| 3471 | |
| 3472 TRACE_LINEAR_SCAN(4, tty->print_cr("process_successor B%d: invalidating slot %d", block->block_id(), i)); | |
| 3473 } | |
| 3474 } | |
| 3475 } | |
| 3476 | |
| 3477 if (saved_state_correct) { | |
| 3478 // already processed block with correct input_state | |
| 3479 TRACE_LINEAR_SCAN(2, tty->print_cr("process_successor B%d: previous visit already correct", block->block_id())); | |
| 3480 } else { | |
| 3481 // must re-visit this block | |
| 3482 TRACE_LINEAR_SCAN(2, tty->print_cr("process_successor B%d: must re-visit because input state changed", block->block_id())); | |
| 3483 add_to_work_list(block); | |
| 3484 } | |
| 3485 | |
| 3486 } else { | |
| 3487 // block was not processed before, so set initial input_state | |
| 3488 TRACE_LINEAR_SCAN(2, tty->print_cr("process_successor B%d: initial visit", block->block_id())); | |
| 3489 | |
| 3490 set_state_for_block(block, copy(input_state)); | |
| 3491 add_to_work_list(block); | |
| 3492 } | |
| 3493 } | |
| 3494 | |
| 3495 | |
| 3496 IntervalList* RegisterVerifier::copy(IntervalList* input_state) { | |
| 3497 IntervalList* copy_state = new IntervalList(input_state->length()); | |
| 3498 copy_state->push_all(input_state); | |
| 3499 return copy_state; | |
| 3500 } | |
| 3501 | |
| 3502 void RegisterVerifier::state_put(IntervalList* input_state, int reg, Interval* interval) { | |
| 3503 if (reg != LinearScan::any_reg && reg < state_size()) { | |
| 3504 if (interval != NULL) { | |
| 3505 TRACE_LINEAR_SCAN(4, tty->print_cr(" reg[%d] = %d", reg, interval->reg_num())); | |
| 3506 } else if (input_state->at(reg) != NULL) { | |
| 3507 TRACE_LINEAR_SCAN(4, tty->print_cr(" reg[%d] = NULL", reg)); | |
| 3508 } | |
| 3509 | |
| 3510 input_state->at_put(reg, interval); | |
| 3511 } | |
| 3512 } | |
| 3513 | |
| 3514 bool RegisterVerifier::check_state(IntervalList* input_state, int reg, Interval* interval) { | |
| 3515 if (reg != LinearScan::any_reg && reg < state_size()) { | |
| 3516 if (input_state->at(reg) != interval) { | |
| 3517 tty->print_cr("!! Error in register allocation: register %d does not contain interval %d", reg, interval->reg_num()); | |
| 3518 return true; | |
| 3519 } | |
| 3520 } | |
| 3521 return false; | |
| 3522 } | |
| 3523 | |
| 3524 void RegisterVerifier::process_operations(LIR_List* ops, IntervalList* input_state) { | |
| 3525 // visit all instructions of the block | |
| 3526 LIR_OpVisitState visitor; | |
| 3527 bool has_error = false; | |
| 3528 | |
| 3529 for (int i = 0; i < ops->length(); i++) { | |
| 3530 LIR_Op* op = ops->at(i); | |
| 3531 visitor.visit(op); | |
| 3532 | |
| 3533 TRACE_LINEAR_SCAN(4, op->print_on(tty)); | |
| 3534 | |
| 3535 // check if input operands are correct | |
| 3536 int j; | |
| 3537 int n = visitor.opr_count(LIR_OpVisitState::inputMode); | |
| 3538 for (j = 0; j < n; j++) { | |
| 3539 LIR_Opr opr = visitor.opr_at(LIR_OpVisitState::inputMode, j); | |
| 3540 if (opr->is_register() && LinearScan::is_processed_reg_num(reg_num(opr))) { | |
| 3541 Interval* interval = interval_at(reg_num(opr)); | |
| 3542 if (op->id() != -1) { | |
| 3543 interval = interval->split_child_at_op_id(op->id(), LIR_OpVisitState::inputMode); | |
| 3544 } | |
| 3545 | |
| 3546 has_error |= check_state(input_state, interval->assigned_reg(), interval->split_parent()); | |
| 3547 has_error |= check_state(input_state, interval->assigned_regHi(), interval->split_parent()); | |
| 3548 | |
| 3549 // When an operand is marked with is_last_use, then the fpu stack allocator | |
| 3550 // removes the register from the fpu stack -> the register contains no value | |
| 3551 if (opr->is_last_use()) { | |
| 3552 state_put(input_state, interval->assigned_reg(), NULL); | |
| 3553 state_put(input_state, interval->assigned_regHi(), NULL); | |
| 3554 } | |
| 3555 } | |
| 3556 } | |
| 3557 | |
| 3558 // invalidate all caller save registers at calls | |
| 3559 if (visitor.has_call()) { | |
| 3560 for (j = 0; j < FrameMap::nof_caller_save_cpu_regs; j++) { | |
| 3561 state_put(input_state, reg_num(FrameMap::caller_save_cpu_reg_at(j)), NULL); | |
| 3562 } | |
| 3563 for (j = 0; j < FrameMap::nof_caller_save_fpu_regs; j++) { | |
| 3564 state_put(input_state, reg_num(FrameMap::caller_save_fpu_reg_at(j)), NULL); | |
| 3565 } | |
| 3566 | |
| 304 | 3567 #ifdef X86 |
| 0 | 3568 for (j = 0; j < FrameMap::nof_caller_save_xmm_regs; j++) { |
| 3569 state_put(input_state, reg_num(FrameMap::caller_save_xmm_reg_at(j)), NULL); | |
| 3570 } | |
| 3571 #endif | |
| 3572 } | |
| 3573 | |
| 3574 // process xhandler before output and temp operands | |
| 3575 XHandlers* xhandlers = visitor.all_xhandler(); | |
| 3576 n = xhandlers->length(); | |
| 3577 for (int k = 0; k < n; k++) { | |
| 3578 process_xhandler(xhandlers->handler_at(k), input_state); | |
| 3579 } | |
| 3580 | |
| 3581 // set temp operands (some operations use temp operands also as output operands, so can't set them NULL) | |
| 3582 n = visitor.opr_count(LIR_OpVisitState::tempMode); | |
| 3583 for (j = 0; j < n; j++) { | |
| 3584 LIR_Opr opr = visitor.opr_at(LIR_OpVisitState::tempMode, j); | |
| 3585 if (opr->is_register() && LinearScan::is_processed_reg_num(reg_num(opr))) { | |
| 3586 Interval* interval = interval_at(reg_num(opr)); | |
| 3587 if (op->id() != -1) { | |
| 3588 interval = interval->split_child_at_op_id(op->id(), LIR_OpVisitState::tempMode); | |
| 3589 } | |
| 3590 | |
| 3591 state_put(input_state, interval->assigned_reg(), interval->split_parent()); | |
| 3592 state_put(input_state, interval->assigned_regHi(), interval->split_parent()); | |
| 3593 } | |
| 3594 } | |
| 3595 | |
| 3596 // set output operands | |
| 3597 n = visitor.opr_count(LIR_OpVisitState::outputMode); | |
| 3598 for (j = 0; j < n; j++) { | |
| 3599 LIR_Opr opr = visitor.opr_at(LIR_OpVisitState::outputMode, j); | |
| 3600 if (opr->is_register() && LinearScan::is_processed_reg_num(reg_num(opr))) { | |
| 3601 Interval* interval = interval_at(reg_num(opr)); | |
| 3602 if (op->id() != -1) { | |
| 3603 interval = interval->split_child_at_op_id(op->id(), LIR_OpVisitState::outputMode); | |
| 3604 } | |
| 3605 | |
| 3606 state_put(input_state, interval->assigned_reg(), interval->split_parent()); | |
| 3607 state_put(input_state, interval->assigned_regHi(), interval->split_parent()); | |
| 3608 } | |
| 3609 } | |
| 3610 } | |
| 3611 assert(has_error == false, "Error in register allocation"); | |
| 3612 } | |
| 3613 | |
| 3614 #endif // ASSERT | |
| 3615 | |
| 3616 | |
| 3617 | |
| 3618 // **** Implementation of MoveResolver ****************************** | |
| 3619 | |
| 3620 MoveResolver::MoveResolver(LinearScan* allocator) : | |
| 3621 _allocator(allocator), | |
| 3622 _multiple_reads_allowed(false), | |
| 3623 _mapping_from(8), | |
| 3624 _mapping_from_opr(8), | |
| 3625 _mapping_to(8), | |
| 3626 _insert_list(NULL), | |
| 3627 _insert_idx(-1), | |
| 3628 _insertion_buffer() | |
| 3629 { | |
| 3630 for (int i = 0; i < LinearScan::nof_regs; i++) { | |
| 3631 _register_blocked[i] = 0; | |
| 3632 } | |
| 3633 DEBUG_ONLY(check_empty()); | |
| 3634 } | |
| 3635 | |
| 3636 | |
| 3637 #ifdef ASSERT | |
| 3638 | |
| 3639 void MoveResolver::check_empty() { | |
| 3640 assert(_mapping_from.length() == 0 && _mapping_from_opr.length() == 0 && _mapping_to.length() == 0, "list must be empty before and after processing"); | |
| 3641 for (int i = 0; i < LinearScan::nof_regs; i++) { | |
| 3642 assert(register_blocked(i) == 0, "register map must be empty before and after processing"); | |
| 3643 } | |
| 3644 assert(_multiple_reads_allowed == false, "must have default value"); | |
| 3645 } | |
| 3646 | |
| 3647 void MoveResolver::verify_before_resolve() { | |
| 3648 assert(_mapping_from.length() == _mapping_from_opr.length(), "length must be equal"); | |
| 3649 assert(_mapping_from.length() == _mapping_to.length(), "length must be equal"); | |
| 3650 assert(_insert_list != NULL && _insert_idx != -1, "insert position not set"); | |
| 3651 | |
| 3652 int i, j; | |
| 3653 if (!_multiple_reads_allowed) { | |
| 3654 for (i = 0; i < _mapping_from.length(); i++) { | |
| 3655 for (j = i + 1; j < _mapping_from.length(); j++) { | |
| 3656 assert(_mapping_from.at(i) == NULL || _mapping_from.at(i) != _mapping_from.at(j), "cannot read from same interval twice"); | |
| 3657 } | |
| 3658 } | |
| 3659 } | |
| 3660 | |
| 3661 for (i = 0; i < _mapping_to.length(); i++) { | |
| 3662 for (j = i + 1; j < _mapping_to.length(); j++) { | |
| 3663 assert(_mapping_to.at(i) != _mapping_to.at(j), "cannot write to same interval twice"); | |
| 3664 } | |
| 3665 } | |
| 3666 | |
| 3667 | |
| 3668 BitMap used_regs(LinearScan::nof_regs + allocator()->frame_map()->argcount() + allocator()->max_spills()); | |
| 3669 used_regs.clear(); | |
| 3670 if (!_multiple_reads_allowed) { | |
| 3671 for (i = 0; i < _mapping_from.length(); i++) { | |
| 3672 Interval* it = _mapping_from.at(i); | |
| 3673 if (it != NULL) { | |
| 3674 assert(!used_regs.at(it->assigned_reg()), "cannot read from same register twice"); | |
| 3675 used_regs.set_bit(it->assigned_reg()); | |
| 3676 | |
| 3677 if (it->assigned_regHi() != LinearScan::any_reg) { | |
| 3678 assert(!used_regs.at(it->assigned_regHi()), "cannot read from same register twice"); | |
| 3679 used_regs.set_bit(it->assigned_regHi()); | |
| 3680 } | |
| 3681 } | |
| 3682 } | |
| 3683 } | |
| 3684 | |
| 3685 used_regs.clear(); | |
| 3686 for (i = 0; i < _mapping_to.length(); i++) { | |
| 3687 Interval* it = _mapping_to.at(i); | |
| 3688 assert(!used_regs.at(it->assigned_reg()), "cannot write to same register twice"); | |
| 3689 used_regs.set_bit(it->assigned_reg()); | |
| 3690 | |
| 3691 if (it->assigned_regHi() != LinearScan::any_reg) { | |
| 3692 assert(!used_regs.at(it->assigned_regHi()), "cannot write to same register twice"); | |
| 3693 used_regs.set_bit(it->assigned_regHi()); | |
| 3694 } | |
| 3695 } | |
| 3696 | |
| 3697 used_regs.clear(); | |
| 3698 for (i = 0; i < _mapping_from.length(); i++) { | |
| 3699 Interval* it = _mapping_from.at(i); | |
| 3700 if (it != NULL && it->assigned_reg() >= LinearScan::nof_regs) { | |
| 3701 used_regs.set_bit(it->assigned_reg()); | |
| 3702 } | |
| 3703 } | |
| 3704 for (i = 0; i < _mapping_to.length(); i++) { | |
| 3705 Interval* it = _mapping_to.at(i); | |
| 3706 assert(!used_regs.at(it->assigned_reg()) || it->assigned_reg() == _mapping_from.at(i)->assigned_reg(), "stack slots used in _mapping_from must be disjoint to _mapping_to"); | |
| 3707 } | |
| 3708 } | |
| 3709 | |
| 3710 #endif // ASSERT | |
| 3711 | |
| 3712 | |
| 3713 // mark assigned_reg and assigned_regHi of the interval as blocked | |
| 3714 void MoveResolver::block_registers(Interval* it) { | |
| 3715 int reg = it->assigned_reg(); | |
| 3716 if (reg < LinearScan::nof_regs) { | |
| 3717 assert(_multiple_reads_allowed || register_blocked(reg) == 0, "register already marked as used"); | |
| 3718 set_register_blocked(reg, 1); | |
| 3719 } | |
| 3720 reg = it->assigned_regHi(); | |
| 3721 if (reg != LinearScan::any_reg && reg < LinearScan::nof_regs) { | |
| 3722 assert(_multiple_reads_allowed || register_blocked(reg) == 0, "register already marked as used"); | |
| 3723 set_register_blocked(reg, 1); | |
| 3724 } | |
| 3725 } | |
| 3726 | |
| 3727 // mark assigned_reg and assigned_regHi of the interval as unblocked | |
| 3728 void MoveResolver::unblock_registers(Interval* it) { | |
| 3729 int reg = it->assigned_reg(); | |
| 3730 if (reg < LinearScan::nof_regs) { | |
| 3731 assert(register_blocked(reg) > 0, "register already marked as unused"); | |
| 3732 set_register_blocked(reg, -1); | |
| 3733 } | |
| 3734 reg = it->assigned_regHi(); | |
| 3735 if (reg != LinearScan::any_reg && reg < LinearScan::nof_regs) { | |
| 3736 assert(register_blocked(reg) > 0, "register already marked as unused"); | |
| 3737 set_register_blocked(reg, -1); | |
| 3738 } | |
| 3739 } | |
| 3740 | |
| 3741 // check if assigned_reg and assigned_regHi of the to-interval are not blocked (or only blocked by from) | |
| 3742 bool MoveResolver::save_to_process_move(Interval* from, Interval* to) { | |
| 3743 int from_reg = -1; | |
| 3744 int from_regHi = -1; | |
| 3745 if (from != NULL) { | |
| 3746 from_reg = from->assigned_reg(); | |
| 3747 from_regHi = from->assigned_regHi(); | |
| 3748 } | |
| 3749 | |
| 3750 int reg = to->assigned_reg(); | |
| 3751 if (reg < LinearScan::nof_regs) { | |
| 3752 if (register_blocked(reg) > 1 || (register_blocked(reg) == 1 && reg != from_reg && reg != from_regHi)) { | |
| 3753 return false; | |
| 3754 } | |
| 3755 } | |
| 3756 reg = to->assigned_regHi(); | |
| 3757 if (reg != LinearScan::any_reg && reg < LinearScan::nof_regs) { | |
| 3758 if (register_blocked(reg) > 1 || (register_blocked(reg) == 1 && reg != from_reg && reg != from_regHi)) { | |
| 3759 return false; | |
| 3760 } | |
| 3761 } | |
| 3762 | |
| 3763 return true; | |
| 3764 } | |
| 3765 | |
| 3766 | |
| 3767 void MoveResolver::create_insertion_buffer(LIR_List* list) { | |
| 3768 assert(!_insertion_buffer.initialized(), "overwriting existing buffer"); | |
| 3769 _insertion_buffer.init(list); | |
| 3770 } | |
| 3771 | |
| 3772 void MoveResolver::append_insertion_buffer() { | |
| 3773 if (_insertion_buffer.initialized()) { | |
| 3774 _insertion_buffer.lir_list()->append(&_insertion_buffer); | |
| 3775 } | |
| 3776 assert(!_insertion_buffer.initialized(), "must be uninitialized now"); | |
| 3777 | |
| 3778 _insert_list = NULL; | |
| 3779 _insert_idx = -1; | |
| 3780 } | |
| 3781 | |
| 3782 void MoveResolver::insert_move(Interval* from_interval, Interval* to_interval) { | |
| 3783 assert(from_interval->reg_num() != to_interval->reg_num(), "from and to interval equal"); | |
| 3784 assert(from_interval->type() == to_interval->type(), "move between different types"); | |
| 3785 assert(_insert_list != NULL && _insert_idx != -1, "must setup insert position first"); | |
| 3786 assert(_insertion_buffer.lir_list() == _insert_list, "wrong insertion buffer"); | |
| 3787 | |
| 3788 LIR_Opr from_opr = LIR_OprFact::virtual_register(from_interval->reg_num(), from_interval->type()); | |
| 3789 LIR_Opr to_opr = LIR_OprFact::virtual_register(to_interval->reg_num(), to_interval->type()); | |
| 3790 | |
| 3791 if (!_multiple_reads_allowed) { | |
| 3792 // the last_use flag is an optimization for FPU stack allocation. When the same | |
| 3793 // input interval is used in more than one move, then it is too difficult to determine | |
| 3794 // if this move is really the last use. | |
| 3795 from_opr = from_opr->make_last_use(); | |
| 3796 } | |
| 3797 _insertion_buffer.move(_insert_idx, from_opr, to_opr); | |
| 3798 | |
| 3799 TRACE_LINEAR_SCAN(4, tty->print_cr("MoveResolver: inserted move from register %d (%d, %d) to %d (%d, %d)", from_interval->reg_num(), from_interval->assigned_reg(), from_interval->assigned_regHi(), to_interval->reg_num(), to_interval->assigned_reg(), to_interval->assigned_regHi())); | |
| 3800 } | |
| 3801 | |
| 3802 void MoveResolver::insert_move(LIR_Opr from_opr, Interval* to_interval) { | |
| 3803 assert(from_opr->type() == to_interval->type(), "move between different types"); | |
| 3804 assert(_insert_list != NULL && _insert_idx != -1, "must setup insert position first"); | |
| 3805 assert(_insertion_buffer.lir_list() == _insert_list, "wrong insertion buffer"); | |
| 3806 | |
| 3807 LIR_Opr to_opr = LIR_OprFact::virtual_register(to_interval->reg_num(), to_interval->type()); | |
| 3808 _insertion_buffer.move(_insert_idx, from_opr, to_opr); | |
| 3809 | |
| 3810 TRACE_LINEAR_SCAN(4, tty->print("MoveResolver: inserted move from constant "); from_opr->print(); tty->print_cr(" to %d (%d, %d)", to_interval->reg_num(), to_interval->assigned_reg(), to_interval->assigned_regHi())); | |
| 3811 } | |
| 3812 | |
| 3813 | |
| 3814 void MoveResolver::resolve_mappings() { | |
| 3815 TRACE_LINEAR_SCAN(4, tty->print_cr("MoveResolver: resolving mappings for Block B%d, index %d", _insert_list->block() != NULL ? _insert_list->block()->block_id() : -1, _insert_idx)); | |
| 3816 DEBUG_ONLY(verify_before_resolve()); | |
| 3817 | |
| 3818 // Block all registers that are used as input operands of a move. | |
| 3819 // When a register is blocked, no move to this register is emitted. | |
| 3820 // This is necessary for detecting cycles in moves. | |
| 3821 int i; | |
| 3822 for (i = _mapping_from.length() - 1; i >= 0; i--) { | |
| 3823 Interval* from_interval = _mapping_from.at(i); | |
| 3824 if (from_interval != NULL) { | |
| 3825 block_registers(from_interval); | |
| 3826 } | |
| 3827 } | |
| 3828 | |
| 3829 int spill_candidate = -1; | |
| 3830 while (_mapping_from.length() > 0) { | |
| 3831 bool processed_interval = false; | |
| 3832 | |
| 3833 for (i = _mapping_from.length() - 1; i >= 0; i--) { | |
| 3834 Interval* from_interval = _mapping_from.at(i); | |
| 3835 Interval* to_interval = _mapping_to.at(i); | |
| 3836 | |
| 3837 if (save_to_process_move(from_interval, to_interval)) { | |
| 3838 // this inverval can be processed because target is free | |
| 3839 if (from_interval != NULL) { | |
| 3840 insert_move(from_interval, to_interval); | |
| 3841 unblock_registers(from_interval); | |
| 3842 } else { | |
| 3843 insert_move(_mapping_from_opr.at(i), to_interval); | |
| 3844 } | |
| 3845 _mapping_from.remove_at(i); | |
| 3846 _mapping_from_opr.remove_at(i); | |
| 3847 _mapping_to.remove_at(i); | |
| 3848 | |
| 3849 processed_interval = true; | |
| 3850 } else if (from_interval != NULL && from_interval->assigned_reg() < LinearScan::nof_regs) { | |
| 3851 // this interval cannot be processed now because target is not free | |
| 3852 // it starts in a register, so it is a possible candidate for spilling | |
| 3853 spill_candidate = i; | |
| 3854 } | |
| 3855 } | |
| 3856 | |
| 3857 if (!processed_interval) { | |
| 3858 // no move could be processed because there is a cycle in the move list | |
| 3859 // (e.g. r1 -> r2, r2 -> r1), so one interval must be spilled to memory | |
| 3860 assert(spill_candidate != -1, "no interval in register for spilling found"); | |
| 3861 | |
| 3862 // create a new spill interval and assign a stack slot to it | |
| 3863 Interval* from_interval = _mapping_from.at(spill_candidate); | |
| 3864 Interval* spill_interval = new Interval(-1); | |
| 3865 spill_interval->set_type(from_interval->type()); | |
| 3866 | |
| 3867 // add a dummy range because real position is difficult to calculate | |
| 3868 // Note: this range is a special case when the integrity of the allocation is checked | |
| 3869 spill_interval->add_range(1, 2); | |
| 3870 | |
| 3871 // do not allocate a new spill slot for temporary interval, but | |
| 3872 // use spill slot assigned to from_interval. Otherwise moves from | |
| 3873 // one stack slot to another can happen (not allowed by LIR_Assembler | |
| 3874 int spill_slot = from_interval->canonical_spill_slot(); | |
| 3875 if (spill_slot < 0) { | |
| 3876 spill_slot = allocator()->allocate_spill_slot(type2spill_size[spill_interval->type()] == 2); | |
| 3877 from_interval->set_canonical_spill_slot(spill_slot); | |
| 3878 } | |
| 3879 spill_interval->assign_reg(spill_slot); | |
| 3880 allocator()->append_interval(spill_interval); | |
| 3881 | |
| 3882 TRACE_LINEAR_SCAN(4, tty->print_cr("created new Interval %d for spilling", spill_interval->reg_num())); | |
| 3883 | |
| 3884 // insert a move from register to stack and update the mapping | |
| 3885 insert_move(from_interval, spill_interval); | |
| 3886 _mapping_from.at_put(spill_candidate, spill_interval); | |
| 3887 unblock_registers(from_interval); | |
| 3888 } | |
| 3889 } | |
| 3890 | |
| 3891 // reset to default value | |
| 3892 _multiple_reads_allowed = false; | |
| 3893 | |
| 3894 // check that all intervals have been processed | |
| 3895 DEBUG_ONLY(check_empty()); | |
| 3896 } | |
| 3897 | |
| 3898 | |
| 3899 void MoveResolver::set_insert_position(LIR_List* insert_list, int insert_idx) { | |
| 3900 TRACE_LINEAR_SCAN(4, tty->print_cr("MoveResolver: setting insert position to Block B%d, index %d", insert_list->block() != NULL ? insert_list->block()->block_id() : -1, insert_idx)); | |
| 3901 assert(_insert_list == NULL && _insert_idx == -1, "use move_insert_position instead of set_insert_position when data already set"); | |
| 3902 | |
| 3903 create_insertion_buffer(insert_list); | |
| 3904 _insert_list = insert_list; | |
| 3905 _insert_idx = insert_idx; | |
| 3906 } | |
| 3907 | |
| 3908 void MoveResolver::move_insert_position(LIR_List* insert_list, int insert_idx) { | |
| 3909 TRACE_LINEAR_SCAN(4, tty->print_cr("MoveResolver: moving insert position to Block B%d, index %d", insert_list->block() != NULL ? insert_list->block()->block_id() : -1, insert_idx)); | |
| 3910 | |
| 3911 if (_insert_list != NULL && (insert_list != _insert_list || insert_idx != _insert_idx)) { | |
| 3912 // insert position changed -> resolve current mappings | |
| 3913 resolve_mappings(); | |
| 3914 } | |
| 3915 | |
| 3916 if (insert_list != _insert_list) { | |
| 3917 // block changed -> append insertion_buffer because it is | |
| 3918 // bound to a specific block and create a new insertion_buffer | |
| 3919 append_insertion_buffer(); | |
| 3920 create_insertion_buffer(insert_list); | |
| 3921 } | |
| 3922 | |
| 3923 _insert_list = insert_list; | |
| 3924 _insert_idx = insert_idx; | |
| 3925 } | |
| 3926 | |
| 3927 void MoveResolver::add_mapping(Interval* from_interval, Interval* to_interval) { | |
| 3928 TRACE_LINEAR_SCAN(4, tty->print_cr("MoveResolver: adding mapping from %d (%d, %d) to %d (%d, %d)", from_interval->reg_num(), from_interval->assigned_reg(), from_interval->assigned_regHi(), to_interval->reg_num(), to_interval->assigned_reg(), to_interval->assigned_regHi())); | |
| 3929 | |
| 3930 _mapping_from.append(from_interval); | |
| 3931 _mapping_from_opr.append(LIR_OprFact::illegalOpr); | |
| 3932 _mapping_to.append(to_interval); | |
| 3933 } | |
| 3934 | |
| 3935 | |
| 3936 void MoveResolver::add_mapping(LIR_Opr from_opr, Interval* to_interval) { | |
| 3937 TRACE_LINEAR_SCAN(4, tty->print("MoveResolver: adding mapping from "); from_opr->print(); tty->print_cr(" to %d (%d, %d)", to_interval->reg_num(), to_interval->assigned_reg(), to_interval->assigned_regHi())); | |
| 3938 assert(from_opr->is_constant(), "only for constants"); | |
| 3939 | |
| 3940 _mapping_from.append(NULL); | |
| 3941 _mapping_from_opr.append(from_opr); | |
| 3942 _mapping_to.append(to_interval); | |
| 3943 } | |
| 3944 | |
| 3945 void MoveResolver::resolve_and_append_moves() { | |
| 3946 if (has_mappings()) { | |
| 3947 resolve_mappings(); | |
| 3948 } | |
| 3949 append_insertion_buffer(); | |
| 3950 } | |
| 3951 | |
| 3952 | |
| 3953 | |
| 3954 // **** Implementation of Range ************************************* | |
| 3955 | |
| 3956 Range::Range(int from, int to, Range* next) : | |
| 3957 _from(from), | |
| 3958 _to(to), | |
| 3959 _next(next) | |
| 3960 { | |
| 3961 } | |
| 3962 | |
| 3963 // initialize sentinel | |
| 3964 Range* Range::_end = NULL; | |
| 1584 | 3965 void Range::initialize(Arena* arena) { |
| 3966 _end = new (arena) Range(max_jint, max_jint, NULL); | |
| 0 | 3967 } |
| 3968 | |
| 3969 int Range::intersects_at(Range* r2) const { | |
| 3970 const Range* r1 = this; | |
| 3971 | |
| 3972 assert(r1 != NULL && r2 != NULL, "null ranges not allowed"); | |
| 3973 assert(r1 != _end && r2 != _end, "empty ranges not allowed"); | |
| 3974 | |
| 3975 do { | |
| 3976 if (r1->from() < r2->from()) { | |
| 3977 if (r1->to() <= r2->from()) { | |
| 3978 r1 = r1->next(); if (r1 == _end) return -1; | |
| 3979 } else { | |
| 3980 return r2->from(); | |
| 3981 } | |
| 3982 } else if (r2->from() < r1->from()) { | |
| 3983 if (r2->to() <= r1->from()) { | |
| 3984 r2 = r2->next(); if (r2 == _end) return -1; | |
| 3985 } else { | |
| 3986 return r1->from(); | |
| 3987 } | |
| 3988 } else { // r1->from() == r2->from() | |
| 3989 if (r1->from() == r1->to()) { | |
| 3990 r1 = r1->next(); if (r1 == _end) return -1; | |
| 3991 } else if (r2->from() == r2->to()) { | |
| 3992 r2 = r2->next(); if (r2 == _end) return -1; | |
| 3993 } else { | |
| 3994 return r1->from(); | |
| 3995 } | |
| 3996 } | |
| 3997 } while (true); | |
| 3998 } | |
| 3999 | |
| 4000 #ifndef PRODUCT | |
| 4001 void Range::print(outputStream* out) const { | |
| 4002 out->print("[%d, %d[ ", _from, _to); | |
| 4003 } | |
| 4004 #endif | |
| 4005 | |
| 4006 | |
| 4007 | |
| 4008 // **** Implementation of Interval ********************************** | |
| 4009 | |
| 4010 // initialize sentinel | |
| 4011 Interval* Interval::_end = NULL; | |
| 1584 | 4012 void Interval::initialize(Arena* arena) { |
| 4013 Range::initialize(arena); | |
| 4014 _end = new (arena) Interval(-1); | |
| 0 | 4015 } |
| 4016 | |
| 4017 Interval::Interval(int reg_num) : | |
| 4018 _reg_num(reg_num), | |
| 4019 _type(T_ILLEGAL), | |
| 4020 _first(Range::end()), | |
| 4021 _use_pos_and_kinds(12), | |
| 4022 _current(Range::end()), | |
| 4023 _next(_end), | |
| 4024 _state(invalidState), | |
| 4025 _assigned_reg(LinearScan::any_reg), | |
| 4026 _assigned_regHi(LinearScan::any_reg), | |
| 4027 _cached_to(-1), | |
| 4028 _cached_opr(LIR_OprFact::illegalOpr), | |
| 4029 _cached_vm_reg(VMRegImpl::Bad()), | |
| 4030 _split_children(0), | |
| 4031 _canonical_spill_slot(-1), | |
| 4032 _insert_move_when_activated(false), | |
| 4033 _register_hint(NULL), | |
| 4034 _spill_state(noDefinitionFound), | |
| 4035 _spill_definition_pos(-1) | |
| 4036 { | |
| 4037 _split_parent = this; | |
| 4038 _current_split_child = this; | |
| 4039 } | |
| 4040 | |
| 4041 int Interval::calc_to() { | |
| 4042 assert(_first != Range::end(), "interval has no range"); | |
| 4043 | |
| 4044 Range* r = _first; | |
| 4045 while (r->next() != Range::end()) { | |
| 4046 r = r->next(); | |
| 4047 } | |
| 4048 return r->to(); | |
| 4049 } | |
| 4050 | |
| 4051 | |
| 4052 #ifdef ASSERT | |
| 4053 // consistency check of split-children | |
| 4054 void Interval::check_split_children() { | |
| 4055 if (_split_children.length() > 0) { | |
| 4056 assert(is_split_parent(), "only split parents can have children"); | |
| 4057 | |
| 4058 for (int i = 0; i < _split_children.length(); i++) { | |
| 4059 Interval* i1 = _split_children.at(i); | |
| 4060 | |
| 4061 assert(i1->split_parent() == this, "not a split child of this interval"); | |
| 4062 assert(i1->type() == type(), "must be equal for all split children"); | |
| 4063 assert(i1->canonical_spill_slot() == canonical_spill_slot(), "must be equal for all split children"); | |
| 4064 | |
| 4065 for (int j = i + 1; j < _split_children.length(); j++) { | |
| 4066 Interval* i2 = _split_children.at(j); | |
| 4067 | |
| 4068 assert(i1->reg_num() != i2->reg_num(), "same register number"); | |
| 4069 | |
| 4070 if (i1->from() < i2->from()) { | |
| 4071 assert(i1->to() <= i2->from() && i1->to() < i2->to(), "intervals overlapping"); | |
| 4072 } else { | |
| 4073 assert(i2->from() < i1->from(), "intervals start at same op_id"); | |
| 4074 assert(i2->to() <= i1->from() && i2->to() < i1->to(), "intervals overlapping"); | |
| 4075 } | |
| 4076 } | |
| 4077 } | |
| 4078 } | |
| 4079 } | |
| 4080 #endif // ASSERT | |
| 4081 | |
| 4082 Interval* Interval::register_hint(bool search_split_child) const { | |
| 4083 if (!search_split_child) { | |
| 4084 return _register_hint; | |
| 4085 } | |
| 4086 | |
| 4087 if (_register_hint != NULL) { | |
| 4088 assert(_register_hint->is_split_parent(), "ony split parents are valid hint registers"); | |
| 4089 | |
| 4090 if (_register_hint->assigned_reg() >= 0 && _register_hint->assigned_reg() < LinearScan::nof_regs) { | |
| 4091 return _register_hint; | |
| 4092 | |
| 4093 } else if (_register_hint->_split_children.length() > 0) { | |
| 4094 // search the first split child that has a register assigned | |
| 4095 int len = _register_hint->_split_children.length(); | |
| 4096 for (int i = 0; i < len; i++) { | |
| 4097 Interval* cur = _register_hint->_split_children.at(i); | |
| 4098 | |
| 4099 if (cur->assigned_reg() >= 0 && cur->assigned_reg() < LinearScan::nof_regs) { | |
| 4100 return cur; | |
| 4101 } | |
| 4102 } | |
| 4103 } | |
| 4104 } | |
| 4105 | |
| 4106 // no hint interval found that has a register assigned | |
| 4107 return NULL; | |
| 4108 } | |
| 4109 | |
| 4110 | |
| 4111 Interval* Interval::split_child_at_op_id(int op_id, LIR_OpVisitState::OprMode mode) { | |
| 4112 assert(is_split_parent(), "can only be called for split parents"); | |
| 4113 assert(op_id >= 0, "invalid op_id (method can not be called for spill moves)"); | |
| 4114 | |
| 4115 Interval* result; | |
| 4116 if (_split_children.length() == 0) { | |
| 4117 result = this; | |
| 4118 } else { | |
| 4119 result = NULL; | |
| 4120 int len = _split_children.length(); | |
| 4121 | |
| 4122 // in outputMode, the end of the interval (op_id == cur->to()) is not valid | |
| 4123 int to_offset = (mode == LIR_OpVisitState::outputMode ? 0 : 1); | |
| 4124 | |
| 4125 int i; | |
| 4126 for (i = 0; i < len; i++) { | |
| 4127 Interval* cur = _split_children.at(i); | |
| 4128 if (cur->from() <= op_id && op_id < cur->to() + to_offset) { | |
| 4129 if (i > 0) { | |
| 4130 // exchange current split child to start of list (faster access for next call) | |
| 4131 _split_children.at_put(i, _split_children.at(0)); | |
| 4132 _split_children.at_put(0, cur); | |
| 4133 } | |
| 4134 | |
| 4135 // interval found | |
| 4136 result = cur; | |
| 4137 break; | |
| 4138 } | |
| 4139 } | |
| 4140 | |
| 4141 #ifdef ASSERT | |
| 4142 for (i = 0; i < len; i++) { | |
| 4143 Interval* tmp = _split_children.at(i); | |
| 4144 if (tmp != result && tmp->from() <= op_id && op_id < tmp->to() + to_offset) { | |
| 4145 tty->print_cr("two valid result intervals found for op_id %d: %d and %d", op_id, result->reg_num(), tmp->reg_num()); | |
| 4146 result->print(); | |
| 4147 tmp->print(); | |
| 4148 assert(false, "two valid result intervals found"); | |
| 4149 } | |
| 4150 } | |
| 4151 #endif | |
| 4152 } | |
| 4153 | |
| 4154 assert(result != NULL, "no matching interval found"); | |
| 4155 assert(result->covers(op_id, mode), "op_id not covered by interval"); | |
| 4156 | |
| 4157 return result; | |
| 4158 } | |
| 4159 | |
| 4160 | |
| 4161 // returns the last split child that ends before the given op_id | |
| 4162 Interval* Interval::split_child_before_op_id(int op_id) { | |
| 4163 assert(op_id >= 0, "invalid op_id"); | |
| 4164 | |
| 4165 Interval* parent = split_parent(); | |
| 4166 Interval* result = NULL; | |
| 4167 | |
| 4168 int len = parent->_split_children.length(); | |
| 4169 assert(len > 0, "no split children available"); | |
| 4170 | |
| 4171 for (int i = len - 1; i >= 0; i--) { | |
| 4172 Interval* cur = parent->_split_children.at(i); | |
| 4173 if (cur->to() <= op_id && (result == NULL || result->to() < cur->to())) { | |
| 4174 result = cur; | |
| 4175 } | |
| 4176 } | |
| 4177 | |
| 4178 assert(result != NULL, "no split child found"); | |
| 4179 return result; | |
| 4180 } | |
| 4181 | |
| 4182 | |
| 4183 // checks if op_id is covered by any split child | |
| 4184 bool Interval::split_child_covers(int op_id, LIR_OpVisitState::OprMode mode) { | |
| 4185 assert(is_split_parent(), "can only be called for split parents"); | |
| 4186 assert(op_id >= 0, "invalid op_id (method can not be called for spill moves)"); | |
| 4187 | |
| 4188 if (_split_children.length() == 0) { | |
| 4189 // simple case if interval was not split | |
| 4190 return covers(op_id, mode); | |
| 4191 | |
| 4192 } else { | |
| 4193 // extended case: check all split children | |
| 4194 int len = _split_children.length(); | |
| 4195 for (int i = 0; i < len; i++) { | |
| 4196 Interval* cur = _split_children.at(i); | |
| 4197 if (cur->covers(op_id, mode)) { | |
| 4198 return true; | |
| 4199 } | |
| 4200 } | |
| 4201 return false; | |
| 4202 } | |
| 4203 } | |
| 4204 | |
| 4205 | |
| 4206 // Note: use positions are sorted descending -> first use has highest index | |
| 4207 int Interval::first_usage(IntervalUseKind min_use_kind) const { | |
| 4208 assert(LinearScan::is_virtual_interval(this), "cannot access use positions for fixed intervals"); | |
| 4209 | |
| 4210 for (int i = _use_pos_and_kinds.length() - 2; i >= 0; i -= 2) { | |
| 4211 if (_use_pos_and_kinds.at(i + 1) >= min_use_kind) { | |
| 4212 return _use_pos_and_kinds.at(i); | |
| 4213 } | |
| 4214 } | |
| 4215 return max_jint; | |
| 4216 } | |
| 4217 | |
| 4218 int Interval::next_usage(IntervalUseKind min_use_kind, int from) const { | |
| 4219 assert(LinearScan::is_virtual_interval(this), "cannot access use positions for fixed intervals"); | |
| 4220 | |
| 4221 for (int i = _use_pos_and_kinds.length() - 2; i >= 0; i -= 2) { | |
| 4222 if (_use_pos_and_kinds.at(i) >= from && _use_pos_and_kinds.at(i + 1) >= min_use_kind) { | |
| 4223 return _use_pos_and_kinds.at(i); | |
| 4224 } | |
| 4225 } | |
| 4226 return max_jint; | |
| 4227 } | |
| 4228 | |
| 4229 int Interval::next_usage_exact(IntervalUseKind exact_use_kind, int from) const { | |
| 4230 assert(LinearScan::is_virtual_interval(this), "cannot access use positions for fixed intervals"); | |
| 4231 | |
| 4232 for (int i = _use_pos_and_kinds.length() - 2; i >= 0; i -= 2) { | |
| 4233 if (_use_pos_and_kinds.at(i) >= from && _use_pos_and_kinds.at(i + 1) == exact_use_kind) { | |
| 4234 return _use_pos_and_kinds.at(i); | |
| 4235 } | |
| 4236 } | |
| 4237 return max_jint; | |
| 4238 } | |
| 4239 | |
| 4240 int Interval::previous_usage(IntervalUseKind min_use_kind, int from) const { | |
| 4241 assert(LinearScan::is_virtual_interval(this), "cannot access use positions for fixed intervals"); | |
| 4242 | |
| 4243 int prev = 0; | |
| 4244 for (int i = _use_pos_and_kinds.length() - 2; i >= 0; i -= 2) { | |
| 4245 if (_use_pos_and_kinds.at(i) > from) { | |
| 4246 return prev; | |
| 4247 } | |
| 4248 if (_use_pos_and_kinds.at(i + 1) >= min_use_kind) { | |
| 4249 prev = _use_pos_and_kinds.at(i); | |
| 4250 } | |
| 4251 } | |
| 4252 return prev; | |
| 4253 } | |
| 4254 | |
| 4255 void Interval::add_use_pos(int pos, IntervalUseKind use_kind) { | |
| 4256 assert(covers(pos, LIR_OpVisitState::inputMode), "use position not covered by live range"); | |
| 4257 | |
| 4258 // do not add use positions for precolored intervals because | |
| 4259 // they are never used | |
| 4260 if (use_kind != noUse && reg_num() >= LIR_OprDesc::vreg_base) { | |
| 4261 #ifdef ASSERT | |
| 4262 assert(_use_pos_and_kinds.length() % 2 == 0, "must be"); | |
| 4263 for (int i = 0; i < _use_pos_and_kinds.length(); i += 2) { | |
| 4264 assert(pos <= _use_pos_and_kinds.at(i), "already added a use-position with lower position"); | |
| 4265 assert(_use_pos_and_kinds.at(i + 1) >= firstValidKind && _use_pos_and_kinds.at(i + 1) <= lastValidKind, "invalid use kind"); | |
| 4266 if (i > 0) { | |
| 4267 assert(_use_pos_and_kinds.at(i) < _use_pos_and_kinds.at(i - 2), "not sorted descending"); | |
| 4268 } | |
| 4269 } | |
| 4270 #endif | |
| 4271 | |
| 4272 // Note: add_use is called in descending order, so list gets sorted | |
| 4273 // automatically by just appending new use positions | |
| 4274 int len = _use_pos_and_kinds.length(); | |
| 4275 if (len == 0 || _use_pos_and_kinds.at(len - 2) > pos) { | |
| 4276 _use_pos_and_kinds.append(pos); | |
| 4277 _use_pos_and_kinds.append(use_kind); | |
| 4278 } else if (_use_pos_and_kinds.at(len - 1) < use_kind) { | |
| 4279 assert(_use_pos_and_kinds.at(len - 2) == pos, "list not sorted correctly"); | |
| 4280 _use_pos_and_kinds.at_put(len - 1, use_kind); | |
| 4281 } | |
| 4282 } | |
| 4283 } | |
| 4284 | |
| 4285 void Interval::add_range(int from, int to) { | |
| 4286 assert(from < to, "invalid range"); | |
| 4287 assert(first() == Range::end() || to < first()->next()->from(), "not inserting at begin of interval"); | |
| 4288 assert(from <= first()->to(), "not inserting at begin of interval"); | |
| 4289 | |
| 4290 if (first()->from() <= to) { | |
| 4291 // join intersecting ranges | |
| 4292 first()->set_from(MIN2(from, first()->from())); | |
| 4293 first()->set_to (MAX2(to, first()->to())); | |
| 4294 } else { | |
| 4295 // insert new range | |
| 4296 _first = new Range(from, to, first()); | |
| 4297 } | |
| 4298 } | |
| 4299 | |
| 4300 Interval* Interval::new_split_child() { | |
| 4301 // allocate new interval | |
| 4302 Interval* result = new Interval(-1); | |
| 4303 result->set_type(type()); | |
| 4304 | |
| 4305 Interval* parent = split_parent(); | |
| 4306 result->_split_parent = parent; | |
| 4307 result->set_register_hint(parent); | |
| 4308 | |
| 4309 // insert new interval in children-list of parent | |
| 4310 if (parent->_split_children.length() == 0) { | |
| 4311 assert(is_split_parent(), "list must be initialized at first split"); | |
| 4312 | |
| 4313 parent->_split_children = IntervalList(4); | |
| 4314 parent->_split_children.append(this); | |
| 4315 } | |
| 4316 parent->_split_children.append(result); | |
| 4317 | |
| 4318 return result; | |
| 4319 } | |
| 4320 | |
| 4321 // split this interval at the specified position and return | |
| 4322 // the remainder as a new interval. | |
| 4323 // | |
| 4324 // when an interval is split, a bi-directional link is established between the original interval | |
| 4325 // (the split parent) and the intervals that are split off this interval (the split children) | |
| 4326 // When a split child is split again, the new created interval is also a direct child | |
| 4327 // of the original parent (there is no tree of split children stored, but a flat list) | |
| 4328 // All split children are spilled to the same stack slot (stored in _canonical_spill_slot) | |
| 4329 // | |
| 4330 // Note: The new interval has no valid reg_num | |
| 4331 Interval* Interval::split(int split_pos) { | |
| 4332 assert(LinearScan::is_virtual_interval(this), "cannot split fixed intervals"); | |
| 4333 | |
| 4334 // allocate new interval | |
| 4335 Interval* result = new_split_child(); | |
| 4336 | |
| 4337 // split the ranges | |
| 4338 Range* prev = NULL; | |
| 4339 Range* cur = _first; | |
| 4340 while (cur != Range::end() && cur->to() <= split_pos) { | |
| 4341 prev = cur; | |
| 4342 cur = cur->next(); | |
| 4343 } | |
| 4344 assert(cur != Range::end(), "split interval after end of last range"); | |
| 4345 | |
| 4346 if (cur->from() < split_pos) { | |
| 4347 result->_first = new Range(split_pos, cur->to(), cur->next()); | |
| 4348 cur->set_to(split_pos); | |
| 4349 cur->set_next(Range::end()); | |
| 4350 | |
| 4351 } else { | |
| 4352 assert(prev != NULL, "split before start of first range"); | |
| 4353 result->_first = cur; | |
| 4354 prev->set_next(Range::end()); | |
| 4355 } | |
| 4356 result->_current = result->_first; | |
| 4357 _cached_to = -1; // clear cached value | |
| 4358 | |
| 4359 // split list of use positions | |
| 4360 int total_len = _use_pos_and_kinds.length(); | |
| 4361 int start_idx = total_len - 2; | |
| 4362 while (start_idx >= 0 && _use_pos_and_kinds.at(start_idx) < split_pos) { | |
| 4363 start_idx -= 2; | |
| 4364 } | |
| 4365 | |
| 4366 intStack new_use_pos_and_kinds(total_len - start_idx); | |
| 4367 int i; | |
| 4368 for (i = start_idx + 2; i < total_len; i++) { | |
| 4369 new_use_pos_and_kinds.append(_use_pos_and_kinds.at(i)); | |
| 4370 } | |
| 4371 | |
| 4372 _use_pos_and_kinds.truncate(start_idx + 2); | |
| 4373 result->_use_pos_and_kinds = _use_pos_and_kinds; | |
| 4374 _use_pos_and_kinds = new_use_pos_and_kinds; | |
| 4375 | |
| 4376 #ifdef ASSERT | |
| 4377 assert(_use_pos_and_kinds.length() % 2 == 0, "must have use kind for each use pos"); | |
| 4378 assert(result->_use_pos_and_kinds.length() % 2 == 0, "must have use kind for each use pos"); | |
| 4379 assert(_use_pos_and_kinds.length() + result->_use_pos_and_kinds.length() == total_len, "missed some entries"); | |
| 4380 | |
| 4381 for (i = 0; i < _use_pos_and_kinds.length(); i += 2) { | |
| 4382 assert(_use_pos_and_kinds.at(i) < split_pos, "must be"); | |
| 4383 assert(_use_pos_and_kinds.at(i + 1) >= firstValidKind && _use_pos_and_kinds.at(i + 1) <= lastValidKind, "invalid use kind"); | |
| 4384 } | |
| 4385 for (i = 0; i < result->_use_pos_and_kinds.length(); i += 2) { | |
| 4386 assert(result->_use_pos_and_kinds.at(i) >= split_pos, "must be"); | |
| 4387 assert(result->_use_pos_and_kinds.at(i + 1) >= firstValidKind && result->_use_pos_and_kinds.at(i + 1) <= lastValidKind, "invalid use kind"); | |
| 4388 } | |
| 4389 #endif | |
| 4390 | |
| 4391 return result; | |
| 4392 } | |
| 4393 | |
| 4394 // split this interval at the specified position and return | |
| 4395 // the head as a new interval (the original interval is the tail) | |
| 4396 // | |
| 4397 // Currently, only the first range can be split, and the new interval | |
| 4398 // must not have split positions | |
| 4399 Interval* Interval::split_from_start(int split_pos) { | |
| 4400 assert(LinearScan::is_virtual_interval(this), "cannot split fixed intervals"); | |
| 4401 assert(split_pos > from() && split_pos < to(), "can only split inside interval"); | |
| 4402 assert(split_pos > _first->from() && split_pos <= _first->to(), "can only split inside first range"); | |
| 4403 assert(first_usage(noUse) > split_pos, "can not split when use positions are present"); | |
| 4404 | |
| 4405 // allocate new interval | |
| 4406 Interval* result = new_split_child(); | |
| 4407 | |
| 4408 // the new created interval has only one range (checked by assertion above), | |
| 4409 // so the splitting of the ranges is very simple | |
| 4410 result->add_range(_first->from(), split_pos); | |
| 4411 | |
| 4412 if (split_pos == _first->to()) { | |
| 4413 assert(_first->next() != Range::end(), "must not be at end"); | |
| 4414 _first = _first->next(); | |
| 4415 } else { | |
| 4416 _first->set_from(split_pos); | |
| 4417 } | |
| 4418 | |
| 4419 return result; | |
| 4420 } | |
| 4421 | |
| 4422 | |
| 4423 // returns true if the op_id is inside the interval | |
| 4424 bool Interval::covers(int op_id, LIR_OpVisitState::OprMode mode) const { | |
| 4425 Range* cur = _first; | |
| 4426 | |
| 4427 while (cur != Range::end() && cur->to() < op_id) { | |
| 4428 cur = cur->next(); | |
| 4429 } | |
| 4430 if (cur != Range::end()) { | |
| 4431 assert(cur->to() != cur->next()->from(), "ranges not separated"); | |
| 4432 | |
| 4433 if (mode == LIR_OpVisitState::outputMode) { | |
| 4434 return cur->from() <= op_id && op_id < cur->to(); | |
| 4435 } else { | |
| 4436 return cur->from() <= op_id && op_id <= cur->to(); | |
| 4437 } | |
| 4438 } | |
| 4439 return false; | |
| 4440 } | |
| 4441 | |
| 4442 // returns true if the interval has any hole between hole_from and hole_to | |
| 4443 // (even if the hole has only the length 1) | |
| 4444 bool Interval::has_hole_between(int hole_from, int hole_to) { | |
| 4445 assert(hole_from < hole_to, "check"); | |
| 4446 assert(from() <= hole_from && hole_to <= to(), "index out of interval"); | |
| 4447 | |
| 4448 Range* cur = _first; | |
| 4449 while (cur != Range::end()) { | |
| 4450 assert(cur->to() < cur->next()->from(), "no space between ranges"); | |
| 4451 | |
| 4452 // hole-range starts before this range -> hole | |
| 4453 if (hole_from < cur->from()) { | |
| 4454 return true; | |
| 4455 | |
| 4456 // hole-range completely inside this range -> no hole | |
| 4457 } else if (hole_to <= cur->to()) { | |
| 4458 return false; | |
| 4459 | |
| 4460 // overlapping of hole-range with this range -> hole | |
| 4461 } else if (hole_from <= cur->to()) { | |
| 4462 return true; | |
| 4463 } | |
| 4464 | |
| 4465 cur = cur->next(); | |
| 4466 } | |
| 4467 | |
| 4468 return false; | |
| 4469 } | |
| 4470 | |
| 4471 | |
| 4472 #ifndef PRODUCT | |
| 4473 void Interval::print(outputStream* out) const { | |
| 4474 const char* SpillState2Name[] = { "no definition", "no spill store", "one spill store", "store at definition", "start in memory", "no optimization" }; | |
| 4475 const char* UseKind2Name[] = { "N", "L", "S", "M" }; | |
| 4476 | |
| 4477 const char* type_name; | |
| 4478 LIR_Opr opr = LIR_OprFact::illegal(); | |
| 4479 if (reg_num() < LIR_OprDesc::vreg_base) { | |
| 4480 type_name = "fixed"; | |
| 4481 // need a temporary operand for fixed intervals because type() cannot be called | |
| 4482 if (assigned_reg() >= pd_first_cpu_reg && assigned_reg() <= pd_last_cpu_reg) { | |
| 4483 opr = LIR_OprFact::single_cpu(assigned_reg()); | |
| 4484 } else if (assigned_reg() >= pd_first_fpu_reg && assigned_reg() <= pd_last_fpu_reg) { | |
| 4485 opr = LIR_OprFact::single_fpu(assigned_reg() - pd_first_fpu_reg); | |
| 304 | 4486 #ifdef X86 |
| 0 | 4487 } else if (assigned_reg() >= pd_first_xmm_reg && assigned_reg() <= pd_last_xmm_reg) { |
| 4488 opr = LIR_OprFact::single_xmm(assigned_reg() - pd_first_xmm_reg); | |
| 4489 #endif | |
| 4490 } else { | |
| 4491 ShouldNotReachHere(); | |
| 4492 } | |
| 4493 } else { | |
| 4494 type_name = type2name(type()); | |
| 4495 if (assigned_reg() != -1) { | |
| 4496 opr = LinearScan::calc_operand_for_interval(this); | |
| 4497 } | |
| 4498 } | |
| 4499 | |
| 4500 out->print("%d %s ", reg_num(), type_name); | |
| 4501 if (opr->is_valid()) { | |
| 4502 out->print("\""); | |
| 4503 opr->print(out); | |
| 4504 out->print("\" "); | |
| 4505 } | |
| 4506 out->print("%d %d ", split_parent()->reg_num(), (register_hint(false) != NULL ? register_hint(false)->reg_num() : -1)); | |
| 4507 | |
| 4508 // print ranges | |
| 4509 Range* cur = _first; | |
| 4510 while (cur != Range::end()) { | |
| 4511 cur->print(out); | |
| 4512 cur = cur->next(); | |
| 4513 assert(cur != NULL, "range list not closed with range sentinel"); | |
| 4514 } | |
| 4515 | |
| 4516 // print use positions | |
| 4517 int prev = 0; | |
| 4518 assert(_use_pos_and_kinds.length() % 2 == 0, "must be"); | |
| 4519 for (int i =_use_pos_and_kinds.length() - 2; i >= 0; i -= 2) { | |
| 4520 assert(_use_pos_and_kinds.at(i + 1) >= firstValidKind && _use_pos_and_kinds.at(i + 1) <= lastValidKind, "invalid use kind"); | |
| 4521 assert(prev < _use_pos_and_kinds.at(i), "use positions not sorted"); | |
| 4522 | |
| 4523 out->print("%d %s ", _use_pos_and_kinds.at(i), UseKind2Name[_use_pos_and_kinds.at(i + 1)]); | |
| 4524 prev = _use_pos_and_kinds.at(i); | |
| 4525 } | |
| 4526 | |
| 4527 out->print(" \"%s\"", SpillState2Name[spill_state()]); | |
| 4528 out->cr(); | |
| 4529 } | |
| 4530 #endif | |
| 4531 | |
| 4532 | |
| 4533 | |
| 4534 // **** Implementation of IntervalWalker **************************** | |
| 4535 | |
| 4536 IntervalWalker::IntervalWalker(LinearScan* allocator, Interval* unhandled_fixed_first, Interval* unhandled_any_first) | |
| 4537 : _compilation(allocator->compilation()) | |
| 4538 , _allocator(allocator) | |
| 4539 { | |
| 4540 _unhandled_first[fixedKind] = unhandled_fixed_first; | |
| 4541 _unhandled_first[anyKind] = unhandled_any_first; | |
| 4542 _active_first[fixedKind] = Interval::end(); | |
| 4543 _inactive_first[fixedKind] = Interval::end(); | |
| 4544 _active_first[anyKind] = Interval::end(); | |
| 4545 _inactive_first[anyKind] = Interval::end(); | |
| 4546 _current_position = -1; | |
| 4547 _current = NULL; | |
| 4548 next_interval(); | |
| 4549 } | |
| 4550 | |
| 4551 | |
| 4552 // append interval at top of list | |
| 4553 void IntervalWalker::append_unsorted(Interval** list, Interval* interval) { | |
| 4554 interval->set_next(*list); *list = interval; | |
| 4555 } | |
| 4556 | |
| 4557 | |
| 4558 // append interval in order of current range from() | |
| 4559 void IntervalWalker::append_sorted(Interval** list, Interval* interval) { | |
| 4560 Interval* prev = NULL; | |
| 4561 Interval* cur = *list; | |
| 4562 while (cur->current_from() < interval->current_from()) { | |
| 4563 prev = cur; cur = cur->next(); | |
| 4564 } | |
| 4565 if (prev == NULL) { | |
| 4566 *list = interval; | |
| 4567 } else { | |
| 4568 prev->set_next(interval); | |
| 4569 } | |
| 4570 interval->set_next(cur); | |
| 4571 } | |
| 4572 | |
| 4573 void IntervalWalker::append_to_unhandled(Interval** list, Interval* interval) { | |
| 4574 assert(interval->from() >= current()->current_from(), "cannot append new interval before current walk position"); | |
| 4575 | |
| 4576 Interval* prev = NULL; | |
| 4577 Interval* cur = *list; | |
| 4578 while (cur->from() < interval->from() || (cur->from() == interval->from() && cur->first_usage(noUse) < interval->first_usage(noUse))) { | |
| 4579 prev = cur; cur = cur->next(); | |
| 4580 } | |
| 4581 if (prev == NULL) { | |
| 4582 *list = interval; | |
| 4583 } else { | |
| 4584 prev->set_next(interval); | |
| 4585 } | |
| 4586 interval->set_next(cur); | |
| 4587 } | |
| 4588 | |
| 4589 | |
| 4590 inline bool IntervalWalker::remove_from_list(Interval** list, Interval* i) { | |
| 4591 while (*list != Interval::end() && *list != i) { | |
| 4592 list = (*list)->next_addr(); | |
| 4593 } | |
| 4594 if (*list != Interval::end()) { | |
| 4595 assert(*list == i, "check"); | |
| 4596 *list = (*list)->next(); | |
| 4597 return true; | |
| 4598 } else { | |
| 4599 return false; | |
| 4600 } | |
| 4601 } | |
| 4602 | |
| 4603 void IntervalWalker::remove_from_list(Interval* i) { | |
| 4604 bool deleted; | |
| 4605 | |
| 4606 if (i->state() == activeState) { | |
| 4607 deleted = remove_from_list(active_first_addr(anyKind), i); | |
| 4608 } else { | |
| 4609 assert(i->state() == inactiveState, "invalid state"); | |
| 4610 deleted = remove_from_list(inactive_first_addr(anyKind), i); | |
| 4611 } | |
| 4612 | |
| 4613 assert(deleted, "interval has not been found in list"); | |
| 4614 } | |
| 4615 | |
| 4616 | |
| 4617 void IntervalWalker::walk_to(IntervalState state, int from) { | |
| 4618 assert (state == activeState || state == inactiveState, "wrong state"); | |
| 4619 for_each_interval_kind(kind) { | |
| 4620 Interval** prev = state == activeState ? active_first_addr(kind) : inactive_first_addr(kind); | |
| 4621 Interval* next = *prev; | |
| 4622 while (next->current_from() <= from) { | |
| 4623 Interval* cur = next; | |
| 4624 next = cur->next(); | |
| 4625 | |
| 4626 bool range_has_changed = false; | |
| 4627 while (cur->current_to() <= from) { | |
| 4628 cur->next_range(); | |
| 4629 range_has_changed = true; | |
| 4630 } | |
| 4631 | |
| 4632 // also handle move from inactive list to active list | |
| 4633 range_has_changed = range_has_changed || (state == inactiveState && cur->current_from() <= from); | |
| 4634 | |
| 4635 if (range_has_changed) { | |
| 4636 // remove cur from list | |
| 4637 *prev = next; | |
| 4638 if (cur->current_at_end()) { | |
| 4639 // move to handled state (not maintained as a list) | |
| 4640 cur->set_state(handledState); | |
| 4641 interval_moved(cur, kind, state, handledState); | |
| 4642 } else if (cur->current_from() <= from){ | |
| 4643 // sort into active list | |
| 4644 append_sorted(active_first_addr(kind), cur); | |
| 4645 cur->set_state(activeState); | |
| 4646 if (*prev == cur) { | |
| 4647 assert(state == activeState, "check"); | |
| 4648 prev = cur->next_addr(); | |
| 4649 } | |
| 4650 interval_moved(cur, kind, state, activeState); | |
| 4651 } else { | |
| 4652 // sort into inactive list | |
| 4653 append_sorted(inactive_first_addr(kind), cur); | |
| 4654 cur->set_state(inactiveState); | |
| 4655 if (*prev == cur) { | |
| 4656 assert(state == inactiveState, "check"); | |
| 4657 prev = cur->next_addr(); | |
| 4658 } | |
| 4659 interval_moved(cur, kind, state, inactiveState); | |
| 4660 } | |
| 4661 } else { | |
| 4662 prev = cur->next_addr(); | |
| 4663 continue; | |
| 4664 } | |
| 4665 } | |
| 4666 } | |
| 4667 } | |
| 4668 | |
| 4669 | |
| 4670 void IntervalWalker::next_interval() { | |
| 4671 IntervalKind kind; | |
| 4672 Interval* any = _unhandled_first[anyKind]; | |
| 4673 Interval* fixed = _unhandled_first[fixedKind]; | |
| 4674 | |
| 4675 if (any != Interval::end()) { | |
| 4676 // intervals may start at same position -> prefer fixed interval | |
| 4677 kind = fixed != Interval::end() && fixed->from() <= any->from() ? fixedKind : anyKind; | |
| 4678 | |
| 4679 assert (kind == fixedKind && fixed->from() <= any->from() || | |
| 4680 kind == anyKind && any->from() <= fixed->from(), "wrong interval!!!"); | |
| 4681 assert(any == Interval::end() || fixed == Interval::end() || any->from() != fixed->from() || kind == fixedKind, "if fixed and any-Interval start at same position, fixed must be processed first"); | |
| 4682 | |
| 4683 } else if (fixed != Interval::end()) { | |
| 4684 kind = fixedKind; | |
| 4685 } else { | |
| 4686 _current = NULL; return; | |
| 4687 } | |
| 4688 _current_kind = kind; | |
| 4689 _current = _unhandled_first[kind]; | |
| 4690 _unhandled_first[kind] = _current->next(); | |
| 4691 _current->set_next(Interval::end()); | |
| 4692 _current->rewind_range(); | |
| 4693 } | |
| 4694 | |
| 4695 | |
| 4696 void IntervalWalker::walk_to(int lir_op_id) { | |
| 4697 assert(_current_position <= lir_op_id, "can not walk backwards"); | |
| 4698 while (current() != NULL) { | |
| 4699 bool is_active = current()->from() <= lir_op_id; | |
| 4700 int id = is_active ? current()->from() : lir_op_id; | |
| 4701 | |
| 4702 TRACE_LINEAR_SCAN(2, if (_current_position < id) { tty->cr(); tty->print_cr("walk_to(%d) **************************************************************", id); }) | |
| 4703 | |
| 4704 // set _current_position prior to call of walk_to | |
| 4705 _current_position = id; | |
| 4706 | |
| 4707 // call walk_to even if _current_position == id | |
| 4708 walk_to(activeState, id); | |
| 4709 walk_to(inactiveState, id); | |
| 4710 | |
| 4711 if (is_active) { | |
| 4712 current()->set_state(activeState); | |
| 4713 if (activate_current()) { | |
| 4714 append_sorted(active_first_addr(current_kind()), current()); | |
| 4715 interval_moved(current(), current_kind(), unhandledState, activeState); | |
| 4716 } | |
| 4717 | |
| 4718 next_interval(); | |
| 4719 } else { | |
| 4720 return; | |
| 4721 } | |
| 4722 } | |
| 4723 } | |
| 4724 | |
| 4725 void IntervalWalker::interval_moved(Interval* interval, IntervalKind kind, IntervalState from, IntervalState to) { | |
| 4726 #ifndef PRODUCT | |
| 4727 if (TraceLinearScanLevel >= 4) { | |
| 4728 #define print_state(state) \ | |
| 4729 switch(state) {\ | |
| 4730 case unhandledState: tty->print("unhandled"); break;\ | |
| 4731 case activeState: tty->print("active"); break;\ | |
| 4732 case inactiveState: tty->print("inactive"); break;\ | |
| 4733 case handledState: tty->print("handled"); break;\ | |
| 4734 default: ShouldNotReachHere(); \ | |
| 4735 } | |
| 4736 | |
| 4737 print_state(from); tty->print(" to "); print_state(to); | |
| 4738 tty->fill_to(23); | |
| 4739 interval->print(); | |
| 4740 | |
| 4741 #undef print_state | |
| 4742 } | |
| 4743 #endif | |
| 4744 } | |
| 4745 | |
| 4746 | |
| 4747 | |
| 4748 // **** Implementation of LinearScanWalker ************************** | |
| 4749 | |
| 4750 LinearScanWalker::LinearScanWalker(LinearScan* allocator, Interval* unhandled_fixed_first, Interval* unhandled_any_first) | |
| 4751 : IntervalWalker(allocator, unhandled_fixed_first, unhandled_any_first) | |
| 4752 , _move_resolver(allocator) | |
| 4753 { | |
| 4754 for (int i = 0; i < LinearScan::nof_regs; i++) { | |
| 4755 _spill_intervals[i] = new IntervalList(2); | |
| 4756 } | |
| 4757 } | |
| 4758 | |
| 4759 | |
| 4760 inline void LinearScanWalker::init_use_lists(bool only_process_use_pos) { | |
| 4761 for (int i = _first_reg; i <= _last_reg; i++) { | |
| 4762 _use_pos[i] = max_jint; | |
| 4763 | |
| 4764 if (!only_process_use_pos) { | |
| 4765 _block_pos[i] = max_jint; | |
| 4766 _spill_intervals[i]->clear(); | |
| 4767 } | |
| 4768 } | |
| 4769 } | |
| 4770 | |
| 4771 inline void LinearScanWalker::exclude_from_use(int reg) { | |
| 4772 assert(reg < LinearScan::nof_regs, "interval must have a register assigned (stack slots not allowed)"); | |
| 4773 if (reg >= _first_reg && reg <= _last_reg) { | |
| 4774 _use_pos[reg] = 0; | |
| 4775 } | |
| 4776 } | |
| 4777 inline void LinearScanWalker::exclude_from_use(Interval* i) { | |
| 4778 assert(i->assigned_reg() != any_reg, "interval has no register assigned"); | |
| 4779 | |
| 4780 exclude_from_use(i->assigned_reg()); | |
| 4781 exclude_from_use(i->assigned_regHi()); | |
| 4782 } | |
| 4783 | |
| 4784 inline void LinearScanWalker::set_use_pos(int reg, Interval* i, int use_pos, bool only_process_use_pos) { | |
| 4785 assert(use_pos != 0, "must use exclude_from_use to set use_pos to 0"); | |
| 4786 | |
| 4787 if (reg >= _first_reg && reg <= _last_reg) { | |
| 4788 if (_use_pos[reg] > use_pos) { | |
| 4789 _use_pos[reg] = use_pos; | |
| 4790 } | |
| 4791 if (!only_process_use_pos) { | |
| 4792 _spill_intervals[reg]->append(i); | |
| 4793 } | |
| 4794 } | |
| 4795 } | |
| 4796 inline void LinearScanWalker::set_use_pos(Interval* i, int use_pos, bool only_process_use_pos) { | |
| 4797 assert(i->assigned_reg() != any_reg, "interval has no register assigned"); | |
| 4798 if (use_pos != -1) { | |
| 4799 set_use_pos(i->assigned_reg(), i, use_pos, only_process_use_pos); | |
| 4800 set_use_pos(i->assigned_regHi(), i, use_pos, only_process_use_pos); | |
| 4801 } | |
| 4802 } | |
| 4803 | |
| 4804 inline void LinearScanWalker::set_block_pos(int reg, Interval* i, int block_pos) { | |
| 4805 if (reg >= _first_reg && reg <= _last_reg) { | |
| 4806 if (_block_pos[reg] > block_pos) { | |
| 4807 _block_pos[reg] = block_pos; | |
| 4808 } | |
| 4809 if (_use_pos[reg] > block_pos) { | |
| 4810 _use_pos[reg] = block_pos; | |
| 4811 } | |
| 4812 } | |
| 4813 } | |
| 4814 inline void LinearScanWalker::set_block_pos(Interval* i, int block_pos) { | |
| 4815 assert(i->assigned_reg() != any_reg, "interval has no register assigned"); | |
| 4816 if (block_pos != -1) { | |
| 4817 set_block_pos(i->assigned_reg(), i, block_pos); | |
| 4818 set_block_pos(i->assigned_regHi(), i, block_pos); | |
| 4819 } | |
| 4820 } | |
| 4821 | |
| 4822 | |
| 4823 void LinearScanWalker::free_exclude_active_fixed() { | |
| 4824 Interval* list = active_first(fixedKind); | |
| 4825 while (list != Interval::end()) { | |
| 4826 assert(list->assigned_reg() < LinearScan::nof_regs, "active interval must have a register assigned"); | |
| 4827 exclude_from_use(list); | |
| 4828 list = list->next(); | |
| 4829 } | |
| 4830 } | |
| 4831 | |
| 4832 void LinearScanWalker::free_exclude_active_any() { | |
| 4833 Interval* list = active_first(anyKind); | |
| 4834 while (list != Interval::end()) { | |
| 4835 exclude_from_use(list); | |
| 4836 list = list->next(); | |
| 4837 } | |
| 4838 } | |
| 4839 | |
| 4840 void LinearScanWalker::free_collect_inactive_fixed(Interval* cur) { | |
| 4841 Interval* list = inactive_first(fixedKind); | |
| 4842 while (list != Interval::end()) { | |
| 4843 if (cur->to() <= list->current_from()) { | |
| 4844 assert(list->current_intersects_at(cur) == -1, "must not intersect"); | |
| 4845 set_use_pos(list, list->current_from(), true); | |
| 4846 } else { | |
| 4847 set_use_pos(list, list->current_intersects_at(cur), true); | |
| 4848 } | |
| 4849 list = list->next(); | |
| 4850 } | |
| 4851 } | |
| 4852 | |
| 4853 void LinearScanWalker::free_collect_inactive_any(Interval* cur) { | |
| 4854 Interval* list = inactive_first(anyKind); | |
| 4855 while (list != Interval::end()) { | |
| 4856 set_use_pos(list, list->current_intersects_at(cur), true); | |
| 4857 list = list->next(); | |
| 4858 } | |
| 4859 } | |
| 4860 | |
| 4861 void LinearScanWalker::free_collect_unhandled(IntervalKind kind, Interval* cur) { | |
| 4862 Interval* list = unhandled_first(kind); | |
| 4863 while (list != Interval::end()) { | |
| 4864 set_use_pos(list, list->intersects_at(cur), true); | |
| 4865 if (kind == fixedKind && cur->to() <= list->from()) { | |
| 4866 set_use_pos(list, list->from(), true); | |
| 4867 } | |
| 4868 list = list->next(); | |
| 4869 } | |
| 4870 } | |
| 4871 | |
| 4872 void LinearScanWalker::spill_exclude_active_fixed() { | |
| 4873 Interval* list = active_first(fixedKind); | |
| 4874 while (list != Interval::end()) { | |
| 4875 exclude_from_use(list); | |
| 4876 list = list->next(); | |
| 4877 } | |
| 4878 } | |
| 4879 | |
| 4880 void LinearScanWalker::spill_block_unhandled_fixed(Interval* cur) { | |
| 4881 Interval* list = unhandled_first(fixedKind); | |
| 4882 while (list != Interval::end()) { | |
| 4883 set_block_pos(list, list->intersects_at(cur)); | |
| 4884 list = list->next(); | |
| 4885 } | |
| 4886 } | |
| 4887 | |
| 4888 void LinearScanWalker::spill_block_inactive_fixed(Interval* cur) { | |
| 4889 Interval* list = inactive_first(fixedKind); | |
| 4890 while (list != Interval::end()) { | |
| 4891 if (cur->to() > list->current_from()) { | |
| 4892 set_block_pos(list, list->current_intersects_at(cur)); | |
| 4893 } else { | |
| 4894 assert(list->current_intersects_at(cur) == -1, "invalid optimization: intervals intersect"); | |
| 4895 } | |
| 4896 | |
| 4897 list = list->next(); | |
| 4898 } | |
| 4899 } | |
| 4900 | |
| 4901 void LinearScanWalker::spill_collect_active_any() { | |
| 4902 Interval* list = active_first(anyKind); | |
| 4903 while (list != Interval::end()) { | |
| 4904 set_use_pos(list, MIN2(list->next_usage(loopEndMarker, _current_position), list->to()), false); | |
| 4905 list = list->next(); | |
| 4906 } | |
| 4907 } | |
| 4908 | |
| 4909 void LinearScanWalker::spill_collect_inactive_any(Interval* cur) { | |
| 4910 Interval* list = inactive_first(anyKind); | |
| 4911 while (list != Interval::end()) { | |
| 4912 if (list->current_intersects(cur)) { | |
| 4913 set_use_pos(list, MIN2(list->next_usage(loopEndMarker, _current_position), list->to()), false); | |
| 4914 } | |
| 4915 list = list->next(); | |
| 4916 } | |
| 4917 } | |
| 4918 | |
| 4919 | |
| 4920 void LinearScanWalker::insert_move(int op_id, Interval* src_it, Interval* dst_it) { | |
| 4921 // output all moves here. When source and target are equal, the move is | |
| 4922 // optimized away later in assign_reg_nums | |
| 4923 | |
| 4924 op_id = (op_id + 1) & ~1; | |
| 4925 BlockBegin* op_block = allocator()->block_of_op_with_id(op_id); | |
| 4926 assert(op_id > 0 && allocator()->block_of_op_with_id(op_id - 2) == op_block, "cannot insert move at block boundary"); | |
| 4927 | |
| 4928 // calculate index of instruction inside instruction list of current block | |
| 4929 // the minimal index (for a block with no spill moves) can be calculated because the | |
| 4930 // numbering of instructions is known. | |
| 4931 // When the block already contains spill moves, the index must be increased until the | |
| 4932 // correct index is reached. | |
| 4933 LIR_OpList* list = op_block->lir()->instructions_list(); | |
| 4934 int index = (op_id - list->at(0)->id()) / 2; | |
| 4935 assert(list->at(index)->id() <= op_id, "error in calculation"); | |
| 4936 | |
| 4937 while (list->at(index)->id() != op_id) { | |
| 4938 index++; | |
| 4939 assert(0 <= index && index < list->length(), "index out of bounds"); | |
| 4940 } | |
| 4941 assert(1 <= index && index < list->length(), "index out of bounds"); | |
| 4942 assert(list->at(index)->id() == op_id, "error in calculation"); | |
| 4943 | |
| 4944 // insert new instruction before instruction at position index | |
| 4945 _move_resolver.move_insert_position(op_block->lir(), index - 1); | |
| 4946 _move_resolver.add_mapping(src_it, dst_it); | |
| 4947 } | |
| 4948 | |
| 4949 | |
| 4950 int LinearScanWalker::find_optimal_split_pos(BlockBegin* min_block, BlockBegin* max_block, int max_split_pos) { | |
| 4951 int from_block_nr = min_block->linear_scan_number(); | |
| 4952 int to_block_nr = max_block->linear_scan_number(); | |
| 4953 | |
| 4954 assert(0 <= from_block_nr && from_block_nr < block_count(), "out of range"); | |
| 4955 assert(0 <= to_block_nr && to_block_nr < block_count(), "out of range"); | |
| 4956 assert(from_block_nr < to_block_nr, "must cross block boundary"); | |
| 4957 | |
| 4958 // Try to split at end of max_block. If this would be after | |
| 4959 // max_split_pos, then use the begin of max_block | |
| 4960 int optimal_split_pos = max_block->last_lir_instruction_id() + 2; | |
| 4961 if (optimal_split_pos > max_split_pos) { | |
| 4962 optimal_split_pos = max_block->first_lir_instruction_id(); | |
| 4963 } | |
| 4964 | |
| 4965 int min_loop_depth = max_block->loop_depth(); | |
| 4966 for (int i = to_block_nr - 1; i >= from_block_nr; i--) { | |
| 4967 BlockBegin* cur = block_at(i); | |
| 4968 | |
| 4969 if (cur->loop_depth() < min_loop_depth) { | |
| 4970 // block with lower loop-depth found -> split at the end of this block | |
| 4971 min_loop_depth = cur->loop_depth(); | |
| 4972 optimal_split_pos = cur->last_lir_instruction_id() + 2; | |
| 4973 } | |
| 4974 } | |
| 4975 assert(optimal_split_pos > allocator()->max_lir_op_id() || allocator()->is_block_begin(optimal_split_pos), "algorithm must move split pos to block boundary"); | |
| 4976 | |
| 4977 return optimal_split_pos; | |
| 4978 } | |
| 4979 | |
| 4980 | |
| 4981 int LinearScanWalker::find_optimal_split_pos(Interval* it, int min_split_pos, int max_split_pos, bool do_loop_optimization) { | |
| 4982 int optimal_split_pos = -1; | |
| 4983 if (min_split_pos == max_split_pos) { | |
| 4984 // trivial case, no optimization of split position possible | |
| 4985 TRACE_LINEAR_SCAN(4, tty->print_cr(" min-pos and max-pos are equal, no optimization possible")); | |
| 4986 optimal_split_pos = min_split_pos; | |
| 4987 | |
| 4988 } else { | |
| 4989 assert(min_split_pos < max_split_pos, "must be true then"); | |
| 4990 assert(min_split_pos > 0, "cannot access min_split_pos - 1 otherwise"); | |
| 4991 | |
| 4992 // reason for using min_split_pos - 1: when the minimal split pos is exactly at the | |
| 4993 // beginning of a block, then min_split_pos is also a possible split position. | |
| 4994 // Use the block before as min_block, because then min_block->last_lir_instruction_id() + 2 == min_split_pos | |
| 4995 BlockBegin* min_block = allocator()->block_of_op_with_id(min_split_pos - 1); | |
| 4996 | |
| 4997 // reason for using max_split_pos - 1: otherwise there would be an assertion failure | |
| 4998 // when an interval ends at the end of the last block of the method | |
| 4999 // (in this case, max_split_pos == allocator()->max_lir_op_id() + 2, and there is no | |
| 5000 // block at this op_id) | |
| 5001 BlockBegin* max_block = allocator()->block_of_op_with_id(max_split_pos - 1); | |
| 5002 | |
| 5003 assert(min_block->linear_scan_number() <= max_block->linear_scan_number(), "invalid order"); | |
| 5004 if (min_block == max_block) { | |
| 5005 // split position cannot be moved to block boundary, so split as late as possible | |
| 5006 TRACE_LINEAR_SCAN(4, tty->print_cr(" cannot move split pos to block boundary because min_pos and max_pos are in same block")); | |
| 5007 optimal_split_pos = max_split_pos; | |
| 5008 | |
| 5009 } else if (it->has_hole_between(max_split_pos - 1, max_split_pos) && !allocator()->is_block_begin(max_split_pos)) { | |
| 5010 // Do not move split position if the interval has a hole before max_split_pos. | |
| 5011 // Intervals resulting from Phi-Functions have more than one definition (marked | |
| 5012 // as mustHaveRegister) with a hole before each definition. When the register is needed | |
| 5013 // for the second definition, an earlier reloading is unnecessary. | |
| 5014 TRACE_LINEAR_SCAN(4, tty->print_cr(" interval has hole just before max_split_pos, so splitting at max_split_pos")); | |
| 5015 optimal_split_pos = max_split_pos; | |
| 5016 | |
| 5017 } else { | |
| 5018 // seach optimal block boundary between min_split_pos and max_split_pos | |
| 5019 TRACE_LINEAR_SCAN(4, tty->print_cr(" moving split pos to optimal block boundary between block B%d and B%d", min_block->block_id(), max_block->block_id())); | |
| 5020 | |
| 5021 if (do_loop_optimization) { | |
| 5022 // Loop optimization: if a loop-end marker is found between min- and max-position, | |
| 5023 // then split before this loop | |
| 5024 int loop_end_pos = it->next_usage_exact(loopEndMarker, min_block->last_lir_instruction_id() + 2); | |
| 5025 TRACE_LINEAR_SCAN(4, tty->print_cr(" loop optimization: loop end found at pos %d", loop_end_pos)); | |
| 5026 | |
| 5027 assert(loop_end_pos > min_split_pos, "invalid order"); | |
| 5028 if (loop_end_pos < max_split_pos) { | |
| 5029 // loop-end marker found between min- and max-position | |
| 5030 // if it is not the end marker for the same loop as the min-position, then move | |
| 5031 // the max-position to this loop block. | |
| 5032 // Desired result: uses tagged as shouldHaveRegister inside a loop cause a reloading | |
| 5033 // of the interval (normally, only mustHaveRegister causes a reloading) | |
| 5034 BlockBegin* loop_block = allocator()->block_of_op_with_id(loop_end_pos); | |
| 5035 | |
| 5036 TRACE_LINEAR_SCAN(4, tty->print_cr(" interval is used in loop that ends in block B%d, so trying to move max_block back from B%d to B%d", loop_block->block_id(), max_block->block_id(), loop_block->block_id())); | |
| 5037 assert(loop_block != min_block, "loop_block and min_block must be different because block boundary is needed between"); | |
| 5038 | |
| 5039 optimal_split_pos = find_optimal_split_pos(min_block, loop_block, loop_block->last_lir_instruction_id() + 2); | |
| 5040 if (optimal_split_pos == loop_block->last_lir_instruction_id() + 2) { | |
| 5041 optimal_split_pos = -1; | |
| 5042 TRACE_LINEAR_SCAN(4, tty->print_cr(" loop optimization not necessary")); | |
| 5043 } else { | |
| 5044 TRACE_LINEAR_SCAN(4, tty->print_cr(" loop optimization successful")); | |
| 5045 } | |
| 5046 } | |
| 5047 } | |
| 5048 | |
| 5049 if (optimal_split_pos == -1) { | |
| 5050 // not calculated by loop optimization | |
| 5051 optimal_split_pos = find_optimal_split_pos(min_block, max_block, max_split_pos); | |
| 5052 } | |
| 5053 } | |
| 5054 } | |
| 5055 TRACE_LINEAR_SCAN(4, tty->print_cr(" optimal split position: %d", optimal_split_pos)); | |
| 5056 | |
| 5057 return optimal_split_pos; | |
| 5058 } | |
| 5059 | |
| 5060 | |
| 5061 /* | |
| 5062 split an interval at the optimal position between min_split_pos and | |
| 5063 max_split_pos in two parts: | |
| 5064 1) the left part has already a location assigned | |
| 5065 2) the right part is sorted into to the unhandled-list | |
| 5066 */ | |
| 5067 void LinearScanWalker::split_before_usage(Interval* it, int min_split_pos, int max_split_pos) { | |
| 5068 TRACE_LINEAR_SCAN(2, tty->print ("----- splitting interval: "); it->print()); | |
| 5069 TRACE_LINEAR_SCAN(2, tty->print_cr(" between %d and %d", min_split_pos, max_split_pos)); | |
| 5070 | |
| 5071 assert(it->from() < min_split_pos, "cannot split at start of interval"); | |
| 5072 assert(current_position() < min_split_pos, "cannot split before current position"); | |
| 5073 assert(min_split_pos <= max_split_pos, "invalid order"); | |
| 5074 assert(max_split_pos <= it->to(), "cannot split after end of interval"); | |
| 5075 | |
| 5076 int optimal_split_pos = find_optimal_split_pos(it, min_split_pos, max_split_pos, true); | |
| 5077 | |
| 5078 assert(min_split_pos <= optimal_split_pos && optimal_split_pos <= max_split_pos, "out of range"); | |
| 5079 assert(optimal_split_pos <= it->to(), "cannot split after end of interval"); | |
| 5080 assert(optimal_split_pos > it->from(), "cannot split at start of interval"); | |
| 5081 | |
| 5082 if (optimal_split_pos == it->to() && it->next_usage(mustHaveRegister, min_split_pos) == max_jint) { | |
| 5083 // the split position would be just before the end of the interval | |
| 5084 // -> no split at all necessary | |
| 5085 TRACE_LINEAR_SCAN(4, tty->print_cr(" no split necessary because optimal split position is at end of interval")); | |
| 5086 return; | |
| 5087 } | |
| 5088 | |
| 5089 // must calculate this before the actual split is performed and before split position is moved to odd op_id | |
| 5090 bool move_necessary = !allocator()->is_block_begin(optimal_split_pos) && !it->has_hole_between(optimal_split_pos - 1, optimal_split_pos); | |
| 5091 | |
| 5092 if (!allocator()->is_block_begin(optimal_split_pos)) { | |
| 5093 // move position before actual instruction (odd op_id) | |
| 5094 optimal_split_pos = (optimal_split_pos - 1) | 1; | |
| 5095 } | |
| 5096 | |
| 5097 TRACE_LINEAR_SCAN(4, tty->print_cr(" splitting at position %d", optimal_split_pos)); | |
| 5098 assert(allocator()->is_block_begin(optimal_split_pos) || (optimal_split_pos % 2 == 1), "split pos must be odd when not on block boundary"); | |
| 5099 assert(!allocator()->is_block_begin(optimal_split_pos) || (optimal_split_pos % 2 == 0), "split pos must be even on block boundary"); | |
| 5100 | |
| 5101 Interval* split_part = it->split(optimal_split_pos); | |
| 5102 | |
| 5103 allocator()->append_interval(split_part); | |
| 5104 allocator()->copy_register_flags(it, split_part); | |
| 5105 split_part->set_insert_move_when_activated(move_necessary); | |
| 5106 append_to_unhandled(unhandled_first_addr(anyKind), split_part); | |
| 5107 | |
| 5108 TRACE_LINEAR_SCAN(2, tty->print_cr(" split interval in two parts (insert_move_when_activated: %d)", move_necessary)); | |
| 5109 TRACE_LINEAR_SCAN(2, tty->print (" "); it->print()); | |
| 5110 TRACE_LINEAR_SCAN(2, tty->print (" "); split_part->print()); | |
| 5111 } | |
| 5112 | |
| 5113 /* | |
| 5114 split an interval at the optimal position between min_split_pos and | |
| 5115 max_split_pos in two parts: | |
| 5116 1) the left part has already a location assigned | |
| 5117 2) the right part is always on the stack and therefore ignored in further processing | |
| 5118 */ | |
| 5119 void LinearScanWalker::split_for_spilling(Interval* it) { | |
| 5120 // calculate allowed range of splitting position | |
| 5121 int max_split_pos = current_position(); | |
| 5122 int min_split_pos = MAX2(it->previous_usage(shouldHaveRegister, max_split_pos) + 1, it->from()); | |
| 5123 | |
| 5124 TRACE_LINEAR_SCAN(2, tty->print ("----- splitting and spilling interval: "); it->print()); | |
| 5125 TRACE_LINEAR_SCAN(2, tty->print_cr(" between %d and %d", min_split_pos, max_split_pos)); | |
| 5126 | |
| 5127 assert(it->state() == activeState, "why spill interval that is not active?"); | |
| 5128 assert(it->from() <= min_split_pos, "cannot split before start of interval"); | |
| 5129 assert(min_split_pos <= max_split_pos, "invalid order"); | |
| 5130 assert(max_split_pos < it->to(), "cannot split at end end of interval"); | |
| 5131 assert(current_position() < it->to(), "interval must not end before current position"); | |
| 5132 | |
| 5133 if (min_split_pos == it->from()) { | |
| 5134 // the whole interval is never used, so spill it entirely to memory | |
| 5135 TRACE_LINEAR_SCAN(2, tty->print_cr(" spilling entire interval because split pos is at beginning of interval")); | |
| 5136 assert(it->first_usage(shouldHaveRegister) > current_position(), "interval must not have use position before current_position"); | |
| 5137 | |
| 5138 allocator()->assign_spill_slot(it); | |
| 5139 allocator()->change_spill_state(it, min_split_pos); | |
| 5140 | |
| 5141 // Also kick parent intervals out of register to memory when they have no use | |
| 5142 // position. This avoids short interval in register surrounded by intervals in | |
| 5143 // memory -> avoid useless moves from memory to register and back | |
| 5144 Interval* parent = it; | |
| 5145 while (parent != NULL && parent->is_split_child()) { | |
| 5146 parent = parent->split_child_before_op_id(parent->from()); | |
| 5147 | |
| 5148 if (parent->assigned_reg() < LinearScan::nof_regs) { | |
| 5149 if (parent->first_usage(shouldHaveRegister) == max_jint) { | |
| 5150 // parent is never used, so kick it out of its assigned register | |
| 5151 TRACE_LINEAR_SCAN(4, tty->print_cr(" kicking out interval %d out of its register because it is never used", parent->reg_num())); | |
| 5152 allocator()->assign_spill_slot(parent); | |
| 5153 } else { | |
| 5154 // do not go further back because the register is actually used by the interval | |
| 5155 parent = NULL; | |
| 5156 } | |
| 5157 } | |
| 5158 } | |
| 5159 | |
| 5160 } else { | |
| 5161 // search optimal split pos, split interval and spill only the right hand part | |
| 5162 int optimal_split_pos = find_optimal_split_pos(it, min_split_pos, max_split_pos, false); | |
| 5163 | |
| 5164 assert(min_split_pos <= optimal_split_pos && optimal_split_pos <= max_split_pos, "out of range"); | |
| 5165 assert(optimal_split_pos < it->to(), "cannot split at end of interval"); | |
| 5166 assert(optimal_split_pos >= it->from(), "cannot split before start of interval"); | |
| 5167 | |
| 5168 if (!allocator()->is_block_begin(optimal_split_pos)) { | |
| 5169 // move position before actual instruction (odd op_id) | |
| 5170 optimal_split_pos = (optimal_split_pos - 1) | 1; | |
| 5171 } | |
| 5172 | |
| 5173 TRACE_LINEAR_SCAN(4, tty->print_cr(" splitting at position %d", optimal_split_pos)); | |
| 5174 assert(allocator()->is_block_begin(optimal_split_pos) || (optimal_split_pos % 2 == 1), "split pos must be odd when not on block boundary"); | |
| 5175 assert(!allocator()->is_block_begin(optimal_split_pos) || (optimal_split_pos % 2 == 0), "split pos must be even on block boundary"); | |
| 5176 | |
| 5177 Interval* spilled_part = it->split(optimal_split_pos); | |
| 5178 allocator()->append_interval(spilled_part); | |
| 5179 allocator()->assign_spill_slot(spilled_part); | |
| 5180 allocator()->change_spill_state(spilled_part, optimal_split_pos); | |
| 5181 | |
| 5182 if (!allocator()->is_block_begin(optimal_split_pos)) { | |
| 5183 TRACE_LINEAR_SCAN(4, tty->print_cr(" inserting move from interval %d to %d", it->reg_num(), spilled_part->reg_num())); | |
| 5184 insert_move(optimal_split_pos, it, spilled_part); | |
| 5185 } | |
| 5186 | |
| 5187 // the current_split_child is needed later when moves are inserted for reloading | |
| 5188 assert(spilled_part->current_split_child() == it, "overwriting wrong current_split_child"); | |
| 5189 spilled_part->make_current_split_child(); | |
| 5190 | |
| 5191 TRACE_LINEAR_SCAN(2, tty->print_cr(" split interval in two parts")); | |
| 5192 TRACE_LINEAR_SCAN(2, tty->print (" "); it->print()); | |
| 5193 TRACE_LINEAR_SCAN(2, tty->print (" "); spilled_part->print()); | |
| 5194 } | |
| 5195 } | |
| 5196 | |
| 5197 | |
| 5198 void LinearScanWalker::split_stack_interval(Interval* it) { | |
| 5199 int min_split_pos = current_position() + 1; | |
| 5200 int max_split_pos = MIN2(it->first_usage(shouldHaveRegister), it->to()); | |
| 5201 | |
| 5202 split_before_usage(it, min_split_pos, max_split_pos); | |
| 5203 } | |
| 5204 | |
| 5205 void LinearScanWalker::split_when_partial_register_available(Interval* it, int register_available_until) { | |
| 5206 int min_split_pos = MAX2(it->previous_usage(shouldHaveRegister, register_available_until), it->from() + 1); | |
| 5207 int max_split_pos = register_available_until; | |
| 5208 | |
| 5209 split_before_usage(it, min_split_pos, max_split_pos); | |
| 5210 } | |
| 5211 | |
| 5212 void LinearScanWalker::split_and_spill_interval(Interval* it) { | |
| 5213 assert(it->state() == activeState || it->state() == inactiveState, "other states not allowed"); | |
| 5214 | |
| 5215 int current_pos = current_position(); | |
| 5216 if (it->state() == inactiveState) { | |
| 5217 // the interval is currently inactive, so no spill slot is needed for now. | |
| 5218 // when the split part is activated, the interval has a new chance to get a register, | |
| 5219 // so in the best case no stack slot is necessary | |
| 5220 assert(it->has_hole_between(current_pos - 1, current_pos + 1), "interval can not be inactive otherwise"); | |
| 5221 split_before_usage(it, current_pos + 1, current_pos + 1); | |
| 5222 | |
| 5223 } else { | |
| 5224 // search the position where the interval must have a register and split | |
| 5225 // at the optimal position before. | |
| 5226 // The new created part is added to the unhandled list and will get a register | |
| 5227 // when it is activated | |
| 5228 int min_split_pos = current_pos + 1; | |
| 5229 int max_split_pos = MIN2(it->next_usage(mustHaveRegister, min_split_pos), it->to()); | |
| 5230 | |
| 5231 split_before_usage(it, min_split_pos, max_split_pos); | |
| 5232 | |
| 5233 assert(it->next_usage(mustHaveRegister, current_pos) == max_jint, "the remaining part is spilled to stack and therefore has no register"); | |
| 5234 split_for_spilling(it); | |
| 5235 } | |
| 5236 } | |
| 5237 | |
| 5238 | |
| 5239 int LinearScanWalker::find_free_reg(int reg_needed_until, int interval_to, int hint_reg, int ignore_reg, bool* need_split) { | |
| 5240 int min_full_reg = any_reg; | |
| 5241 int max_partial_reg = any_reg; | |
| 5242 | |
| 5243 for (int i = _first_reg; i <= _last_reg; i++) { | |
| 5244 if (i == ignore_reg) { | |
| 5245 // this register must be ignored | |
| 5246 | |
| 5247 } else if (_use_pos[i] >= interval_to) { | |
| 5248 // this register is free for the full interval | |
| 5249 if (min_full_reg == any_reg || i == hint_reg || (_use_pos[i] < _use_pos[min_full_reg] && min_full_reg != hint_reg)) { | |
| 5250 min_full_reg = i; | |
| 5251 } | |
| 5252 } else if (_use_pos[i] > reg_needed_until) { | |
| 5253 // this register is at least free until reg_needed_until | |
| 5254 if (max_partial_reg == any_reg || i == hint_reg || (_use_pos[i] > _use_pos[max_partial_reg] && max_partial_reg != hint_reg)) { | |
| 5255 max_partial_reg = i; | |
| 5256 } | |
| 5257 } | |
| 5258 } | |
| 5259 | |
| 5260 if (min_full_reg != any_reg) { | |
| 5261 return min_full_reg; | |
| 5262 } else if (max_partial_reg != any_reg) { | |
| 5263 *need_split = true; | |
| 5264 return max_partial_reg; | |
| 5265 } else { | |
| 5266 return any_reg; | |
| 5267 } | |
| 5268 } | |
| 5269 | |
| 5270 int LinearScanWalker::find_free_double_reg(int reg_needed_until, int interval_to, int hint_reg, bool* need_split) { | |
| 5271 assert((_last_reg - _first_reg + 1) % 2 == 0, "adjust algorithm"); | |
| 5272 | |
| 5273 int min_full_reg = any_reg; | |
| 5274 int max_partial_reg = any_reg; | |
| 5275 | |
| 5276 for (int i = _first_reg; i < _last_reg; i+=2) { | |
| 5277 if (_use_pos[i] >= interval_to && _use_pos[i + 1] >= interval_to) { | |
| 5278 // this register is free for the full interval | |
| 5279 if (min_full_reg == any_reg || i == hint_reg || (_use_pos[i] < _use_pos[min_full_reg] && min_full_reg != hint_reg)) { | |
| 5280 min_full_reg = i; | |
| 5281 } | |
| 5282 } else if (_use_pos[i] > reg_needed_until && _use_pos[i + 1] > reg_needed_until) { | |
| 5283 // this register is at least free until reg_needed_until | |
| 5284 if (max_partial_reg == any_reg || i == hint_reg || (_use_pos[i] > _use_pos[max_partial_reg] && max_partial_reg != hint_reg)) { | |
| 5285 max_partial_reg = i; | |
| 5286 } | |
| 5287 } | |
| 5288 } | |
| 5289 | |
| 5290 if (min_full_reg != any_reg) { | |
| 5291 return min_full_reg; | |
| 5292 } else if (max_partial_reg != any_reg) { | |
| 5293 *need_split = true; | |
| 5294 return max_partial_reg; | |
| 5295 } else { | |
| 5296 return any_reg; | |
| 5297 } | |
| 5298 } | |
| 5299 | |
| 5300 | |
| 5301 bool LinearScanWalker::alloc_free_reg(Interval* cur) { | |
| 5302 TRACE_LINEAR_SCAN(2, tty->print("trying to find free register for "); cur->print()); | |
| 5303 | |
| 5304 init_use_lists(true); | |
| 5305 free_exclude_active_fixed(); | |
| 5306 free_exclude_active_any(); | |
| 5307 free_collect_inactive_fixed(cur); | |
| 5308 free_collect_inactive_any(cur); | |
| 5309 // free_collect_unhandled(fixedKind, cur); | |
| 5310 assert(unhandled_first(fixedKind) == Interval::end(), "must not have unhandled fixed intervals because all fixed intervals have a use at position 0"); | |
| 5311 | |
| 5312 // _use_pos contains the start of the next interval that has this register assigned | |
| 5313 // (either as a fixed register or a normal allocated register in the past) | |
| 5314 // only intervals overlapping with cur are processed, non-overlapping invervals can be ignored safely | |
| 5315 TRACE_LINEAR_SCAN(4, tty->print_cr(" state of registers:")); | |
| 5316 TRACE_LINEAR_SCAN(4, for (int i = _first_reg; i <= _last_reg; i++) tty->print_cr(" reg %d: use_pos: %d", i, _use_pos[i])); | |
| 5317 | |
| 5318 int hint_reg, hint_regHi; | |
| 5319 Interval* register_hint = cur->register_hint(); | |
| 5320 if (register_hint != NULL) { | |
| 5321 hint_reg = register_hint->assigned_reg(); | |
| 5322 hint_regHi = register_hint->assigned_regHi(); | |
| 5323 | |
| 5324 if (allocator()->is_precolored_cpu_interval(register_hint)) { | |
| 5325 assert(hint_reg != any_reg && hint_regHi == any_reg, "must be for fixed intervals"); | |
| 5326 hint_regHi = hint_reg + 1; // connect e.g. eax-edx | |
| 5327 } | |
| 5328 TRACE_LINEAR_SCAN(4, tty->print(" hint registers %d, %d from interval ", hint_reg, hint_regHi); register_hint->print()); | |
| 5329 | |
| 5330 } else { | |
| 5331 hint_reg = any_reg; | |
| 5332 hint_regHi = any_reg; | |
| 5333 } | |
| 5334 assert(hint_reg == any_reg || hint_reg != hint_regHi, "hint reg and regHi equal"); | |
| 5335 assert(cur->assigned_reg() == any_reg && cur->assigned_regHi() == any_reg, "register already assigned to interval"); | |
| 5336 | |
| 5337 // the register must be free at least until this position | |
| 5338 int reg_needed_until = cur->from() + 1; | |
| 5339 int interval_to = cur->to(); | |
| 5340 | |
| 5341 bool need_split = false; | |
| 5342 int split_pos = -1; | |
| 5343 int reg = any_reg; | |
| 5344 int regHi = any_reg; | |
| 5345 | |
| 5346 if (_adjacent_regs) { | |
| 5347 reg = find_free_double_reg(reg_needed_until, interval_to, hint_reg, &need_split); | |
| 5348 regHi = reg + 1; | |
| 5349 if (reg == any_reg) { | |
| 5350 return false; | |
| 5351 } | |
| 5352 split_pos = MIN2(_use_pos[reg], _use_pos[regHi]); | |
| 5353 | |
| 5354 } else { | |
| 5355 reg = find_free_reg(reg_needed_until, interval_to, hint_reg, any_reg, &need_split); | |
| 5356 if (reg == any_reg) { | |
| 5357 return false; | |
| 5358 } | |
| 5359 split_pos = _use_pos[reg]; | |
| 5360 | |
| 5361 if (_num_phys_regs == 2) { | |
| 5362 regHi = find_free_reg(reg_needed_until, interval_to, hint_regHi, reg, &need_split); | |
| 5363 | |
| 5364 if (_use_pos[reg] < interval_to && regHi == any_reg) { | |
| 5365 // do not split interval if only one register can be assigned until the split pos | |
| 5366 // (when one register is found for the whole interval, split&spill is only | |
| 5367 // performed for the hi register) | |
| 5368 return false; | |
| 5369 | |
| 5370 } else if (regHi != any_reg) { | |
| 5371 split_pos = MIN2(split_pos, _use_pos[regHi]); | |
| 5372 | |
| 5373 // sort register numbers to prevent e.g. a move from eax,ebx to ebx,eax | |
| 5374 if (reg > regHi) { | |
| 5375 int temp = reg; | |
| 5376 reg = regHi; | |
| 5377 regHi = temp; | |
| 5378 } | |
| 5379 } | |
| 5380 } | |
| 5381 } | |
| 5382 | |
| 5383 cur->assign_reg(reg, regHi); | |
| 5384 TRACE_LINEAR_SCAN(2, tty->print_cr("selected register %d, %d", reg, regHi)); | |
| 5385 | |
| 5386 assert(split_pos > 0, "invalid split_pos"); | |
| 5387 if (need_split) { | |
| 5388 // register not available for full interval, so split it | |
| 5389 split_when_partial_register_available(cur, split_pos); | |
| 5390 } | |
| 5391 | |
| 5392 // only return true if interval is completely assigned | |
| 5393 return _num_phys_regs == 1 || regHi != any_reg; | |
| 5394 } | |
| 5395 | |
| 5396 | |
| 5397 int LinearScanWalker::find_locked_reg(int reg_needed_until, int interval_to, int hint_reg, int ignore_reg, bool* need_split) { | |
| 5398 int max_reg = any_reg; | |
| 5399 | |
| 5400 for (int i = _first_reg; i <= _last_reg; i++) { | |
| 5401 if (i == ignore_reg) { | |
| 5402 // this register must be ignored | |
| 5403 | |
| 5404 } else if (_use_pos[i] > reg_needed_until) { | |
| 5405 if (max_reg == any_reg || i == hint_reg || (_use_pos[i] > _use_pos[max_reg] && max_reg != hint_reg)) { | |
| 5406 max_reg = i; | |
| 5407 } | |
| 5408 } | |
| 5409 } | |
| 5410 | |
| 5411 if (max_reg != any_reg && _block_pos[max_reg] <= interval_to) { | |
| 5412 *need_split = true; | |
| 5413 } | |
| 5414 | |
| 5415 return max_reg; | |
| 5416 } | |
| 5417 | |
| 5418 int LinearScanWalker::find_locked_double_reg(int reg_needed_until, int interval_to, int hint_reg, bool* need_split) { | |
| 5419 assert((_last_reg - _first_reg + 1) % 2 == 0, "adjust algorithm"); | |
| 5420 | |
| 5421 int max_reg = any_reg; | |
| 5422 | |
| 5423 for (int i = _first_reg; i < _last_reg; i+=2) { | |
| 5424 if (_use_pos[i] > reg_needed_until && _use_pos[i + 1] > reg_needed_until) { | |
| 5425 if (max_reg == any_reg || _use_pos[i] > _use_pos[max_reg]) { | |
| 5426 max_reg = i; | |
| 5427 } | |
| 5428 } | |
| 5429 } | |
| 5430 | |
| 5431 if (_block_pos[max_reg] <= interval_to || _block_pos[max_reg + 1] <= interval_to) { | |
| 5432 *need_split = true; | |
| 5433 } | |
| 5434 | |
| 5435 return max_reg; | |
| 5436 } | |
| 5437 | |
| 5438 void LinearScanWalker::split_and_spill_intersecting_intervals(int reg, int regHi) { | |
| 5439 assert(reg != any_reg, "no register assigned"); | |
| 5440 | |
| 5441 for (int i = 0; i < _spill_intervals[reg]->length(); i++) { | |
| 5442 Interval* it = _spill_intervals[reg]->at(i); | |
| 5443 remove_from_list(it); | |
| 5444 split_and_spill_interval(it); | |
| 5445 } | |
| 5446 | |
| 5447 if (regHi != any_reg) { | |
| 5448 IntervalList* processed = _spill_intervals[reg]; | |
| 5449 for (int i = 0; i < _spill_intervals[regHi]->length(); i++) { | |
| 5450 Interval* it = _spill_intervals[regHi]->at(i); | |
| 5451 if (processed->index_of(it) == -1) { | |
| 5452 remove_from_list(it); | |
| 5453 split_and_spill_interval(it); | |
| 5454 } | |
| 5455 } | |
| 5456 } | |
| 5457 } | |
| 5458 | |
| 5459 | |
| 5460 // Split an Interval and spill it to memory so that cur can be placed in a register | |
| 5461 void LinearScanWalker::alloc_locked_reg(Interval* cur) { | |
| 5462 TRACE_LINEAR_SCAN(2, tty->print("need to split and spill to get register for "); cur->print()); | |
| 5463 | |
| 5464 // collect current usage of registers | |
| 5465 init_use_lists(false); | |
| 5466 spill_exclude_active_fixed(); | |
| 5467 // spill_block_unhandled_fixed(cur); | |
| 5468 assert(unhandled_first(fixedKind) == Interval::end(), "must not have unhandled fixed intervals because all fixed intervals have a use at position 0"); | |
| 5469 spill_block_inactive_fixed(cur); | |
| 5470 spill_collect_active_any(); | |
| 5471 spill_collect_inactive_any(cur); | |
| 5472 | |
| 5473 #ifndef PRODUCT | |
| 5474 if (TraceLinearScanLevel >= 4) { | |
| 5475 tty->print_cr(" state of registers:"); | |
| 5476 for (int i = _first_reg; i <= _last_reg; i++) { | |
| 5477 tty->print(" reg %d: use_pos: %d, block_pos: %d, intervals: ", i, _use_pos[i], _block_pos[i]); | |
| 5478 for (int j = 0; j < _spill_intervals[i]->length(); j++) { | |
| 5479 tty->print("%d ", _spill_intervals[i]->at(j)->reg_num()); | |
| 5480 } | |
| 5481 tty->cr(); | |
| 5482 } | |
| 5483 } | |
| 5484 #endif | |
| 5485 | |
| 5486 // the register must be free at least until this position | |
| 5487 int reg_needed_until = MIN2(cur->first_usage(mustHaveRegister), cur->from() + 1); | |
| 5488 int interval_to = cur->to(); | |
| 5489 assert (reg_needed_until > 0 && reg_needed_until < max_jint, "interval has no use"); | |
| 5490 | |
| 5491 int split_pos = 0; | |
| 5492 int use_pos = 0; | |
| 5493 bool need_split = false; | |
| 5494 int reg, regHi; | |
| 5495 | |
| 5496 if (_adjacent_regs) { | |
| 5497 reg = find_locked_double_reg(reg_needed_until, interval_to, any_reg, &need_split); | |
| 5498 regHi = reg + 1; | |
| 5499 | |
| 5500 if (reg != any_reg) { | |
| 5501 use_pos = MIN2(_use_pos[reg], _use_pos[regHi]); | |
| 5502 split_pos = MIN2(_block_pos[reg], _block_pos[regHi]); | |
| 5503 } | |
| 5504 } else { | |
| 5505 reg = find_locked_reg(reg_needed_until, interval_to, any_reg, cur->assigned_reg(), &need_split); | |
| 5506 regHi = any_reg; | |
| 5507 | |
| 5508 if (reg != any_reg) { | |
| 5509 use_pos = _use_pos[reg]; | |
| 5510 split_pos = _block_pos[reg]; | |
| 5511 | |
| 5512 if (_num_phys_regs == 2) { | |
| 5513 if (cur->assigned_reg() != any_reg) { | |
| 5514 regHi = reg; | |
| 5515 reg = cur->assigned_reg(); | |
| 5516 } else { | |
| 5517 regHi = find_locked_reg(reg_needed_until, interval_to, any_reg, reg, &need_split); | |
| 5518 if (regHi != any_reg) { | |
| 5519 use_pos = MIN2(use_pos, _use_pos[regHi]); | |
| 5520 split_pos = MIN2(split_pos, _block_pos[regHi]); | |
| 5521 } | |
| 5522 } | |
| 5523 | |
| 5524 if (regHi != any_reg && reg > regHi) { | |
| 5525 // sort register numbers to prevent e.g. a move from eax,ebx to ebx,eax | |
| 5526 int temp = reg; | |
| 5527 reg = regHi; | |
| 5528 regHi = temp; | |
| 5529 } | |
| 5530 } | |
| 5531 } | |
| 5532 } | |
| 5533 | |
| 5534 if (reg == any_reg || (_num_phys_regs == 2 && regHi == any_reg) || use_pos <= cur->first_usage(mustHaveRegister)) { | |
| 5535 // the first use of cur is later than the spilling position -> spill cur | |
| 5536 TRACE_LINEAR_SCAN(4, tty->print_cr("able to spill current interval. first_usage(register): %d, use_pos: %d", cur->first_usage(mustHaveRegister), use_pos)); | |
| 5537 | |
| 5538 if (cur->first_usage(mustHaveRegister) <= cur->from() + 1) { | |
| 5539 assert(false, "cannot spill interval that is used in first instruction (possible reason: no register found)"); | |
| 5540 // assign a reasonable register and do a bailout in product mode to avoid errors | |
| 5541 allocator()->assign_spill_slot(cur); | |
| 5542 BAILOUT("LinearScan: no register found"); | |
| 5543 } | |
| 5544 | |
| 5545 split_and_spill_interval(cur); | |
| 5546 } else { | |
| 5547 TRACE_LINEAR_SCAN(4, tty->print_cr("decided to use register %d, %d", reg, regHi)); | |
| 5548 assert(reg != any_reg && (_num_phys_regs == 1 || regHi != any_reg), "no register found"); | |
| 5549 assert(split_pos > 0, "invalid split_pos"); | |
| 5550 assert(need_split == false || split_pos > cur->from(), "splitting interval at from"); | |
| 5551 | |
| 5552 cur->assign_reg(reg, regHi); | |
| 5553 if (need_split) { | |
| 5554 // register not available for full interval, so split it | |
| 5555 split_when_partial_register_available(cur, split_pos); | |
| 5556 } | |
| 5557 | |
| 5558 // perform splitting and spilling for all affected intervalls | |
| 5559 split_and_spill_intersecting_intervals(reg, regHi); | |
| 5560 } | |
| 5561 } | |
| 5562 | |
| 5563 bool LinearScanWalker::no_allocation_possible(Interval* cur) { | |
| 304 | 5564 #ifdef X86 |
| 0 | 5565 // fast calculation of intervals that can never get a register because the |
| 5566 // the next instruction is a call that blocks all registers | |
| 5567 // Note: this does not work if callee-saved registers are available (e.g. on Sparc) | |
| 5568 | |
| 5569 // check if this interval is the result of a split operation | |
| 5570 // (an interval got a register until this position) | |
| 5571 int pos = cur->from(); | |
| 5572 if ((pos & 1) == 1) { | |
| 5573 // the current instruction is a call that blocks all registers | |
| 5574 if (pos < allocator()->max_lir_op_id() && allocator()->has_call(pos + 1)) { | |
| 5575 TRACE_LINEAR_SCAN(4, tty->print_cr(" free register cannot be available because all registers blocked by following call")); | |
| 5576 | |
| 5577 // safety check that there is really no register available | |
| 5578 assert(alloc_free_reg(cur) == false, "found a register for this interval"); | |
| 5579 return true; | |
| 5580 } | |
| 5581 | |
| 5582 } | |
| 5583 #endif | |
| 5584 return false; | |
| 5585 } | |
| 5586 | |
| 5587 void LinearScanWalker::init_vars_for_alloc(Interval* cur) { | |
| 5588 BasicType type = cur->type(); | |
| 5589 _num_phys_regs = LinearScan::num_physical_regs(type); | |
| 5590 _adjacent_regs = LinearScan::requires_adjacent_regs(type); | |
| 5591 | |
| 5592 if (pd_init_regs_for_alloc(cur)) { | |
| 5593 // the appropriate register range was selected. | |
| 5594 } else if (type == T_FLOAT || type == T_DOUBLE) { | |
| 5595 _first_reg = pd_first_fpu_reg; | |
| 5596 _last_reg = pd_last_fpu_reg; | |
| 5597 } else { | |
| 5598 _first_reg = pd_first_cpu_reg; | |
| 5599 _last_reg = pd_last_cpu_reg; | |
| 5600 } | |
| 5601 | |
| 5602 assert(0 <= _first_reg && _first_reg < LinearScan::nof_regs, "out of range"); | |
| 5603 assert(0 <= _last_reg && _last_reg < LinearScan::nof_regs, "out of range"); | |
| 5604 } | |
| 5605 | |
| 5606 | |
| 5607 bool LinearScanWalker::is_move(LIR_Op* op, Interval* from, Interval* to) { | |
| 5608 if (op->code() != lir_move) { | |
| 5609 return false; | |
| 5610 } | |
| 5611 assert(op->as_Op1() != NULL, "move must be LIR_Op1"); | |
| 5612 | |
| 5613 LIR_Opr in = ((LIR_Op1*)op)->in_opr(); | |
| 5614 LIR_Opr res = ((LIR_Op1*)op)->result_opr(); | |
| 5615 return in->is_virtual() && res->is_virtual() && in->vreg_number() == from->reg_num() && res->vreg_number() == to->reg_num(); | |
| 5616 } | |
| 5617 | |
| 5618 // optimization (especially for phi functions of nested loops): | |
| 5619 // assign same spill slot to non-intersecting intervals | |
| 5620 void LinearScanWalker::combine_spilled_intervals(Interval* cur) { | |
| 5621 if (cur->is_split_child()) { | |
| 5622 // optimization is only suitable for split parents | |
| 5623 return; | |
| 5624 } | |
| 5625 | |
| 5626 Interval* register_hint = cur->register_hint(false); | |
| 5627 if (register_hint == NULL) { | |
| 5628 // cur is not the target of a move, otherwise register_hint would be set | |
| 5629 return; | |
| 5630 } | |
| 5631 assert(register_hint->is_split_parent(), "register hint must be split parent"); | |
| 5632 | |
| 5633 if (cur->spill_state() != noOptimization || register_hint->spill_state() != noOptimization) { | |
| 5634 // combining the stack slots for intervals where spill move optimization is applied | |
| 5635 // is not benefitial and would cause problems | |
| 5636 return; | |
| 5637 } | |
| 5638 | |
| 5639 int begin_pos = cur->from(); | |
| 5640 int end_pos = cur->to(); | |
| 5641 if (end_pos > allocator()->max_lir_op_id() || (begin_pos & 1) != 0 || (end_pos & 1) != 0) { | |
| 5642 // safety check that lir_op_with_id is allowed | |
| 5643 return; | |
| 5644 } | |
| 5645 | |
| 5646 if (!is_move(allocator()->lir_op_with_id(begin_pos), register_hint, cur) || !is_move(allocator()->lir_op_with_id(end_pos), cur, register_hint)) { | |
| 5647 // cur and register_hint are not connected with two moves | |
| 5648 return; | |
| 5649 } | |
| 5650 | |
| 5651 Interval* begin_hint = register_hint->split_child_at_op_id(begin_pos, LIR_OpVisitState::inputMode); | |
| 5652 Interval* end_hint = register_hint->split_child_at_op_id(end_pos, LIR_OpVisitState::outputMode); | |
| 5653 if (begin_hint == end_hint || begin_hint->to() != begin_pos || end_hint->from() != end_pos) { | |
| 5654 // register_hint must be split, otherwise the re-writing of use positions does not work | |
| 5655 return; | |
| 5656 } | |
| 5657 | |
| 5658 assert(begin_hint->assigned_reg() != any_reg, "must have register assigned"); | |
| 5659 assert(end_hint->assigned_reg() == any_reg, "must not have register assigned"); | |
| 5660 assert(cur->first_usage(mustHaveRegister) == begin_pos, "must have use position at begin of interval because of move"); | |
| 5661 assert(end_hint->first_usage(mustHaveRegister) == end_pos, "must have use position at begin of interval because of move"); | |
| 5662 | |
| 5663 if (begin_hint->assigned_reg() < LinearScan::nof_regs) { | |
| 5664 // register_hint is not spilled at begin_pos, so it would not be benefitial to immediately spill cur | |
| 5665 return; | |
| 5666 } | |
| 5667 assert(register_hint->canonical_spill_slot() != -1, "must be set when part of interval was spilled"); | |
| 5668 | |
| 5669 // modify intervals such that cur gets the same stack slot as register_hint | |
| 5670 // delete use positions to prevent the intervals to get a register at beginning | |
| 5671 cur->set_canonical_spill_slot(register_hint->canonical_spill_slot()); | |
| 5672 cur->remove_first_use_pos(); | |
| 5673 end_hint->remove_first_use_pos(); | |
| 5674 } | |
| 5675 | |
| 5676 | |
| 5677 // allocate a physical register or memory location to an interval | |
| 5678 bool LinearScanWalker::activate_current() { | |
| 5679 Interval* cur = current(); | |
| 5680 bool result = true; | |
| 5681 | |
| 5682 TRACE_LINEAR_SCAN(2, tty->print ("+++++ activating interval "); cur->print()); | |
| 5683 TRACE_LINEAR_SCAN(4, tty->print_cr(" split_parent: %d, insert_move_when_activated: %d", cur->split_parent()->reg_num(), cur->insert_move_when_activated())); | |
| 5684 | |
| 5685 if (cur->assigned_reg() >= LinearScan::nof_regs) { | |
| 5686 // activating an interval that has a stack slot assigned -> split it at first use position | |
| 5687 // used for method parameters | |
| 5688 TRACE_LINEAR_SCAN(4, tty->print_cr(" interval has spill slot assigned (method parameter) -> split it before first use")); | |
| 5689 | |
| 5690 split_stack_interval(cur); | |
| 5691 result = false; | |
| 5692 | |
| 5693 } else if (allocator()->gen()->is_vreg_flag_set(cur->reg_num(), LIRGenerator::must_start_in_memory)) { | |
| 5694 // activating an interval that must start in a stack slot, but may get a register later | |
| 5695 // used for lir_roundfp: rounding is done by store to stack and reload later | |
| 5696 TRACE_LINEAR_SCAN(4, tty->print_cr(" interval must start in stack slot -> split it before first use")); | |
| 5697 assert(cur->assigned_reg() == any_reg && cur->assigned_regHi() == any_reg, "register already assigned"); | |
| 5698 | |
| 5699 allocator()->assign_spill_slot(cur); | |
| 5700 split_stack_interval(cur); | |
| 5701 result = false; | |
| 5702 | |
| 5703 } else if (cur->assigned_reg() == any_reg) { | |
| 5704 // interval has not assigned register -> normal allocation | |
| 5705 // (this is the normal case for most intervals) | |
| 5706 TRACE_LINEAR_SCAN(4, tty->print_cr(" normal allocation of register")); | |
| 5707 | |
| 5708 // assign same spill slot to non-intersecting intervals | |
| 5709 combine_spilled_intervals(cur); | |
| 5710 | |
| 5711 init_vars_for_alloc(cur); | |
| 5712 if (no_allocation_possible(cur) || !alloc_free_reg(cur)) { | |
| 5713 // no empty register available. | |
| 5714 // split and spill another interval so that this interval gets a register | |
| 5715 alloc_locked_reg(cur); | |
| 5716 } | |
| 5717 | |
| 5718 // spilled intervals need not be move to active-list | |
| 5719 if (cur->assigned_reg() >= LinearScan::nof_regs) { | |
| 5720 result = false; | |
| 5721 } | |
| 5722 } | |
| 5723 | |
| 5724 // load spilled values that become active from stack slot to register | |
| 5725 if (cur->insert_move_when_activated()) { | |
| 5726 assert(cur->is_split_child(), "must be"); | |
| 5727 assert(cur->current_split_child() != NULL, "must be"); | |
| 5728 assert(cur->current_split_child()->reg_num() != cur->reg_num(), "cannot insert move between same interval"); | |
| 5729 TRACE_LINEAR_SCAN(4, tty->print_cr("Inserting move from interval %d to %d because insert_move_when_activated is set", cur->current_split_child()->reg_num(), cur->reg_num())); | |
| 5730 | |
| 5731 insert_move(cur->from(), cur->current_split_child(), cur); | |
| 5732 } | |
| 5733 cur->make_current_split_child(); | |
| 5734 | |
| 5735 return result; // true = interval is moved to active list | |
| 5736 } | |
| 5737 | |
| 5738 | |
| 5739 // Implementation of EdgeMoveOptimizer | |
| 5740 | |
| 5741 EdgeMoveOptimizer::EdgeMoveOptimizer() : | |
| 5742 _edge_instructions(4), | |
| 5743 _edge_instructions_idx(4) | |
| 5744 { | |
| 5745 } | |
| 5746 | |
| 5747 void EdgeMoveOptimizer::optimize(BlockList* code) { | |
| 5748 EdgeMoveOptimizer optimizer = EdgeMoveOptimizer(); | |
| 5749 | |
| 5750 // ignore the first block in the list (index 0 is not processed) | |
| 5751 for (int i = code->length() - 1; i >= 1; i--) { | |
| 5752 BlockBegin* block = code->at(i); | |
| 5753 | |
| 5754 if (block->number_of_preds() > 1 && !block->is_set(BlockBegin::exception_entry_flag)) { | |
| 5755 optimizer.optimize_moves_at_block_end(block); | |
| 5756 } | |
| 5757 if (block->number_of_sux() == 2) { | |
| 5758 optimizer.optimize_moves_at_block_begin(block); | |
| 5759 } | |
| 5760 } | |
| 5761 } | |
| 5762 | |
| 5763 | |
| 5764 // clear all internal data structures | |
| 5765 void EdgeMoveOptimizer::init_instructions() { | |
| 5766 _edge_instructions.clear(); | |
| 5767 _edge_instructions_idx.clear(); | |
| 5768 } | |
| 5769 | |
| 5770 // append a lir-instruction-list and the index of the current operation in to the list | |
| 5771 void EdgeMoveOptimizer::append_instructions(LIR_OpList* instructions, int instructions_idx) { | |
| 5772 _edge_instructions.append(instructions); | |
| 5773 _edge_instructions_idx.append(instructions_idx); | |
| 5774 } | |
| 5775 | |
| 5776 // return the current operation of the given edge (predecessor or successor) | |
| 5777 LIR_Op* EdgeMoveOptimizer::instruction_at(int edge) { | |
| 5778 LIR_OpList* instructions = _edge_instructions.at(edge); | |
| 5779 int idx = _edge_instructions_idx.at(edge); | |
| 5780 | |
| 5781 if (idx < instructions->length()) { | |
| 5782 return instructions->at(idx); | |
| 5783 } else { | |
| 5784 return NULL; | |
| 5785 } | |
| 5786 } | |
| 5787 | |
| 5788 // removes the current operation of the given edge (predecessor or successor) | |
| 5789 void EdgeMoveOptimizer::remove_cur_instruction(int edge, bool decrement_index) { | |
| 5790 LIR_OpList* instructions = _edge_instructions.at(edge); | |
| 5791 int idx = _edge_instructions_idx.at(edge); | |
| 5792 instructions->remove_at(idx); | |
| 5793 | |
| 5794 if (decrement_index) { | |
| 5795 _edge_instructions_idx.at_put(edge, idx - 1); | |
| 5796 } | |
| 5797 } | |
| 5798 | |
| 5799 | |
| 5800 bool EdgeMoveOptimizer::operations_different(LIR_Op* op1, LIR_Op* op2) { | |
| 5801 if (op1 == NULL || op2 == NULL) { | |
| 5802 // at least one block is already empty -> no optimization possible | |
| 5803 return true; | |
| 5804 } | |
| 5805 | |
| 5806 if (op1->code() == lir_move && op2->code() == lir_move) { | |
| 5807 assert(op1->as_Op1() != NULL, "move must be LIR_Op1"); | |
| 5808 assert(op2->as_Op1() != NULL, "move must be LIR_Op1"); | |
| 5809 LIR_Op1* move1 = (LIR_Op1*)op1; | |
| 5810 LIR_Op1* move2 = (LIR_Op1*)op2; | |
| 5811 if (move1->info() == move2->info() && move1->in_opr() == move2->in_opr() && move1->result_opr() == move2->result_opr()) { | |
| 5812 // these moves are exactly equal and can be optimized | |
| 5813 return false; | |
| 5814 } | |
| 5815 | |
| 5816 } else if (op1->code() == lir_fxch && op2->code() == lir_fxch) { | |
| 5817 assert(op1->as_Op1() != NULL, "fxch must be LIR_Op1"); | |
| 5818 assert(op2->as_Op1() != NULL, "fxch must be LIR_Op1"); | |
| 5819 LIR_Op1* fxch1 = (LIR_Op1*)op1; | |
| 5820 LIR_Op1* fxch2 = (LIR_Op1*)op2; | |
| 5821 if (fxch1->in_opr()->as_jint() == fxch2->in_opr()->as_jint()) { | |
| 5822 // equal FPU stack operations can be optimized | |
| 5823 return false; | |
| 5824 } | |
| 5825 | |
| 5826 } else if (op1->code() == lir_fpop_raw && op2->code() == lir_fpop_raw) { | |
| 5827 // equal FPU stack operations can be optimized | |
| 5828 return false; | |
| 5829 } | |
| 5830 | |
| 5831 // no optimization possible | |
| 5832 return true; | |
| 5833 } | |
| 5834 | |
| 5835 void EdgeMoveOptimizer::optimize_moves_at_block_end(BlockBegin* block) { | |
| 5836 TRACE_LINEAR_SCAN(4, tty->print_cr("optimizing moves at end of block B%d", block->block_id())); | |
| 5837 | |
| 5838 if (block->is_predecessor(block)) { | |
| 5839 // currently we can't handle this correctly. | |
| 5840 return; | |
| 5841 } | |
| 5842 | |
| 5843 init_instructions(); | |
| 5844 int num_preds = block->number_of_preds(); | |
| 5845 assert(num_preds > 1, "do not call otherwise"); | |
| 5846 assert(!block->is_set(BlockBegin::exception_entry_flag), "exception handlers not allowed"); | |
| 5847 | |
| 5848 // setup a list with the lir-instructions of all predecessors | |
| 5849 int i; | |
| 5850 for (i = 0; i < num_preds; i++) { | |
| 5851 BlockBegin* pred = block->pred_at(i); | |
| 5852 LIR_OpList* pred_instructions = pred->lir()->instructions_list(); | |
| 5853 | |
| 5854 if (pred->number_of_sux() != 1) { | |
| 5855 // this can happen with switch-statements where multiple edges are between | |
| 5856 // the same blocks. | |
| 5857 return; | |
| 5858 } | |
| 5859 | |
| 5860 assert(pred->number_of_sux() == 1, "can handle only one successor"); | |
| 5861 assert(pred->sux_at(0) == block, "invalid control flow"); | |
| 5862 assert(pred_instructions->last()->code() == lir_branch, "block with successor must end with branch"); | |
| 5863 assert(pred_instructions->last()->as_OpBranch() != NULL, "branch must be LIR_OpBranch"); | |
| 5864 assert(pred_instructions->last()->as_OpBranch()->cond() == lir_cond_always, "block must end with unconditional branch"); | |
| 5865 | |
| 5866 if (pred_instructions->last()->info() != NULL) { | |
| 5867 // can not optimize instructions when debug info is needed | |
| 5868 return; | |
| 5869 } | |
| 5870 | |
| 5871 // ignore the unconditional branch at the end of the block | |
| 5872 append_instructions(pred_instructions, pred_instructions->length() - 2); | |
| 5873 } | |
| 5874 | |
| 5875 | |
| 5876 // process lir-instructions while all predecessors end with the same instruction | |
| 5877 while (true) { | |
| 5878 LIR_Op* op = instruction_at(0); | |
| 5879 for (i = 1; i < num_preds; i++) { | |
| 5880 if (operations_different(op, instruction_at(i))) { | |
| 5881 // these instructions are different and cannot be optimized -> | |
| 5882 // no further optimization possible | |
| 5883 return; | |
| 5884 } | |
| 5885 } | |
| 5886 | |
| 5887 TRACE_LINEAR_SCAN(4, tty->print("found instruction that is equal in all %d predecessors: ", num_preds); op->print()); | |
| 5888 | |
| 5889 // insert the instruction at the beginning of the current block | |
| 5890 block->lir()->insert_before(1, op); | |
| 5891 | |
| 5892 // delete the instruction at the end of all predecessors | |
| 5893 for (i = 0; i < num_preds; i++) { | |
| 5894 remove_cur_instruction(i, true); | |
| 5895 } | |
| 5896 } | |
| 5897 } | |
| 5898 | |
| 5899 | |
| 5900 void EdgeMoveOptimizer::optimize_moves_at_block_begin(BlockBegin* block) { | |
| 5901 TRACE_LINEAR_SCAN(4, tty->print_cr("optimization moves at begin of block B%d", block->block_id())); | |
| 5902 | |
| 5903 init_instructions(); | |
| 5904 int num_sux = block->number_of_sux(); | |
| 5905 | |
| 5906 LIR_OpList* cur_instructions = block->lir()->instructions_list(); | |
| 5907 | |
| 5908 assert(num_sux == 2, "method should not be called otherwise"); | |
| 5909 assert(cur_instructions->last()->code() == lir_branch, "block with successor must end with branch"); | |
| 5910 assert(cur_instructions->last()->as_OpBranch() != NULL, "branch must be LIR_OpBranch"); | |
| 5911 assert(cur_instructions->last()->as_OpBranch()->cond() == lir_cond_always, "block must end with unconditional branch"); | |
| 5912 | |
| 5913 if (cur_instructions->last()->info() != NULL) { | |
| 5914 // can no optimize instructions when debug info is needed | |
| 5915 return; | |
| 5916 } | |
| 5917 | |
| 5918 LIR_Op* branch = cur_instructions->at(cur_instructions->length() - 2); | |
| 5919 if (branch->info() != NULL || (branch->code() != lir_branch && branch->code() != lir_cond_float_branch)) { | |
| 5920 // not a valid case for optimization | |
| 5921 // currently, only blocks that end with two branches (conditional branch followed | |
| 5922 // by unconditional branch) are optimized | |
| 5923 return; | |
| 5924 } | |
| 5925 | |
| 5926 // now it is guaranteed that the block ends with two branch instructions. | |
| 5927 // the instructions are inserted at the end of the block before these two branches | |
| 5928 int insert_idx = cur_instructions->length() - 2; | |
| 5929 | |
| 5930 int i; | |
| 5931 #ifdef ASSERT | |
| 5932 for (i = insert_idx - 1; i >= 0; i--) { | |
| 5933 LIR_Op* op = cur_instructions->at(i); | |
| 5934 if ((op->code() == lir_branch || op->code() == lir_cond_float_branch) && ((LIR_OpBranch*)op)->block() != NULL) { | |
| 5935 assert(false, "block with two successors can have only two branch instructions"); | |
| 5936 } | |
| 5937 } | |
| 5938 #endif | |
| 5939 | |
| 5940 // setup a list with the lir-instructions of all successors | |
| 5941 for (i = 0; i < num_sux; i++) { | |
| 5942 BlockBegin* sux = block->sux_at(i); | |
| 5943 LIR_OpList* sux_instructions = sux->lir()->instructions_list(); | |
| 5944 | |
| 5945 assert(sux_instructions->at(0)->code() == lir_label, "block must start with label"); | |
| 5946 | |
| 5947 if (sux->number_of_preds() != 1) { | |
| 5948 // this can happen with switch-statements where multiple edges are between | |
| 5949 // the same blocks. | |
| 5950 return; | |
| 5951 } | |
| 5952 assert(sux->pred_at(0) == block, "invalid control flow"); | |
| 5953 assert(!sux->is_set(BlockBegin::exception_entry_flag), "exception handlers not allowed"); | |
| 5954 | |
| 5955 // ignore the label at the beginning of the block | |
| 5956 append_instructions(sux_instructions, 1); | |
| 5957 } | |
| 5958 | |
| 5959 // process lir-instructions while all successors begin with the same instruction | |
| 5960 while (true) { | |
| 5961 LIR_Op* op = instruction_at(0); | |
| 5962 for (i = 1; i < num_sux; i++) { | |
| 5963 if (operations_different(op, instruction_at(i))) { | |
| 5964 // these instructions are different and cannot be optimized -> | |
| 5965 // no further optimization possible | |
| 5966 return; | |
| 5967 } | |
| 5968 } | |
| 5969 | |
| 5970 TRACE_LINEAR_SCAN(4, tty->print("----- found instruction that is equal in all %d successors: ", num_sux); op->print()); | |
| 5971 | |
| 5972 // insert instruction at end of current block | |
| 5973 block->lir()->insert_before(insert_idx, op); | |
| 5974 insert_idx++; | |
| 5975 | |
| 5976 // delete the instructions at the beginning of all successors | |
| 5977 for (i = 0; i < num_sux; i++) { | |
| 5978 remove_cur_instruction(i, false); | |
| 5979 } | |
| 5980 } | |
| 5981 } | |
| 5982 | |
| 5983 | |
| 5984 // Implementation of ControlFlowOptimizer | |
| 5985 | |
| 5986 ControlFlowOptimizer::ControlFlowOptimizer() : | |
| 5987 _original_preds(4) | |
| 5988 { | |
| 5989 } | |
| 5990 | |
| 5991 void ControlFlowOptimizer::optimize(BlockList* code) { | |
| 5992 ControlFlowOptimizer optimizer = ControlFlowOptimizer(); | |
| 5993 | |
| 5994 // push the OSR entry block to the end so that we're not jumping over it. | |
| 5995 BlockBegin* osr_entry = code->at(0)->end()->as_Base()->osr_entry(); | |
| 5996 if (osr_entry) { | |
| 5997 int index = osr_entry->linear_scan_number(); | |
| 5998 assert(code->at(index) == osr_entry, "wrong index"); | |
| 5999 code->remove_at(index); | |
| 6000 code->append(osr_entry); | |
| 6001 } | |
| 6002 | |
| 6003 optimizer.reorder_short_loops(code); | |
| 6004 optimizer.delete_empty_blocks(code); | |
| 6005 optimizer.delete_unnecessary_jumps(code); | |
| 6006 optimizer.delete_jumps_to_return(code); | |
| 6007 } | |
| 6008 | |
| 6009 void ControlFlowOptimizer::reorder_short_loop(BlockList* code, BlockBegin* header_block, int header_idx) { | |
| 6010 int i = header_idx + 1; | |
| 6011 int max_end = MIN2(header_idx + ShortLoopSize, code->length()); | |
| 6012 while (i < max_end && code->at(i)->loop_depth() >= header_block->loop_depth()) { | |
| 6013 i++; | |
| 6014 } | |
| 6015 | |
| 6016 if (i == code->length() || code->at(i)->loop_depth() < header_block->loop_depth()) { | |
| 6017 int end_idx = i - 1; | |
| 6018 BlockBegin* end_block = code->at(end_idx); | |
| 6019 | |
| 6020 if (end_block->number_of_sux() == 1 && end_block->sux_at(0) == header_block) { | |
| 6021 // short loop from header_idx to end_idx found -> reorder blocks such that | |
| 6022 // the header_block is the last block instead of the first block of the loop | |
| 6023 TRACE_LINEAR_SCAN(1, tty->print_cr("Reordering short loop: length %d, header B%d, end B%d", | |
| 6024 end_idx - header_idx + 1, | |
| 6025 header_block->block_id(), end_block->block_id())); | |
| 6026 | |
| 6027 for (int j = header_idx; j < end_idx; j++) { | |
| 6028 code->at_put(j, code->at(j + 1)); | |
| 6029 } | |
| 6030 code->at_put(end_idx, header_block); | |
| 6031 | |
| 6032 // correct the flags so that any loop alignment occurs in the right place. | |
| 6033 assert(code->at(end_idx)->is_set(BlockBegin::backward_branch_target_flag), "must be backward branch target"); | |
| 6034 code->at(end_idx)->clear(BlockBegin::backward_branch_target_flag); | |
| 6035 code->at(header_idx)->set(BlockBegin::backward_branch_target_flag); | |
| 6036 } | |
| 6037 } | |
| 6038 } | |
| 6039 | |
| 6040 void ControlFlowOptimizer::reorder_short_loops(BlockList* code) { | |
| 6041 for (int i = code->length() - 1; i >= 0; i--) { | |
| 6042 BlockBegin* block = code->at(i); | |
| 6043 | |
| 6044 if (block->is_set(BlockBegin::linear_scan_loop_header_flag)) { | |
| 6045 reorder_short_loop(code, block, i); | |
| 6046 } | |
| 6047 } | |
| 6048 | |
| 6049 DEBUG_ONLY(verify(code)); | |
| 6050 } | |
| 6051 | |
| 6052 // only blocks with exactly one successor can be deleted. Such blocks | |
| 6053 // must always end with an unconditional branch to this successor | |
| 6054 bool ControlFlowOptimizer::can_delete_block(BlockBegin* block) { | |
| 6055 if (block->number_of_sux() != 1 || block->number_of_exception_handlers() != 0 || block->is_entry_block()) { | |
| 6056 return false; | |
| 6057 } | |
| 6058 | |
| 6059 LIR_OpList* instructions = block->lir()->instructions_list(); | |
| 6060 | |
| 6061 assert(instructions->length() >= 2, "block must have label and branch"); | |
| 6062 assert(instructions->at(0)->code() == lir_label, "first instruction must always be a label"); | |
| 6063 assert(instructions->last()->as_OpBranch() != NULL, "last instrcution must always be a branch"); | |
| 6064 assert(instructions->last()->as_OpBranch()->cond() == lir_cond_always, "branch must be unconditional"); | |
| 6065 assert(instructions->last()->as_OpBranch()->block() == block->sux_at(0), "branch target must be the successor"); | |
| 6066 | |
| 6067 // block must have exactly one successor | |
| 6068 | |
| 6069 if (instructions->length() == 2 && instructions->last()->info() == NULL) { | |
| 6070 return true; | |
| 6071 } | |
| 6072 return false; | |
| 6073 } | |
| 6074 | |
| 6075 // substitute branch targets in all branch-instructions of this blocks | |
| 6076 void ControlFlowOptimizer::substitute_branch_target(BlockBegin* block, BlockBegin* target_from, BlockBegin* target_to) { | |
| 6077 TRACE_LINEAR_SCAN(3, tty->print_cr("Deleting empty block: substituting from B%d to B%d inside B%d", target_from->block_id(), target_to->block_id(), block->block_id())); | |
| 6078 | |
| 6079 LIR_OpList* instructions = block->lir()->instructions_list(); | |
| 6080 | |
| 6081 assert(instructions->at(0)->code() == lir_label, "first instruction must always be a label"); | |
| 6082 for (int i = instructions->length() - 1; i >= 1; i--) { | |
| 6083 LIR_Op* op = instructions->at(i); | |
| 6084 | |
| 6085 if (op->code() == lir_branch || op->code() == lir_cond_float_branch) { | |
| 6086 assert(op->as_OpBranch() != NULL, "branch must be of type LIR_OpBranch"); | |
| 6087 LIR_OpBranch* branch = (LIR_OpBranch*)op; | |
| 6088 | |
| 6089 if (branch->block() == target_from) { | |
| 6090 branch->change_block(target_to); | |
| 6091 } | |
| 6092 if (branch->ublock() == target_from) { | |
| 6093 branch->change_ublock(target_to); | |
| 6094 } | |
| 6095 } | |
| 6096 } | |
| 6097 } | |
| 6098 | |
| 6099 void ControlFlowOptimizer::delete_empty_blocks(BlockList* code) { | |
| 6100 int old_pos = 0; | |
| 6101 int new_pos = 0; | |
| 6102 int num_blocks = code->length(); | |
| 6103 | |
| 6104 while (old_pos < num_blocks) { | |
| 6105 BlockBegin* block = code->at(old_pos); | |
| 6106 | |
| 6107 if (can_delete_block(block)) { | |
| 6108 BlockBegin* new_target = block->sux_at(0); | |
| 6109 | |
| 6110 // propagate backward branch target flag for correct code alignment | |
| 6111 if (block->is_set(BlockBegin::backward_branch_target_flag)) { | |
| 6112 new_target->set(BlockBegin::backward_branch_target_flag); | |
| 6113 } | |
| 6114 | |
| 6115 // collect a list with all predecessors that contains each predecessor only once | |
| 6116 // the predecessors of cur are changed during the substitution, so a copy of the | |
| 6117 // predecessor list is necessary | |
| 6118 int j; | |
| 6119 _original_preds.clear(); | |
| 6120 for (j = block->number_of_preds() - 1; j >= 0; j--) { | |
| 6121 BlockBegin* pred = block->pred_at(j); | |
| 6122 if (_original_preds.index_of(pred) == -1) { | |
| 6123 _original_preds.append(pred); | |
| 6124 } | |
| 6125 } | |
| 6126 | |
| 6127 for (j = _original_preds.length() - 1; j >= 0; j--) { | |
| 6128 BlockBegin* pred = _original_preds.at(j); | |
| 6129 substitute_branch_target(pred, block, new_target); | |
| 6130 pred->substitute_sux(block, new_target); | |
| 6131 } | |
| 6132 } else { | |
| 6133 // adjust position of this block in the block list if blocks before | |
| 6134 // have been deleted | |
| 6135 if (new_pos != old_pos) { | |
| 6136 code->at_put(new_pos, code->at(old_pos)); | |
| 6137 } | |
| 6138 new_pos++; | |
| 6139 } | |
| 6140 old_pos++; | |
| 6141 } | |
| 6142 code->truncate(new_pos); | |
| 6143 | |
| 6144 DEBUG_ONLY(verify(code)); | |
| 6145 } | |
| 6146 | |
| 6147 void ControlFlowOptimizer::delete_unnecessary_jumps(BlockList* code) { | |
| 6148 // skip the last block because there a branch is always necessary | |
| 6149 for (int i = code->length() - 2; i >= 0; i--) { | |
| 6150 BlockBegin* block = code->at(i); | |
| 6151 LIR_OpList* instructions = block->lir()->instructions_list(); | |
| 6152 | |
| 6153 LIR_Op* last_op = instructions->last(); | |
| 6154 if (last_op->code() == lir_branch) { | |
| 6155 assert(last_op->as_OpBranch() != NULL, "branch must be of type LIR_OpBranch"); | |
| 6156 LIR_OpBranch* last_branch = (LIR_OpBranch*)last_op; | |
| 6157 | |
| 6158 assert(last_branch->block() != NULL, "last branch must always have a block as target"); | |
| 6159 assert(last_branch->label() == last_branch->block()->label(), "must be equal"); | |
| 6160 | |
| 6161 if (last_branch->info() == NULL) { | |
| 6162 if (last_branch->block() == code->at(i + 1)) { | |
| 6163 | |
| 6164 TRACE_LINEAR_SCAN(3, tty->print_cr("Deleting unconditional branch at end of block B%d", block->block_id())); | |
| 6165 | |
| 6166 // delete last branch instruction | |
| 6167 instructions->truncate(instructions->length() - 1); | |
| 6168 | |
| 6169 } else { | |
| 6170 LIR_Op* prev_op = instructions->at(instructions->length() - 2); | |
| 6171 if (prev_op->code() == lir_branch || prev_op->code() == lir_cond_float_branch) { | |
| 6172 assert(prev_op->as_OpBranch() != NULL, "branch must be of type LIR_OpBranch"); | |
| 6173 LIR_OpBranch* prev_branch = (LIR_OpBranch*)prev_op; | |
| 6174 | |
|
1681
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
6175 LIR_Op2* prev_cmp = NULL; |
|
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
6176 |
|
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
6177 for(int j = instructions->length() - 3; j >= 0 && prev_cmp == NULL; j--) { |
|
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
6178 prev_op = instructions->at(j); |
|
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
6179 if(prev_op->code() == lir_cmp) { |
|
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
6180 assert(prev_op->as_Op2() != NULL, "branch must be of type LIR_Op2"); |
|
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
6181 prev_cmp = (LIR_Op2*)prev_op; |
|
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
6182 assert(prev_branch->cond() == prev_cmp->condition(), "should be the same"); |
|
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
6183 } |
|
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
6184 } |
|
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
6185 assert(prev_cmp != NULL, "should have found comp instruction for branch"); |
| 0 | 6186 if (prev_branch->block() == code->at(i + 1) && prev_branch->info() == NULL) { |
| 6187 | |
| 6188 TRACE_LINEAR_SCAN(3, tty->print_cr("Negating conditional branch and deleting unconditional branch at end of block B%d", block->block_id())); | |
| 6189 | |
| 6190 // eliminate a conditional branch to the immediate successor | |
| 6191 prev_branch->change_block(last_branch->block()); | |
| 6192 prev_branch->negate_cond(); | |
|
1681
126ea7725993
6953477: Increase portability and flexibility of building Hotspot
bobv
parents:
1584
diff
changeset
|
6193 prev_cmp->set_condition(prev_branch->cond()); |
| 0 | 6194 instructions->truncate(instructions->length() - 1); |
| 6195 } | |
| 6196 } | |
| 6197 } | |
| 6198 } | |
| 6199 } | |
| 6200 } | |
| 6201 | |
| 6202 DEBUG_ONLY(verify(code)); | |
| 6203 } | |
| 6204 | |
| 6205 void ControlFlowOptimizer::delete_jumps_to_return(BlockList* code) { | |
| 6206 #ifdef ASSERT | |
| 6207 BitMap return_converted(BlockBegin::number_of_blocks()); | |
| 6208 return_converted.clear(); | |
| 6209 #endif | |
| 6210 | |
| 6211 for (int i = code->length() - 1; i >= 0; i--) { | |
| 6212 BlockBegin* block = code->at(i); | |
| 6213 LIR_OpList* cur_instructions = block->lir()->instructions_list(); | |
| 6214 LIR_Op* cur_last_op = cur_instructions->last(); | |
| 6215 | |
| 6216 assert(cur_instructions->at(0)->code() == lir_label, "first instruction must always be a label"); | |
| 6217 if (cur_instructions->length() == 2 && cur_last_op->code() == lir_return) { | |
| 6218 // the block contains only a label and a return | |
| 6219 // if a predecessor ends with an unconditional jump to this block, then the jump | |
| 6220 // can be replaced with a return instruction | |
| 6221 // | |
| 6222 // Note: the original block with only a return statement cannot be deleted completely | |
| 6223 // because the predecessors might have other (conditional) jumps to this block | |
| 6224 // -> this may lead to unnecesary return instructions in the final code | |
| 6225 | |
| 6226 assert(cur_last_op->info() == NULL, "return instructions do not have debug information"); | |
| 6227 assert(block->number_of_sux() == 0 || | |
| 6228 (return_converted.at(block->block_id()) && block->number_of_sux() == 1), | |
| 6229 "blocks that end with return must not have successors"); | |
| 6230 | |
| 6231 assert(cur_last_op->as_Op1() != NULL, "return must be LIR_Op1"); | |
| 6232 LIR_Opr return_opr = ((LIR_Op1*)cur_last_op)->in_opr(); | |
| 6233 | |
| 6234 for (int j = block->number_of_preds() - 1; j >= 0; j--) { | |
| 6235 BlockBegin* pred = block->pred_at(j); | |
| 6236 LIR_OpList* pred_instructions = pred->lir()->instructions_list(); | |
| 6237 LIR_Op* pred_last_op = pred_instructions->last(); | |
| 6238 | |
| 6239 if (pred_last_op->code() == lir_branch) { | |
| 6240 assert(pred_last_op->as_OpBranch() != NULL, "branch must be LIR_OpBranch"); | |
| 6241 LIR_OpBranch* pred_last_branch = (LIR_OpBranch*)pred_last_op; | |
| 6242 | |
| 6243 if (pred_last_branch->block() == block && pred_last_branch->cond() == lir_cond_always && pred_last_branch->info() == NULL) { | |
| 6244 // replace the jump to a return with a direct return | |
| 6245 // Note: currently the edge between the blocks is not deleted | |
| 6246 pred_instructions->at_put(pred_instructions->length() - 1, new LIR_Op1(lir_return, return_opr)); | |
| 6247 #ifdef ASSERT | |
| 6248 return_converted.set_bit(pred->block_id()); | |
| 6249 #endif | |
| 6250 } | |
| 6251 } | |
| 6252 } | |
| 6253 } | |
| 6254 } | |
| 6255 } | |
| 6256 | |
| 6257 | |
| 6258 #ifdef ASSERT | |
| 6259 void ControlFlowOptimizer::verify(BlockList* code) { | |
| 6260 for (int i = 0; i < code->length(); i++) { | |
| 6261 BlockBegin* block = code->at(i); | |
| 6262 LIR_OpList* instructions = block->lir()->instructions_list(); | |
| 6263 | |
| 6264 int j; | |
| 6265 for (j = 0; j < instructions->length(); j++) { | |
| 6266 LIR_OpBranch* op_branch = instructions->at(j)->as_OpBranch(); | |
| 6267 | |
| 6268 if (op_branch != NULL) { | |
| 6269 assert(op_branch->block() == NULL || code->index_of(op_branch->block()) != -1, "branch target not valid"); | |
| 6270 assert(op_branch->ublock() == NULL || code->index_of(op_branch->ublock()) != -1, "branch target not valid"); | |
| 6271 } | |
| 6272 } | |
| 6273 | |
| 6274 for (j = 0; j < block->number_of_sux() - 1; j++) { | |
| 6275 BlockBegin* sux = block->sux_at(j); | |
| 6276 assert(code->index_of(sux) != -1, "successor not valid"); | |
| 6277 } | |
| 6278 | |
| 6279 for (j = 0; j < block->number_of_preds() - 1; j++) { | |
| 6280 BlockBegin* pred = block->pred_at(j); | |
| 6281 assert(code->index_of(pred) != -1, "successor not valid"); | |
| 6282 } | |
| 6283 } | |
| 6284 } | |
| 6285 #endif | |
| 6286 | |
| 6287 | |
| 6288 #ifndef PRODUCT | |
| 6289 | |
| 6290 // Implementation of LinearStatistic | |
| 6291 | |
| 6292 const char* LinearScanStatistic::counter_name(int counter_idx) { | |
| 6293 switch (counter_idx) { | |
| 6294 case counter_method: return "compiled methods"; | |
| 6295 case counter_fpu_method: return "methods using fpu"; | |
| 6296 case counter_loop_method: return "methods with loops"; | |
| 6297 case counter_exception_method:return "methods with xhandler"; | |
| 6298 | |
| 6299 case counter_loop: return "loops"; | |
| 6300 case counter_block: return "blocks"; | |
| 6301 case counter_loop_block: return "blocks inside loop"; | |
| 6302 case counter_exception_block: return "exception handler entries"; | |
| 6303 case counter_interval: return "intervals"; | |
| 6304 case counter_fixed_interval: return "fixed intervals"; | |
| 6305 case counter_range: return "ranges"; | |
| 6306 case counter_fixed_range: return "fixed ranges"; | |
| 6307 case counter_use_pos: return "use positions"; | |
| 6308 case counter_fixed_use_pos: return "fixed use positions"; | |
| 6309 case counter_spill_slots: return "spill slots"; | |
| 6310 | |
| 6311 // counter for classes of lir instructions | |
| 6312 case counter_instruction: return "total instructions"; | |
| 6313 case counter_label: return "labels"; | |
| 6314 case counter_entry: return "method entries"; | |
| 6315 case counter_return: return "method returns"; | |
| 6316 case counter_call: return "method calls"; | |
| 6317 case counter_move: return "moves"; | |
| 6318 case counter_cmp: return "compare"; | |
| 6319 case counter_cond_branch: return "conditional branches"; | |
| 6320 case counter_uncond_branch: return "unconditional branches"; | |
| 6321 case counter_stub_branch: return "branches to stub"; | |
| 6322 case counter_alu: return "artithmetic + logic"; | |
| 6323 case counter_alloc: return "allocations"; | |
| 6324 case counter_sync: return "synchronisation"; | |
| 6325 case counter_throw: return "throw"; | |
| 6326 case counter_unwind: return "unwind"; | |
| 6327 case counter_typecheck: return "type+null-checks"; | |
| 6328 case counter_fpu_stack: return "fpu-stack"; | |
| 6329 case counter_misc_inst: return "other instructions"; | |
| 6330 case counter_other_inst: return "misc. instructions"; | |
| 6331 | |
| 6332 // counter for different types of moves | |
| 6333 case counter_move_total: return "total moves"; | |
| 6334 case counter_move_reg_reg: return "register->register"; | |
| 6335 case counter_move_reg_stack: return "register->stack"; | |
| 6336 case counter_move_stack_reg: return "stack->register"; | |
| 6337 case counter_move_stack_stack:return "stack->stack"; | |
| 6338 case counter_move_reg_mem: return "register->memory"; | |
| 6339 case counter_move_mem_reg: return "memory->register"; | |
| 6340 case counter_move_const_any: return "constant->any"; | |
| 6341 | |
| 6342 case blank_line_1: return ""; | |
| 6343 case blank_line_2: return ""; | |
| 6344 | |
| 6345 default: ShouldNotReachHere(); return ""; | |
| 6346 } | |
| 6347 } | |
| 6348 | |
| 6349 LinearScanStatistic::Counter LinearScanStatistic::base_counter(int counter_idx) { | |
| 6350 if (counter_idx == counter_fpu_method || counter_idx == counter_loop_method || counter_idx == counter_exception_method) { | |
| 6351 return counter_method; | |
| 6352 } else if (counter_idx == counter_loop_block || counter_idx == counter_exception_block) { | |
| 6353 return counter_block; | |
| 6354 } else if (counter_idx >= counter_instruction && counter_idx <= counter_other_inst) { | |
| 6355 return counter_instruction; | |
| 6356 } else if (counter_idx >= counter_move_total && counter_idx <= counter_move_const_any) { | |
| 6357 return counter_move_total; | |
| 6358 } | |
| 6359 return invalid_counter; | |
| 6360 } | |
| 6361 | |
| 6362 LinearScanStatistic::LinearScanStatistic() { | |
| 6363 for (int i = 0; i < number_of_counters; i++) { | |
| 6364 _counters_sum[i] = 0; | |
| 6365 _counters_max[i] = -1; | |
| 6366 } | |
| 6367 | |
| 6368 } | |
| 6369 | |
| 6370 // add the method-local numbers to the total sum | |
| 6371 void LinearScanStatistic::sum_up(LinearScanStatistic &method_statistic) { | |
| 6372 for (int i = 0; i < number_of_counters; i++) { | |
| 6373 _counters_sum[i] += method_statistic._counters_sum[i]; | |
| 6374 _counters_max[i] = MAX2(_counters_max[i], method_statistic._counters_sum[i]); | |
| 6375 } | |
| 6376 } | |
| 6377 | |
| 6378 void LinearScanStatistic::print(const char* title) { | |
| 6379 if (CountLinearScan || TraceLinearScanLevel > 0) { | |
| 6380 tty->cr(); | |
| 6381 tty->print_cr("***** LinearScan statistic - %s *****", title); | |
| 6382 | |
| 6383 for (int i = 0; i < number_of_counters; i++) { | |
| 6384 if (_counters_sum[i] > 0 || _counters_max[i] >= 0) { | |
| 6385 tty->print("%25s: %8d", counter_name(i), _counters_sum[i]); | |
| 6386 | |
| 6387 if (base_counter(i) != invalid_counter) { | |
| 6388 tty->print(" (%5.1f%%) ", _counters_sum[i] * 100.0 / _counters_sum[base_counter(i)]); | |
| 6389 } else { | |
| 6390 tty->print(" "); | |
| 6391 } | |
| 6392 | |
| 6393 if (_counters_max[i] >= 0) { | |
| 6394 tty->print("%8d", _counters_max[i]); | |
| 6395 } | |
| 6396 } | |
| 6397 tty->cr(); | |
| 6398 } | |
| 6399 } | |
| 6400 } | |
| 6401 | |
| 6402 void LinearScanStatistic::collect(LinearScan* allocator) { | |
| 6403 inc_counter(counter_method); | |
| 6404 if (allocator->has_fpu_registers()) { | |
| 6405 inc_counter(counter_fpu_method); | |
| 6406 } | |
| 6407 if (allocator->num_loops() > 0) { | |
| 6408 inc_counter(counter_loop_method); | |
| 6409 } | |
| 6410 inc_counter(counter_loop, allocator->num_loops()); | |
| 6411 inc_counter(counter_spill_slots, allocator->max_spills()); | |
| 6412 | |
| 6413 int i; | |
| 6414 for (i = 0; i < allocator->interval_count(); i++) { | |
| 6415 Interval* cur = allocator->interval_at(i); | |
| 6416 | |
| 6417 if (cur != NULL) { | |
| 6418 inc_counter(counter_interval); | |
| 6419 inc_counter(counter_use_pos, cur->num_use_positions()); | |
| 6420 if (LinearScan::is_precolored_interval(cur)) { | |
| 6421 inc_counter(counter_fixed_interval); | |
| 6422 inc_counter(counter_fixed_use_pos, cur->num_use_positions()); | |
| 6423 } | |
| 6424 | |
| 6425 Range* range = cur->first(); | |
| 6426 while (range != Range::end()) { | |
| 6427 inc_counter(counter_range); | |
| 6428 if (LinearScan::is_precolored_interval(cur)) { | |
| 6429 inc_counter(counter_fixed_range); | |
| 6430 } | |
| 6431 range = range->next(); | |
| 6432 } | |
| 6433 } | |
| 6434 } | |
| 6435 | |
| 6436 bool has_xhandlers = false; | |
| 6437 // Note: only count blocks that are in code-emit order | |
| 6438 for (i = 0; i < allocator->ir()->code()->length(); i++) { | |
| 6439 BlockBegin* cur = allocator->ir()->code()->at(i); | |
| 6440 | |
| 6441 inc_counter(counter_block); | |
| 6442 if (cur->loop_depth() > 0) { | |
| 6443 inc_counter(counter_loop_block); | |
| 6444 } | |
| 6445 if (cur->is_set(BlockBegin::exception_entry_flag)) { | |
| 6446 inc_counter(counter_exception_block); | |
| 6447 has_xhandlers = true; | |
| 6448 } | |
| 6449 | |
| 6450 LIR_OpList* instructions = cur->lir()->instructions_list(); | |
| 6451 for (int j = 0; j < instructions->length(); j++) { | |
| 6452 LIR_Op* op = instructions->at(j); | |
| 6453 | |
| 6454 inc_counter(counter_instruction); | |
| 6455 | |
| 6456 switch (op->code()) { | |
| 6457 case lir_label: inc_counter(counter_label); break; | |
| 6458 case lir_std_entry: | |
| 6459 case lir_osr_entry: inc_counter(counter_entry); break; | |
| 6460 case lir_return: inc_counter(counter_return); break; | |
| 6461 | |
| 6462 case lir_rtcall: | |
| 6463 case lir_static_call: | |
| 6464 case lir_optvirtual_call: | |
| 6465 case lir_virtual_call: inc_counter(counter_call); break; | |
| 6466 | |
| 6467 case lir_move: { | |
| 6468 inc_counter(counter_move); | |
| 6469 inc_counter(counter_move_total); | |
| 6470 | |
| 6471 LIR_Opr in = op->as_Op1()->in_opr(); | |
| 6472 LIR_Opr res = op->as_Op1()->result_opr(); | |
| 6473 if (in->is_register()) { | |
| 6474 if (res->is_register()) { | |
| 6475 inc_counter(counter_move_reg_reg); | |
| 6476 } else if (res->is_stack()) { | |
| 6477 inc_counter(counter_move_reg_stack); | |
| 6478 } else if (res->is_address()) { | |
| 6479 inc_counter(counter_move_reg_mem); | |
| 6480 } else { | |
| 6481 ShouldNotReachHere(); | |
| 6482 } | |
| 6483 } else if (in->is_stack()) { | |
| 6484 if (res->is_register()) { | |
| 6485 inc_counter(counter_move_stack_reg); | |
| 6486 } else { | |
| 6487 inc_counter(counter_move_stack_stack); | |
| 6488 } | |
| 6489 } else if (in->is_address()) { | |
| 6490 assert(res->is_register(), "must be"); | |
| 6491 inc_counter(counter_move_mem_reg); | |
| 6492 } else if (in->is_constant()) { | |
| 6493 inc_counter(counter_move_const_any); | |
| 6494 } else { | |
| 6495 ShouldNotReachHere(); | |
| 6496 } | |
| 6497 break; | |
| 6498 } | |
| 6499 | |
| 6500 case lir_cmp: inc_counter(counter_cmp); break; | |
| 6501 | |
| 6502 case lir_branch: | |
| 6503 case lir_cond_float_branch: { | |
| 6504 LIR_OpBranch* branch = op->as_OpBranch(); | |
| 6505 if (branch->block() == NULL) { | |
| 6506 inc_counter(counter_stub_branch); | |
| 6507 } else if (branch->cond() == lir_cond_always) { | |
| 6508 inc_counter(counter_uncond_branch); | |
| 6509 } else { | |
| 6510 inc_counter(counter_cond_branch); | |
| 6511 } | |
| 6512 break; | |
| 6513 } | |
| 6514 | |
| 6515 case lir_neg: | |
| 6516 case lir_add: | |
| 6517 case lir_sub: | |
| 6518 case lir_mul: | |
| 6519 case lir_mul_strictfp: | |
| 6520 case lir_div: | |
| 6521 case lir_div_strictfp: | |
| 6522 case lir_rem: | |
| 6523 case lir_sqrt: | |
| 6524 case lir_sin: | |
| 6525 case lir_cos: | |
| 6526 case lir_abs: | |
| 6527 case lir_log10: | |
| 6528 case lir_log: | |
| 6529 case lir_logic_and: | |
| 6530 case lir_logic_or: | |
| 6531 case lir_logic_xor: | |
| 6532 case lir_shl: | |
| 6533 case lir_shr: | |
| 6534 case lir_ushr: inc_counter(counter_alu); break; | |
| 6535 | |
| 6536 case lir_alloc_object: | |
| 6537 case lir_alloc_array: inc_counter(counter_alloc); break; | |
| 6538 | |
| 6539 case lir_monaddr: | |
| 6540 case lir_lock: | |
| 6541 case lir_unlock: inc_counter(counter_sync); break; | |
| 6542 | |
| 6543 case lir_throw: inc_counter(counter_throw); break; | |
| 6544 | |
| 6545 case lir_unwind: inc_counter(counter_unwind); break; | |
| 6546 | |
| 6547 case lir_null_check: | |
| 6548 case lir_leal: | |
| 6549 case lir_instanceof: | |
| 6550 case lir_checkcast: | |
| 6551 case lir_store_check: inc_counter(counter_typecheck); break; | |
| 6552 | |
| 6553 case lir_fpop_raw: | |
| 6554 case lir_fxch: | |
| 6555 case lir_fld: inc_counter(counter_fpu_stack); break; | |
| 6556 | |
| 6557 case lir_nop: | |
| 6558 case lir_push: | |
| 6559 case lir_pop: | |
| 6560 case lir_convert: | |
| 6561 case lir_roundfp: | |
| 6562 case lir_cmove: inc_counter(counter_misc_inst); break; | |
| 6563 | |
| 6564 default: inc_counter(counter_other_inst); break; | |
| 6565 } | |
| 6566 } | |
| 6567 } | |
| 6568 | |
| 6569 if (has_xhandlers) { | |
| 6570 inc_counter(counter_exception_method); | |
| 6571 } | |
| 6572 } | |
| 6573 | |
| 6574 void LinearScanStatistic::compute(LinearScan* allocator, LinearScanStatistic &global_statistic) { | |
| 6575 if (CountLinearScan || TraceLinearScanLevel > 0) { | |
| 6576 | |
| 6577 LinearScanStatistic local_statistic = LinearScanStatistic(); | |
| 6578 | |
| 6579 local_statistic.collect(allocator); | |
| 6580 global_statistic.sum_up(local_statistic); | |
| 6581 | |
| 6582 if (TraceLinearScanLevel > 2) { | |
| 6583 local_statistic.print("current local statistic"); | |
| 6584 } | |
| 6585 } | |
| 6586 } | |
| 6587 | |
| 6588 | |
| 6589 // Implementation of LinearTimers | |
| 6590 | |
| 6591 LinearScanTimers::LinearScanTimers() { | |
| 6592 for (int i = 0; i < number_of_timers; i++) { | |
| 6593 timer(i)->reset(); | |
| 6594 } | |
| 6595 } | |
| 6596 | |
| 6597 const char* LinearScanTimers::timer_name(int idx) { | |
| 6598 switch (idx) { | |
| 6599 case timer_do_nothing: return "Nothing (Time Check)"; | |
| 6600 case timer_number_instructions: return "Number Instructions"; | |
| 6601 case timer_compute_local_live_sets: return "Local Live Sets"; | |
| 6602 case timer_compute_global_live_sets: return "Global Live Sets"; | |
| 6603 case timer_build_intervals: return "Build Intervals"; | |
| 6604 case timer_sort_intervals_before: return "Sort Intervals Before"; | |
| 6605 case timer_allocate_registers: return "Allocate Registers"; | |
| 6606 case timer_resolve_data_flow: return "Resolve Data Flow"; | |
| 6607 case timer_sort_intervals_after: return "Sort Intervals After"; | |
| 6608 case timer_eliminate_spill_moves: return "Spill optimization"; | |
| 6609 case timer_assign_reg_num: return "Assign Reg Num"; | |
| 6610 case timer_allocate_fpu_stack: return "Allocate FPU Stack"; | |
| 6611 case timer_optimize_lir: return "Optimize LIR"; | |
| 6612 default: ShouldNotReachHere(); return ""; | |
| 6613 } | |
| 6614 } | |
| 6615 | |
| 6616 void LinearScanTimers::begin_method() { | |
| 6617 if (TimeEachLinearScan) { | |
| 6618 // reset all timers to measure only current method | |
| 6619 for (int i = 0; i < number_of_timers; i++) { | |
| 6620 timer(i)->reset(); | |
| 6621 } | |
| 6622 } | |
| 6623 } | |
| 6624 | |
| 6625 void LinearScanTimers::end_method(LinearScan* allocator) { | |
| 6626 if (TimeEachLinearScan) { | |
| 6627 | |
| 6628 double c = timer(timer_do_nothing)->seconds(); | |
| 6629 double total = 0; | |
| 6630 for (int i = 1; i < number_of_timers; i++) { | |
| 6631 total += timer(i)->seconds() - c; | |
| 6632 } | |
| 6633 | |
| 6634 if (total >= 0.0005) { | |
| 6635 // print all information in one line for automatic processing | |
| 6636 tty->print("@"); allocator->compilation()->method()->print_name(); | |
| 6637 | |
| 6638 tty->print("@ %d ", allocator->compilation()->method()->code_size()); | |
| 6639 tty->print("@ %d ", allocator->block_at(allocator->block_count() - 1)->last_lir_instruction_id() / 2); | |
| 6640 tty->print("@ %d ", allocator->block_count()); | |
| 6641 tty->print("@ %d ", allocator->num_virtual_regs()); | |
| 6642 tty->print("@ %d ", allocator->interval_count()); | |
| 6643 tty->print("@ %d ", allocator->_num_calls); | |
| 6644 tty->print("@ %d ", allocator->num_loops()); | |
| 6645 | |
| 6646 tty->print("@ %6.6f ", total); | |
| 6647 for (int i = 1; i < number_of_timers; i++) { | |
| 6648 tty->print("@ %4.1f ", ((timer(i)->seconds() - c) / total) * 100); | |
| 6649 } | |
| 6650 tty->cr(); | |
| 6651 } | |
| 6652 } | |
| 6653 } | |
| 6654 | |
| 6655 void LinearScanTimers::print(double total_time) { | |
| 6656 if (TimeLinearScan) { | |
| 6657 // correction value: sum of dummy-timer that only measures the time that | |
| 6658 // is necesary to start and stop itself | |
| 6659 double c = timer(timer_do_nothing)->seconds(); | |
| 6660 | |
| 6661 for (int i = 0; i < number_of_timers; i++) { | |
| 6662 double t = timer(i)->seconds(); | |
| 6663 tty->print_cr(" %25s: %6.3f s (%4.1f%%) corrected: %6.3f s (%4.1f%%)", timer_name(i), t, (t / total_time) * 100.0, t - c, (t - c) / (total_time - 2 * number_of_timers * c) * 100); | |
| 6664 } | |
| 6665 } | |
| 6666 } | |
| 6667 | |
| 6668 #endif // #ifndef PRODUCT |