Mercurial > hg > truffle
diff src/share/vm/c1/c1_LIRGenerator.cpp @ 1930:2d26b0046e0d
Merge.
author | Thomas Wuerthinger <wuerthinger@ssw.jku.at> |
---|---|
date | Tue, 30 Nov 2010 14:53:30 +0100 |
parents | 0ba67bb5392c 07a218de38cb |
children | 06f017f7daa7 |
line wrap: on
line diff
--- a/src/share/vm/c1/c1_LIRGenerator.cpp Mon Nov 29 18:32:30 2010 +0100 +++ b/src/share/vm/c1/c1_LIRGenerator.cpp Tue Nov 30 14:53:30 2010 +0100 @@ -1,5 +1,5 @@ /* - * Copyright 2005-2010 Sun Microsystems, Inc. All Rights Reserved. + * Copyright (c) 2005, 2010, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -16,9 +16,9 @@ * 2 along with this work; if not, write to the Free Software Foundation, * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. * - * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, - * CA 95054 USA or visit www.sun.com if you need additional information or - * have any questions. + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. * */ @@ -31,6 +31,12 @@ #define __ gen()->lir()-> #endif +// TODO: ARM - Use some recognizable constant which still fits architectural constraints +#ifdef ARM +#define PATCHED_ADDR (204) +#else +#define PATCHED_ADDR (max_jint) +#endif void PhiResolverState::reset(int max_vregs) { // Initialize array sizes @@ -225,13 +231,13 @@ void LIRItem::load_item_force(LIR_Opr reg) { LIR_Opr r = result(); if (r != reg) { +#if !defined(ARM) && !defined(E500V2) if (r->type() != reg->type()) { // moves between different types need an intervening spill slot - LIR_Opr tmp = _gen->force_to_spill(r, reg->type()); - __ move(tmp, reg); - } else { - __ move(r, reg); + r = _gen->force_to_spill(r, reg->type()); } +#endif + __ move(r, reg); _result = reg; } } @@ -304,7 +310,7 @@ __ branch_destination(block->label()); if (LIRTraceExecution && - Compilation::current_compilation()->hir()->start()->block_id() != block->block_id() && + Compilation::current()->hir()->start()->block_id() != block->block_id() && !block->is_set(BlockBegin::exception_entry_flag)) { assert(block->lir()->instructions_list()->length() == 1, "should come right after br_dst"); trace_block_entry(block); @@ -380,18 +386,26 @@ CodeEmitInfo* LIRGenerator::state_for(Instruction* x, ValueStack* state, bool ignore_xhandler) { - int index; - Value value; - for_each_stack_value(state, index, value) { - assert(value->subst() == value, "missed substition"); - if (!value->is_pinned() && value->as_Constant() == NULL && value->as_Local() == NULL) { - walk(value); - assert(value->operand()->is_valid(), "must be evaluated now"); + assert(state != NULL, "state must be defined"); + + ValueStack* s = state; + for_each_state(s) { + if (s->kind() == ValueStack::EmptyExceptionState) { + assert(s->stack_size() == 0 && s->locals_size() == 0 && (s->locks_size() == 0 || s->locks_size() == 1), "state must be empty"); + continue; } - } - ValueStack* s = state; - int bci = x->bci(); - for_each_state(s) { + + int index; + Value value; + for_each_stack_value(s, index, value) { + assert(value->subst() == value, "missed substitution"); + if (!value->is_pinned() && value->as_Constant() == NULL && value->as_Local() == NULL) { + walk(value); + assert(value->operand()->is_valid(), "must be evaluated now"); + } + } + + int bci = s->bci(); IRScope* scope = s->scope(); ciMethod* method = scope->method(); @@ -422,15 +436,14 @@ } } } - bci = scope->caller_bci(); } - return new CodeEmitInfo(x->bci(), state, ignore_xhandler ? NULL : x->exception_handlers()); + return new CodeEmitInfo(state, ignore_xhandler ? NULL : x->exception_handlers()); } CodeEmitInfo* LIRGenerator::state_for(Instruction* x) { - return state_for(x, x->lock_stack()); + return state_for(x, x->exception_state()); } @@ -474,16 +487,6 @@ } -// increment a counter returning the incremented value -LIR_Opr LIRGenerator::increment_and_return_counter(LIR_Opr base, int offset, int increment) { - LIR_Address* counter = new LIR_Address(base, offset, T_INT); - LIR_Opr result = new_register(T_INT); - __ load(counter, result); - __ add(result, LIR_OprFact::intConst(increment), result); - __ store(result, counter); - return result; -} - void LIRGenerator::arithmetic_op(Bytecodes::Code code, LIR_Opr result, LIR_Opr left, LIR_Opr right, bool is_strictfp, LIR_Opr tmp_op, CodeEmitInfo* info) { LIR_Opr result_op = result; @@ -628,14 +631,14 @@ } -void LIRGenerator::monitor_exit(LIR_Opr object, LIR_Opr lock, LIR_Opr new_hdr, int monitor_no) { +void LIRGenerator::monitor_exit(LIR_Opr object, LIR_Opr lock, LIR_Opr new_hdr, LIR_Opr scratch, int monitor_no) { if (!GenerateSynchronizationCode) return; // setup registers LIR_Opr hdr = lock; lock = new_hdr; CodeStub* slow_path = new MonitorExitStub(lock, UseFastLocking, monitor_no); __ load_stack_address_monitor(monitor_no, lock); - __ unlock_object(hdr, object, lock, slow_path); + __ unlock_object(hdr, object, lock, scratch, slow_path); } @@ -815,7 +818,6 @@ return tmp; } - void LIRGenerator::profile_branch(If* if_instr, If::Condition cond) { if (if_instr->should_profile()) { ciMethod* method = if_instr->profiled_method(); @@ -830,24 +832,32 @@ assert(data->is_BranchData(), "need BranchData for two-way branches"); int taken_count_offset = md->byte_offset_of_slot(data, BranchData::taken_offset()); int not_taken_count_offset = md->byte_offset_of_slot(data, BranchData::not_taken_offset()); + if (if_instr->is_swapped()) { + int t = taken_count_offset; + taken_count_offset = not_taken_count_offset; + not_taken_count_offset = t; + } + LIR_Opr md_reg = new_register(T_OBJECT); - __ move(LIR_OprFact::oopConst(md->constant_encoding()), md_reg); - LIR_Opr data_offset_reg = new_register(T_INT); + __ oop2reg(md->constant_encoding(), md_reg); + + LIR_Opr data_offset_reg = new_pointer_register(); __ cmove(lir_cond(cond), - LIR_OprFact::intConst(taken_count_offset), - LIR_OprFact::intConst(not_taken_count_offset), + LIR_OprFact::intptrConst(taken_count_offset), + LIR_OprFact::intptrConst(not_taken_count_offset), data_offset_reg); - LIR_Opr data_reg = new_register(T_INT); - LIR_Address* data_addr = new LIR_Address(md_reg, data_offset_reg, T_INT); + + // MDO cells are intptr_t, so the data_reg width is arch-dependent. + LIR_Opr data_reg = new_pointer_register(); + LIR_Address* data_addr = new LIR_Address(md_reg, data_offset_reg, data_reg->type()); __ move(LIR_OprFact::address(data_addr), data_reg); + // Use leal instead of add to avoid destroying condition codes on x86 LIR_Address* fake_incr_value = new LIR_Address(data_reg, DataLayout::counter_increment, T_INT); - // Use leal instead of add to avoid destroying condition codes on x86 __ leal(LIR_OprFact::address(fake_incr_value), data_reg); __ move(data_reg, LIR_OprFact::address(data_addr)); } } - // Phi technique: // This is about passing live values from one basic block to the other. // In code generated with Java it is rather rare that more than one @@ -897,18 +907,14 @@ Value sux_value; int index; + assert(cur_state->scope() == sux_state->scope(), "not matching"); + assert(cur_state->locals_size() == sux_state->locals_size(), "not matching"); + assert(cur_state->stack_size() == sux_state->stack_size(), "not matching"); + for_each_stack_value(sux_state, index, sux_value) { move_to_phi(&resolver, cur_state->stack_at(index), sux_value); } - // Inlining may cause the local state not to match up, so walk up - // the caller state until we get to the same scope as the - // successor and then start processing from there. - while (cur_state->scope() != sux_state->scope()) { - cur_state = cur_state->caller_state(); - assert(cur_state != NULL, "scopes don't match up"); - } - for_each_local_value(sux_state, index, sux_value) { move_to_phi(&resolver, cur_state->local_at(index), sux_value); } @@ -933,7 +939,6 @@ } } _virtual_register_number += 1; - if (type == T_ADDRESS) type = T_INT; return LIR_OprFact::virtual_register(vreg, type); } @@ -1021,10 +1026,10 @@ // Code for a constant is generated lazily unless the constant is frequently used and can't be inlined. void LIRGenerator::do_Constant(Constant* x) { - if (x->state() != NULL) { + if (x->state_before() != NULL) { // Any constant with a ValueStack requires patching so emit the patch here LIR_Opr reg = rlock_result(x); - CodeEmitInfo* info = state_for(x, x->state()); + CodeEmitInfo* info = state_for(x, x->state_before()); __ oop2reg_patch(NULL, reg, info); } else if (x->use_count() > 1 && !can_inline_as_constant(x)) { if (!x->is_pinned()) { @@ -1100,7 +1105,7 @@ // need to perform the null check on the rcvr CodeEmitInfo* info = NULL; if (x->needs_null_check()) { - info = state_for(x, x->state()->copy_locks()); + info = state_for(x); } __ move(new LIR_Address(rcvr.result(), oopDesc::klass_offset_in_bytes(), T_OBJECT), result, info); __ move(new LIR_Address(result, Klass::java_mirror_offset_in_bytes() + @@ -1299,8 +1304,6 @@ LIR_Opr flag_val = new_register(T_INT); __ load(mark_active_flag_addr, flag_val); - LabelObj* start_store = new LabelObj(); - LIR_PatchCode pre_val_patch_code = patch ? lir_patch_normal : lir_patch_none; @@ -1309,7 +1312,7 @@ __ cmp(lir_cond_notEqual, flag_val, LIR_OprFact::intConst(0)); if (!addr_opr->is_address()) { assert(addr_opr->is_register(), "must be"); - addr_opr = LIR_OprFact::address(new LIR_Address(addr_opr, 0, T_OBJECT)); + addr_opr = LIR_OprFact::address(new LIR_Address(addr_opr, T_OBJECT)); } CodeStub* slow = new G1PreBarrierStub(addr_opr, pre_val, pre_val_patch_code, info); @@ -1325,7 +1328,7 @@ new_val->as_constant_ptr()->as_jobject() == NULL) return; if (!new_val->is_register()) { - LIR_Opr new_val_reg = new_pointer_register(); + LIR_Opr new_val_reg = new_register(T_OBJECT); if (new_val->is_constant()) { __ move(new_val, new_val_reg); } else { @@ -1337,7 +1340,7 @@ if (addr->is_address()) { LIR_Address* address = addr->as_address_ptr(); - LIR_Opr ptr = new_pointer_register(); + LIR_Opr ptr = new_register(T_OBJECT); if (!address->index()->is_valid() && address->disp() == 0) { __ move(address->base(), ptr); } else { @@ -1350,7 +1353,6 @@ LIR_Opr xor_res = new_pointer_register(); LIR_Opr xor_shift_res = new_pointer_register(); - if (TwoOperandLIRForm ) { __ move(addr, xor_res); __ logical_xor(xor_res, new_val, xor_res); @@ -1368,7 +1370,7 @@ } if (!new_val->is_register()) { - LIR_Opr new_val_reg = new_pointer_register(); + LIR_Opr new_val_reg = new_register(T_OBJECT); __ leal(new_val, new_val_reg); new_val = new_val_reg; } @@ -1377,7 +1379,7 @@ __ cmp(lir_cond_notEqual, xor_shift_res, LIR_OprFact::intptrConst(NULL_WORD)); CodeStub* slow = new G1PostBarrierStub(addr, new_val); - __ branch(lir_cond_notEqual, T_INT, slow); + __ branch(lir_cond_notEqual, LP64_ONLY(T_LONG) NOT_LP64(T_INT), slow); __ branch_destination(slow->continuation()); } @@ -1401,6 +1403,25 @@ } assert(addr->is_register(), "must be a register at this point"); +#ifdef ARM + // TODO: ARM - move to platform-dependent code + LIR_Opr tmp = FrameMap::R14_opr; + if (VM_Version::supports_movw()) { + __ move((LIR_Opr)card_table_base, tmp); + } else { + __ move(new LIR_Address(FrameMap::Rthread_opr, in_bytes(JavaThread::card_table_base_offset()), T_ADDRESS), tmp); + } + + CardTableModRefBS* ct = (CardTableModRefBS*)_bs; + LIR_Address *card_addr = new LIR_Address(tmp, addr, (LIR_Address::Scale) -CardTableModRefBS::card_shift, 0, T_BYTE); + if(((int)ct->byte_map_base & 0xff) == 0) { + __ move(tmp, card_addr); + } else { + LIR_Opr tmp_zero = new_register(T_INT); + __ move(LIR_OprFact::intConst(0), tmp_zero); + __ move(tmp_zero, card_addr); + } +#else // ARM LIR_Opr tmp = new_pointer_register(); if (TwoOperandLIRForm) { __ move(addr, tmp); @@ -1416,6 +1437,7 @@ new LIR_Address(tmp, load_constant(card_table_base), T_BYTE)); } +#endif // ARM } @@ -1462,7 +1484,7 @@ } else if (x->needs_null_check()) { NullCheck* nc = x->explicit_null_check(); if (nc == NULL) { - info = state_for(x, x->lock_stack()); + info = state_for(x); } else { info = state_for(nc); } @@ -1490,10 +1512,12 @@ set_no_result(x); +#ifndef PRODUCT if (PrintNotLoaded && needs_patching) { tty->print_cr(" ###class not loaded at store_%s bci %d", - x->is_static() ? "static" : "field", x->bci()); + x->is_static() ? "static" : "field", x->printable_bci()); } +#endif if (x->needs_null_check() && (needs_patching || @@ -1508,7 +1532,7 @@ // generate_address to try to be smart about emitting the -1. // Otherwise the patching code won't know how to find the // instruction to patch. - address = new LIR_Address(object.result(), max_jint, field_type); + address = new LIR_Address(object.result(), PATCHED_ADDR, field_type); } else { address = generate_address(object.result(), x->offset(), field_type); } @@ -1556,7 +1580,7 @@ } else if (x->needs_null_check()) { NullCheck* nc = x->explicit_null_check(); if (nc == NULL) { - info = state_for(x, x->lock_stack()); + info = state_for(x); } else { info = state_for(nc); } @@ -1566,10 +1590,12 @@ object.load_item(); +#ifndef PRODUCT if (PrintNotLoaded && needs_patching) { tty->print_cr(" ###class not loaded at load_%s bci %d", - x->is_static() ? "static" : "field", x->bci()); + x->is_static() ? "static" : "field", x->printable_bci()); } +#endif if (x->needs_null_check() && (needs_patching || @@ -1585,7 +1611,7 @@ // generate_address to try to be smart about emitting the -1. // Otherwise the patching code won't know how to find the // instruction to patch. - address = new LIR_Address(object.result(), max_jint, field_type); + address = new LIR_Address(object.result(), PATCHED_ADDR, field_type); } else { address = generate_address(object.result(), x->offset(), field_type); } @@ -1732,7 +1758,7 @@ #ifndef PRODUCT if (PrintC1Statistics) { - increment_counter(Runtime1::throw_count_address()); + increment_counter(Runtime1::throw_count_address(), T_INT); } #endif @@ -1762,7 +1788,7 @@ if (GenerateCompilerNullChecks && (x->exception()->as_NewInstance() == NULL && x->exception()->as_ExceptionObject() == NULL)) { // if the exception object wasn't created using new then it might be null. - __ null_check(exception_opr, new CodeEmitInfo(info, true)); + __ null_check(exception_opr, new CodeEmitInfo(info, x->state()->copy(ValueStack::ExceptionState, x->state()->bci()))); } if (compilation()->env()->jvmti_can_post_on_exceptions()) { @@ -1845,6 +1871,8 @@ } #endif addr = new LIR_Address(base_op, index_op, LIR_Address::Scale(log2_scale), 0, dst_type); +#elif defined(ARM) + addr = generate_address(base_op, index_op, log2_scale, 0, dst_type); #else if (index_op->is_illegal() || log2_scale == 0) { #ifdef _LP64 @@ -1917,6 +1945,7 @@ __ convert(Bytecodes::_i2l, idx.result(), index_op); } else { #endif + // TODO: ARM also allows embedded shift in the address __ move(idx.result(), index_op); #ifdef _LP64 } @@ -2105,7 +2134,6 @@ int lo_key = x->lo_key(); int hi_key = x->hi_key(); int len = x->length(); - CodeEmitInfo* info = state_for(x, x->state()); LIR_Opr value = tag.result(); if (UseTableRanges) { do_SwitchRanges(create_lookup_ranges(x), value, x->default_sux()); @@ -2163,12 +2191,41 @@ ValueStack* state = x->state_before() ? x->state_before() : x->state(); // increment backedge counter if needed - increment_backedge_counter(state_for(x, state)); - + CodeEmitInfo* info = state_for(x, state); + increment_backedge_counter(info, info->stack()->bci()); CodeEmitInfo* safepoint_info = state_for(x, state); __ safepoint(safepoint_poll_register(), safepoint_info); } + // Gotos can be folded Ifs, handle this case. + if (x->should_profile()) { + ciMethod* method = x->profiled_method(); + assert(method != NULL, "method should be set if branch is profiled"); + ciMethodData* md = method->method_data(); + if (md == NULL) { + bailout("out of memory building methodDataOop"); + return; + } + ciProfileData* data = md->bci_to_data(x->profiled_bci()); + assert(data != NULL, "must have profiling data"); + int offset; + if (x->direction() == Goto::taken) { + assert(data->is_BranchData(), "need BranchData for two-way branches"); + offset = md->byte_offset_of_slot(data, BranchData::taken_offset()); + } else if (x->direction() == Goto::not_taken) { + assert(data->is_BranchData(), "need BranchData for two-way branches"); + offset = md->byte_offset_of_slot(data, BranchData::not_taken_offset()); + } else { + assert(data->is_JumpData(), "need JumpData for branches"); + offset = md->byte_offset_of_slot(data, JumpData::taken_offset()); + } + LIR_Opr md_reg = new_register(T_OBJECT); + __ oop2reg(md->constant_encoding(), md_reg); + + increment_counter(new LIR_Address(md_reg, offset, + NOT_LP64(T_INT) LP64_ONLY(T_LONG)), DataLayout::counter_increment); + } + // emit phi-instruction move after safepoint since this simplifies // describing the state as the safepoint. move_to_phi(x->state()); @@ -2205,7 +2262,10 @@ // Assign new location to Local instruction for this local Local* local = x->state()->local_at(java_index)->as_Local(); assert(local != NULL, "Locals for incoming arguments must have been created"); +#ifndef __SOFTFP__ + // The java calling convention passes double as long and float as int. assert(as_ValueType(t)->tag() == local->type()->tag(), "check"); +#endif // __SOFTFP__ local->set_operand(dest); _instruction_for_operand.at_put_grow(dest->vreg_number(), local, NULL); java_index += type2size[t]; @@ -2239,7 +2299,7 @@ LIR_Opr lock = new_register(T_INT); __ load_stack_address_monitor(0, lock); - CodeEmitInfo* info = new CodeEmitInfo(SynchronizationEntryBCI, scope()->start()->state(), NULL); + CodeEmitInfo* info = new CodeEmitInfo(scope()->start()->state()->copy(ValueStack::StateBefore, SynchronizationEntryBCI), NULL); CodeStub* slow_path = new MonitorEnterStub(obj, lock, info); // receiver is guaranteed non-NULL so don't need CodeEmitInfo @@ -2248,7 +2308,10 @@ } // increment invocation counters if needed - increment_invocation_counter(new CodeEmitInfo(0, scope()->start()->state(), NULL)); + if (!method()->is_accessor()) { // Accessors do not have MDOs, so no counting. + CodeEmitInfo* info = new CodeEmitInfo(scope()->start()->state()->copy(ValueStack::StateBefore, SynchronizationEntryBCI), NULL); + increment_invocation_counter(info); + } // all blocks with a successor must end with an unconditional jump // to the successor even if they are consecutive @@ -2371,9 +2434,17 @@ bool optimized = x->target_is_loaded() && x->target_is_final(); assert(receiver->is_illegal() || receiver->is_equal(LIR_Assembler::receiverOpr()), "must match"); + // JSR 292 + // Preserve the SP over MethodHandle call sites. + ciMethod* target = x->target(); + if (target->is_method_handle_invoke()) { + info->set_is_method_handle_invoke(true); + __ move(FrameMap::stack_pointer(), FrameMap::method_handle_invoke_SP_save_opr()); + } + switch (x->code()) { case Bytecodes::_invokestatic: - __ call_static(x->target(), result_register, + __ call_static(target, result_register, SharedRuntime::get_resolve_static_call_stub(), arg_list, info); break; @@ -2383,22 +2454,22 @@ // for final target we still produce an inline cache, in order // to be able to call mixed mode if (x->code() == Bytecodes::_invokespecial || optimized) { - __ call_opt_virtual(x->target(), receiver, result_register, + __ call_opt_virtual(target, receiver, result_register, SharedRuntime::get_resolve_opt_virtual_call_stub(), arg_list, info); } else if (x->vtable_index() < 0) { - __ call_icvirtual(x->target(), receiver, result_register, + __ call_icvirtual(target, receiver, result_register, SharedRuntime::get_resolve_virtual_call_stub(), arg_list, info); } else { int entry_offset = instanceKlass::vtable_start_offset() + x->vtable_index() * vtableEntry::size(); int vtable_offset = entry_offset * wordSize + vtableEntry::method_offset_in_bytes(); - __ call_virtual(x->target(), receiver, result_register, vtable_offset, arg_list, info); + __ call_virtual(target, receiver, result_register, vtable_offset, arg_list, info); } break; case Bytecodes::_invokedynamic: { ciBytecodeStream bcs(x->scope()->method()); - bcs.force_bci(x->bci()); + bcs.force_bci(x->state()->bci()); assert(bcs.cur_bc() == Bytecodes::_invokedynamic, "wrong stream"); ciCPCache* cpcache = bcs.get_cpcache(); @@ -2432,7 +2503,7 @@ // Load target MethodHandle from CallSite object. __ load(new LIR_Address(tmp, java_dyn_CallSite::target_offset_in_bytes(), T_OBJECT), receiver); - __ call_dynamic(x->target(), receiver, result_register, + __ call_dynamic(target, receiver, result_register, SharedRuntime::get_resolve_opt_virtual_call_stub(), arg_list, info); break; @@ -2442,6 +2513,12 @@ break; } + // JSR 292 + // Restore the SP after MethodHandle call sites. + if (target->is_method_handle_invoke()) { + __ move(FrameMap::method_handle_invoke_SP_save_opr(), FrameMap::stack_pointer()); + } + if (x->type()->is_float() || x->type()->is_double()) { // Force rounding of results from non-strictfp when in strictfp // scope (or when we don't know the strictness of the callee, to @@ -2568,12 +2645,12 @@ } } - void LIRGenerator::do_ProfileCall(ProfileCall* x) { // Need recv in a temporary register so it interferes with the other temporaries LIR_Opr recv = LIR_OprFact::illegalOpr; LIR_Opr mdo = new_register(T_OBJECT); - LIR_Opr tmp = new_register(T_INT); + // tmp is used to hold the counters on SPARC + LIR_Opr tmp = new_pointer_register(); if (x->recv() != NULL) { LIRItem value(x->recv(), this); value.load_item(); @@ -2583,14 +2660,69 @@ __ profile_call(x->method(), x->bci_of_invoke(), mdo, recv, tmp, x->known_holder()); } - -void LIRGenerator::do_ProfileCounter(ProfileCounter* x) { - LIRItem mdo(x->mdo(), this); - mdo.load_item(); - - increment_counter(new LIR_Address(mdo.result(), x->offset(), T_INT), x->increment()); +void LIRGenerator::do_ProfileInvoke(ProfileInvoke* x) { + // We can safely ignore accessors here, since c2 will inline them anyway, + // accessors are also always mature. + if (!x->inlinee()->is_accessor()) { + CodeEmitInfo* info = state_for(x, x->state(), true); + // Increment invocation counter, don't notify the runtime, because we don't inline loops, + increment_event_counter_impl(info, x->inlinee(), 0, InvocationEntryBci, false, false); + } +} + +void LIRGenerator::increment_event_counter(CodeEmitInfo* info, int bci, bool backedge) { + int freq_log; + int level = compilation()->env()->comp_level(); + if (level == CompLevel_limited_profile) { + freq_log = (backedge ? Tier2BackedgeNotifyFreqLog : Tier2InvokeNotifyFreqLog); + } else if (level == CompLevel_full_profile) { + freq_log = (backedge ? Tier3BackedgeNotifyFreqLog : Tier3InvokeNotifyFreqLog); + } else { + ShouldNotReachHere(); + } + // Increment the appropriate invocation/backedge counter and notify the runtime. + increment_event_counter_impl(info, info->scope()->method(), (1 << freq_log) - 1, bci, backedge, true); } +void LIRGenerator::increment_event_counter_impl(CodeEmitInfo* info, + ciMethod *method, int frequency, + int bci, bool backedge, bool notify) { + assert(frequency == 0 || is_power_of_2(frequency + 1), "Frequency must be x^2 - 1 or 0"); + int level = _compilation->env()->comp_level(); + assert(level > CompLevel_simple, "Shouldn't be here"); + + int offset = -1; + LIR_Opr counter_holder = new_register(T_OBJECT); + LIR_Opr meth; + if (level == CompLevel_limited_profile) { + offset = in_bytes(backedge ? methodOopDesc::backedge_counter_offset() : + methodOopDesc::invocation_counter_offset()); + __ oop2reg(method->constant_encoding(), counter_holder); + meth = counter_holder; + } else if (level == CompLevel_full_profile) { + offset = in_bytes(backedge ? methodDataOopDesc::backedge_counter_offset() : + methodDataOopDesc::invocation_counter_offset()); + __ oop2reg(method->method_data()->constant_encoding(), counter_holder); + meth = new_register(T_OBJECT); + __ oop2reg(method->constant_encoding(), meth); + } else { + ShouldNotReachHere(); + } + LIR_Address* counter = new LIR_Address(counter_holder, offset, T_INT); + LIR_Opr result = new_register(T_INT); + __ load(counter, result); + __ add(result, LIR_OprFact::intConst(InvocationCounter::count_increment), result); + __ store(result, counter); + if (notify) { + LIR_Opr mask = load_immediate(frequency << InvocationCounter::count_shift, T_INT); + __ logical_and(result, mask, result); + __ cmp(lir_cond_equal, result, LIR_OprFact::intConst(0)); + // The bci for info can point to cmp for if's we want the if bci + CodeStub* overflow = new CounterOverflowStub(info, bci, meth); + __ branch(lir_cond_equal, T_INT, overflow); + __ branch_destination(overflow->continuation()); + } +} LIR_Opr LIRGenerator::call_runtime(Value arg1, address entry, ValueType* result_type, CodeEmitInfo* info) { LIRItemList args(1); @@ -2702,29 +2834,3 @@ } return result; } - - - -void LIRGenerator::increment_invocation_counter(CodeEmitInfo* info, bool backedge) { -#ifdef TIERED - if (_compilation->env()->comp_level() == CompLevel_fast_compile && - (method()->code_size() >= Tier1BytecodeLimit || backedge)) { - int limit = InvocationCounter::Tier1InvocationLimit; - int offset = in_bytes(methodOopDesc::invocation_counter_offset() + - InvocationCounter::counter_offset()); - if (backedge) { - limit = InvocationCounter::Tier1BackEdgeLimit; - offset = in_bytes(methodOopDesc::backedge_counter_offset() + - InvocationCounter::counter_offset()); - } - - LIR_Opr meth = new_register(T_OBJECT); - __ oop2reg(method()->constant_encoding(), meth); - LIR_Opr result = increment_and_return_counter(meth, offset, InvocationCounter::count_increment); - __ cmp(lir_cond_aboveEqual, result, LIR_OprFact::intConst(limit)); - CodeStub* overflow = new CounterOverflowStub(info, info->bci()); - __ branch(lir_cond_aboveEqual, T_INT, overflow); - __ branch_destination(overflow->continuation()); - } -#endif -}