Mercurial > hg > graal-jvmci-8
diff src/cpu/x86/vm/macroAssembler_x86.cpp @ 17780:606acabe7b5c
8031320: Use Intel RTM instructions for locks
Summary: Use RTM for inflated locks and stack locks.
Reviewed-by: iveresov, twisti, roland, dcubed
author | kvn |
---|---|
date | Thu, 20 Mar 2014 17:49:27 -0700 |
parents | 4d4ea046d32a |
children | 526acaf3626f |
line wrap: on
line diff
--- a/src/cpu/x86/vm/macroAssembler_x86.cpp Sat Mar 22 00:26:48 2014 +0400 +++ b/src/cpu/x86/vm/macroAssembler_x86.cpp Thu Mar 20 17:49:27 2014 -0700 @@ -301,7 +301,9 @@ mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate()); } -void MacroAssembler::movptr(Register dst, AddressLiteral src) { +void MacroAssembler::movptr(Register dst, AddressLiteral src, Register scratch) { + // scratch register is not used, + // it is defined to match parameters of 64-bit version of this method. if (src.is_lval()) { mov_literal32(dst, (intptr_t)src.target(), src.rspec()); } else { @@ -613,6 +615,15 @@ /* else */ { subq(dst, value) ; return; } } +void MacroAssembler::incrementq(AddressLiteral dst) { + if (reachable(dst)) { + incrementq(as_Address(dst)); + } else { + lea(rscratch1, dst); + incrementq(Address(rscratch1, 0)); + } +} + void MacroAssembler::incrementq(Register reg, int value) { if (value == min_jint) { addq(reg, value); return; } if (value < 0) { decrementq(reg, -value); return; } @@ -681,15 +692,15 @@ movq(dst, rscratch1); } -void MacroAssembler::movptr(Register dst, AddressLiteral src) { +void MacroAssembler::movptr(Register dst, AddressLiteral src, Register scratch) { if (src.is_lval()) { mov_literal64(dst, (intptr_t)src.target(), src.rspec()); } else { if (reachable(src)) { movq(dst, as_Address(src)); } else { - lea(rscratch1, src); - movq(dst, Address(rscratch1,0)); + lea(scratch, src); + movq(dst, Address(scratch, 0)); } } } @@ -988,20 +999,37 @@ LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32)); } -void MacroAssembler::atomic_incl(AddressLiteral counter_addr) { - pushf(); +void MacroAssembler::atomic_incl(Address counter_addr) { + if (os::is_MP()) + lock(); + incrementl(counter_addr); +} + +void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register scr) { if (reachable(counter_addr)) { - if (os::is_MP()) - lock(); - incrementl(as_Address(counter_addr)); + atomic_incl(as_Address(counter_addr)); } else { - lea(rscratch1, counter_addr); - if (os::is_MP()) - lock(); - incrementl(Address(rscratch1, 0)); - } - popf(); -} + lea(scr, counter_addr); + atomic_incl(Address(scr, 0)); + } +} + +#ifdef _LP64 +void MacroAssembler::atomic_incq(Address counter_addr) { + if (os::is_MP()) + lock(); + incrementq(counter_addr); +} + +void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register scr) { + if (reachable(counter_addr)) { + atomic_incq(as_Address(counter_addr)); + } else { + lea(scr, counter_addr); + atomic_incq(Address(scr, 0)); + } +} +#endif // Writes to stack successive pages until offset reached to check for // stack overflow + shadow pages. This clobbers tmp. @@ -1274,6 +1302,325 @@ } #ifdef COMPILER2 + +#if INCLUDE_RTM_OPT + +// Update rtm_counters based on abort status +// input: abort_status +// rtm_counters (RTMLockingCounters*) +// flags are killed +void MacroAssembler::rtm_counters_update(Register abort_status, Register rtm_counters) { + + atomic_incptr(Address(rtm_counters, RTMLockingCounters::abort_count_offset())); + if (PrintPreciseRTMLockingStatistics) { + for (int i = 0; i < RTMLockingCounters::ABORT_STATUS_LIMIT; i++) { + Label check_abort; + testl(abort_status, (1<<i)); + jccb(Assembler::equal, check_abort); + atomic_incptr(Address(rtm_counters, RTMLockingCounters::abortX_count_offset() + (i * sizeof(uintx)))); + bind(check_abort); + } + } +} + +// Branch if (random & (count-1) != 0), count is 2^n +// tmp, scr and flags are killed +void MacroAssembler::branch_on_random_using_rdtsc(Register tmp, Register scr, int count, Label& brLabel) { + assert(tmp == rax, ""); + assert(scr == rdx, ""); + rdtsc(); // modifies EDX:EAX + andptr(tmp, count-1); + jccb(Assembler::notZero, brLabel); +} + +// Perform abort ratio calculation, set no_rtm bit if high ratio +// input: rtm_counters_Reg (RTMLockingCounters* address) +// tmpReg, rtm_counters_Reg and flags are killed +void MacroAssembler::rtm_abort_ratio_calculation(Register tmpReg, + Register rtm_counters_Reg, + RTMLockingCounters* rtm_counters, + Metadata* method_data) { + Label L_done, L_check_always_rtm1, L_check_always_rtm2; + + if (RTMLockingCalculationDelay > 0) { + // Delay calculation + movptr(tmpReg, ExternalAddress((address) RTMLockingCounters::rtm_calculation_flag_addr()), tmpReg); + testptr(tmpReg, tmpReg); + jccb(Assembler::equal, L_done); + } + // Abort ratio calculation only if abort_count > RTMAbortThreshold + // Aborted transactions = abort_count * 100 + // All transactions = total_count * RTMTotalCountIncrRate + // Set no_rtm bit if (Aborted transactions >= All transactions * RTMAbortRatio) + + movptr(tmpReg, Address(rtm_counters_Reg, RTMLockingCounters::abort_count_offset())); + cmpptr(tmpReg, RTMAbortThreshold); + jccb(Assembler::below, L_check_always_rtm2); + imulptr(tmpReg, tmpReg, 100); + + Register scrReg = rtm_counters_Reg; + movptr(scrReg, Address(rtm_counters_Reg, RTMLockingCounters::total_count_offset())); + imulptr(scrReg, scrReg, RTMTotalCountIncrRate); + imulptr(scrReg, scrReg, RTMAbortRatio); + cmpptr(tmpReg, scrReg); + jccb(Assembler::below, L_check_always_rtm1); + if (method_data != NULL) { + // set rtm_state to "no rtm" in MDO + mov_metadata(tmpReg, method_data); + if (os::is_MP()) { + lock(); + } + orl(Address(tmpReg, MethodData::rtm_state_offset_in_bytes()), NoRTM); + } + jmpb(L_done); + bind(L_check_always_rtm1); + // Reload RTMLockingCounters* address + lea(rtm_counters_Reg, ExternalAddress((address)rtm_counters)); + bind(L_check_always_rtm2); + movptr(tmpReg, Address(rtm_counters_Reg, RTMLockingCounters::total_count_offset())); + cmpptr(tmpReg, RTMLockingThreshold / RTMTotalCountIncrRate); + jccb(Assembler::below, L_done); + if (method_data != NULL) { + // set rtm_state to "always rtm" in MDO + mov_metadata(tmpReg, method_data); + if (os::is_MP()) { + lock(); + } + orl(Address(tmpReg, MethodData::rtm_state_offset_in_bytes()), UseRTM); + } + bind(L_done); +} + +// Update counters and perform abort ratio calculation +// input: abort_status_Reg +// rtm_counters_Reg, flags are killed +void MacroAssembler::rtm_profiling(Register abort_status_Reg, + Register rtm_counters_Reg, + RTMLockingCounters* rtm_counters, + Metadata* method_data, + bool profile_rtm) { + + assert(rtm_counters != NULL, "should not be NULL when profiling RTM"); + // update rtm counters based on rax value at abort + // reads abort_status_Reg, updates flags + lea(rtm_counters_Reg, ExternalAddress((address)rtm_counters)); + rtm_counters_update(abort_status_Reg, rtm_counters_Reg); + if (profile_rtm) { + // Save abort status because abort_status_Reg is used by following code. + if (RTMRetryCount > 0) { + push(abort_status_Reg); + } + assert(rtm_counters != NULL, "should not be NULL when profiling RTM"); + rtm_abort_ratio_calculation(abort_status_Reg, rtm_counters_Reg, rtm_counters, method_data); + // restore abort status + if (RTMRetryCount > 0) { + pop(abort_status_Reg); + } + } +} + +// Retry on abort if abort's status is 0x6: can retry (0x2) | memory conflict (0x4) +// inputs: retry_count_Reg +// : abort_status_Reg +// output: retry_count_Reg decremented by 1 +// flags are killed +void MacroAssembler::rtm_retry_lock_on_abort(Register retry_count_Reg, Register abort_status_Reg, Label& retryLabel) { + Label doneRetry; + assert(abort_status_Reg == rax, ""); + // The abort reason bits are in eax (see all states in rtmLocking.hpp) + // 0x6 = conflict on which we can retry (0x2) | memory conflict (0x4) + // if reason is in 0x6 and retry count != 0 then retry + andptr(abort_status_Reg, 0x6); + jccb(Assembler::zero, doneRetry); + testl(retry_count_Reg, retry_count_Reg); + jccb(Assembler::zero, doneRetry); + pause(); + decrementl(retry_count_Reg); + jmp(retryLabel); + bind(doneRetry); +} + +// Spin and retry if lock is busy, +// inputs: box_Reg (monitor address) +// : retry_count_Reg +// output: retry_count_Reg decremented by 1 +// : clear z flag if retry count exceeded +// tmp_Reg, scr_Reg, flags are killed +void MacroAssembler::rtm_retry_lock_on_busy(Register retry_count_Reg, Register box_Reg, + Register tmp_Reg, Register scr_Reg, Label& retryLabel) { + Label SpinLoop, SpinExit, doneRetry; + // Clean monitor_value bit to get valid pointer + int owner_offset = ObjectMonitor::owner_offset_in_bytes() - markOopDesc::monitor_value; + + testl(retry_count_Reg, retry_count_Reg); + jccb(Assembler::zero, doneRetry); + decrementl(retry_count_Reg); + movptr(scr_Reg, RTMSpinLoopCount); + + bind(SpinLoop); + pause(); + decrementl(scr_Reg); + jccb(Assembler::lessEqual, SpinExit); + movptr(tmp_Reg, Address(box_Reg, owner_offset)); + testptr(tmp_Reg, tmp_Reg); + jccb(Assembler::notZero, SpinLoop); + + bind(SpinExit); + jmp(retryLabel); + bind(doneRetry); + incrementl(retry_count_Reg); // clear z flag +} + +// Use RTM for normal stack locks +// Input: objReg (object to lock) +void MacroAssembler::rtm_stack_locking(Register objReg, Register tmpReg, Register scrReg, + Register retry_on_abort_count_Reg, + RTMLockingCounters* stack_rtm_counters, + Metadata* method_data, bool profile_rtm, + Label& DONE_LABEL, Label& IsInflated) { + assert(UseRTMForStackLocks, "why call this otherwise?"); + assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking"); + assert(tmpReg == rax, ""); + assert(scrReg == rdx, ""); + Label L_rtm_retry, L_decrement_retry, L_on_abort; + + if (RTMRetryCount > 0) { + movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort + bind(L_rtm_retry); + } + if (!UseRTMXendForLockBusy) { + movptr(tmpReg, Address(objReg, 0)); + testptr(tmpReg, markOopDesc::monitor_value); // inflated vs stack-locked|neutral|biased + jcc(Assembler::notZero, IsInflated); + } + if (PrintPreciseRTMLockingStatistics || profile_rtm) { + Label L_noincrement; + if (RTMTotalCountIncrRate > 1) { + // tmpReg, scrReg and flags are killed + branch_on_random_using_rdtsc(tmpReg, scrReg, (int)RTMTotalCountIncrRate, L_noincrement); + } + assert(stack_rtm_counters != NULL, "should not be NULL when profiling RTM"); + atomic_incptr(ExternalAddress((address)stack_rtm_counters->total_count_addr()), scrReg); + bind(L_noincrement); + } + xbegin(L_on_abort); + movptr(tmpReg, Address(objReg, 0)); // fetch markword + andptr(tmpReg, markOopDesc::biased_lock_mask_in_place); // look at 3 lock bits + cmpptr(tmpReg, markOopDesc::unlocked_value); // bits = 001 unlocked + jcc(Assembler::equal, DONE_LABEL); // all done if unlocked + + Register abort_status_Reg = tmpReg; // status of abort is stored in RAX + if (UseRTMXendForLockBusy) { + xend(); + movptr(tmpReg, Address(objReg, 0)); + testptr(tmpReg, markOopDesc::monitor_value); // inflated vs stack-locked|neutral|biased + jcc(Assembler::notZero, IsInflated); + movptr(abort_status_Reg, 0x1); // Set the abort status to 1 (as xabort does) + jmp(L_decrement_retry); + } + else { + xabort(0); + } + bind(L_on_abort); + if (PrintPreciseRTMLockingStatistics || profile_rtm) { + rtm_profiling(abort_status_Reg, scrReg, stack_rtm_counters, method_data, profile_rtm); + } + bind(L_decrement_retry); + if (RTMRetryCount > 0) { + // retry on lock abort if abort status is 'can retry' (0x2) or 'memory conflict' (0x4) + rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry); + } +} + +// Use RTM for inflating locks +// inputs: objReg (object to lock) +// boxReg (on-stack box address (displaced header location) - KILLED) +// tmpReg (ObjectMonitor address + 2(monitor_value)) +void MacroAssembler::rtm_inflated_locking(Register objReg, Register boxReg, Register tmpReg, + Register scrReg, Register retry_on_busy_count_Reg, + Register retry_on_abort_count_Reg, + RTMLockingCounters* rtm_counters, + Metadata* method_data, bool profile_rtm, + Label& DONE_LABEL) { + assert(UseRTMLocking, "why call this otherwise?"); + assert(tmpReg == rax, ""); + assert(scrReg == rdx, ""); + Label L_rtm_retry, L_decrement_retry, L_on_abort; + // Clean monitor_value bit to get valid pointer + int owner_offset = ObjectMonitor::owner_offset_in_bytes() - markOopDesc::monitor_value; + + // Without cast to int32_t a movptr will destroy r10 which is typically obj + movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark())); + movptr(boxReg, tmpReg); // Save ObjectMonitor address + + if (RTMRetryCount > 0) { + movl(retry_on_busy_count_Reg, RTMRetryCount); // Retry on lock busy + movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort + bind(L_rtm_retry); + } + if (PrintPreciseRTMLockingStatistics || profile_rtm) { + Label L_noincrement; + if (RTMTotalCountIncrRate > 1) { + // tmpReg, scrReg and flags are killed + branch_on_random_using_rdtsc(tmpReg, scrReg, (int)RTMTotalCountIncrRate, L_noincrement); + } + assert(rtm_counters != NULL, "should not be NULL when profiling RTM"); + atomic_incptr(ExternalAddress((address)rtm_counters->total_count_addr()), scrReg); + bind(L_noincrement); + } + xbegin(L_on_abort); + movptr(tmpReg, Address(objReg, 0)); + movptr(tmpReg, Address(tmpReg, owner_offset)); + testptr(tmpReg, tmpReg); + jcc(Assembler::zero, DONE_LABEL); + if (UseRTMXendForLockBusy) { + xend(); + jmp(L_decrement_retry); + } + else { + xabort(0); + } + bind(L_on_abort); + Register abort_status_Reg = tmpReg; // status of abort is stored in RAX + if (PrintPreciseRTMLockingStatistics || profile_rtm) { + rtm_profiling(abort_status_Reg, scrReg, rtm_counters, method_data, profile_rtm); + } + if (RTMRetryCount > 0) { + // retry on lock abort if abort status is 'can retry' (0x2) or 'memory conflict' (0x4) + rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry); + } + + movptr(tmpReg, Address(boxReg, owner_offset)) ; + testptr(tmpReg, tmpReg) ; + jccb(Assembler::notZero, L_decrement_retry) ; + + // Appears unlocked - try to swing _owner from null to non-null. + // Invariant: tmpReg == 0. tmpReg is EAX which is the implicit cmpxchg comparand. +#ifdef _LP64 + Register threadReg = r15_thread; +#else + get_thread(scrReg); + Register threadReg = scrReg; +#endif + if (os::is_MP()) { + lock(); + } + cmpxchgptr(threadReg, Address(boxReg, owner_offset)); // Updates tmpReg + + if (RTMRetryCount > 0) { + // success done else retry + jccb(Assembler::equal, DONE_LABEL) ; + bind(L_decrement_retry); + // Spin and retry if lock is busy. + rtm_retry_lock_on_busy(retry_on_busy_count_Reg, boxReg, tmpReg, scrReg, L_rtm_retry); + } + else { + bind(L_decrement_retry); + } +} + +#endif // INCLUDE_RTM_OPT + // Fast_Lock and Fast_Unlock used by C2 // Because the transitions from emitted code to the runtime @@ -1350,17 +1697,26 @@ // box: on-stack box address (displaced header location) - KILLED // rax,: tmp -- KILLED // scr: tmp -- KILLED -void MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmpReg, Register scrReg, BiasedLockingCounters* counters) { +void MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmpReg, + Register scrReg, Register cx1Reg, Register cx2Reg, + BiasedLockingCounters* counters, + RTMLockingCounters* rtm_counters, + RTMLockingCounters* stack_rtm_counters, + Metadata* method_data, + bool use_rtm, bool profile_rtm) { // Ensure the register assignents are disjoint - guarantee (objReg != boxReg, ""); - guarantee (objReg != tmpReg, ""); - guarantee (objReg != scrReg, ""); - guarantee (boxReg != tmpReg, ""); - guarantee (boxReg != scrReg, ""); - guarantee (tmpReg == rax, ""); + assert(tmpReg == rax, ""); + + if (use_rtm) { + assert_different_registers(objReg, boxReg, tmpReg, scrReg, cx1Reg, cx2Reg); + } else { + assert(cx1Reg == noreg, ""); + assert(cx2Reg == noreg, ""); + assert_different_registers(objReg, boxReg, tmpReg, scrReg); + } if (counters != NULL) { - atomic_incl(ExternalAddress((address)counters->total_entry_count_addr())); + atomic_incl(ExternalAddress((address)counters->total_entry_count_addr()), scrReg); } if (EmitSync & 1) { // set box->dhw = unused_mark (3) @@ -1419,12 +1775,20 @@ biased_locking_enter(boxReg, objReg, tmpReg, scrReg, true, DONE_LABEL, NULL, counters); } +#if INCLUDE_RTM_OPT + if (UseRTMForStackLocks && use_rtm) { + rtm_stack_locking(objReg, tmpReg, scrReg, cx2Reg, + stack_rtm_counters, method_data, profile_rtm, + DONE_LABEL, IsInflated); + } +#endif // INCLUDE_RTM_OPT + movptr(tmpReg, Address(objReg, 0)); // [FETCH] - testl (tmpReg, markOopDesc::monitor_value); // inflated vs stack-locked|neutral|biased - jccb (Assembler::notZero, IsInflated); + testptr(tmpReg, markOopDesc::monitor_value); // inflated vs stack-locked|neutral|biased + jccb(Assembler::notZero, IsInflated); // Attempt stack-locking ... - orptr (tmpReg, 0x1); + orptr (tmpReg, markOopDesc::unlocked_value); movptr(Address(boxReg, 0), tmpReg); // Anticipate successful CAS if (os::is_MP()) { lock(); @@ -1434,19 +1798,32 @@ cond_inc32(Assembler::equal, ExternalAddress((address)counters->fast_path_entry_count_addr())); } - jccb(Assembler::equal, DONE_LABEL); - - // Recursive locking + jcc(Assembler::equal, DONE_LABEL); // Success + + // Recursive locking. + // The object is stack-locked: markword contains stack pointer to BasicLock. + // Locked by current thread if difference with current SP is less than one page. subptr(tmpReg, rsp); + // Next instruction set ZFlag == 1 (Success) if difference is less then one page. andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - os::vm_page_size())) ); movptr(Address(boxReg, 0), tmpReg); if (counters != NULL) { cond_inc32(Assembler::equal, ExternalAddress((address)counters->fast_path_entry_count_addr())); } - jmpb(DONE_LABEL); + jmp(DONE_LABEL); bind(IsInflated); + // The object is inflated. tmpReg contains pointer to ObjectMonitor* + 2(monitor_value) + +#if INCLUDE_RTM_OPT + // Use the same RTM locking code in 32- and 64-bit VM. + if (use_rtm) { + rtm_inflated_locking(objReg, boxReg, tmpReg, scrReg, cx1Reg, cx2Reg, + rtm_counters, method_data, profile_rtm, DONE_LABEL); + } else { +#endif // INCLUDE_RTM_OPT + #ifndef _LP64 // The object is inflated. // @@ -1576,7 +1953,7 @@ // Without cast to int32_t a movptr will destroy r10 which is typically obj movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark())); - mov (boxReg, tmpReg); + movptr (boxReg, tmpReg); movptr (tmpReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)); testptr(tmpReg, tmpReg); jccb (Assembler::notZero, DONE_LABEL); @@ -1587,9 +1964,11 @@ } cmpxchgptr(r15_thread, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)); // Intentional fall-through into DONE_LABEL ... - +#endif // _LP64 + +#if INCLUDE_RTM_OPT + } // use_rtm() #endif - // DONE_LABEL is a hot target - we'd really like to place it at the // start of cache line by padding with NOPs. // See the AMD and Intel software optimization manuals for the @@ -1631,11 +2010,9 @@ // should not be unlocked by "normal" java-level locking and vice-versa. The specification // doesn't specify what will occur if a program engages in such mixed-mode locking, however. -void MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register tmpReg) { - guarantee (objReg != boxReg, ""); - guarantee (objReg != tmpReg, ""); - guarantee (boxReg != tmpReg, ""); - guarantee (boxReg == rax, ""); +void MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register tmpReg, bool use_rtm) { + assert(boxReg == rax, ""); + assert_different_registers(objReg, boxReg, tmpReg); if (EmitSync & 4) { // Disable - inhibit all inlining. Force control through the slow-path @@ -1667,14 +2044,41 @@ biased_locking_exit(objReg, tmpReg, DONE_LABEL); } +#if INCLUDE_RTM_OPT + if (UseRTMForStackLocks && use_rtm) { + assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking"); + Label L_regular_unlock; + movptr(tmpReg, Address(objReg, 0)); // fetch markword + andptr(tmpReg, markOopDesc::biased_lock_mask_in_place); // look at 3 lock bits + cmpptr(tmpReg, markOopDesc::unlocked_value); // bits = 001 unlocked + jccb(Assembler::notEqual, L_regular_unlock); // if !HLE RegularLock + xend(); // otherwise end... + jmp(DONE_LABEL); // ... and we're done + bind(L_regular_unlock); + } +#endif + cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD); // Examine the displaced header + jcc (Assembler::zero, DONE_LABEL); // 0 indicates recursive stack-lock movptr(tmpReg, Address(objReg, 0)); // Examine the object's markword - jccb (Assembler::zero, DONE_LABEL); // 0 indicates recursive stack-lock - - testptr(tmpReg, 0x02); // Inflated? + testptr(tmpReg, markOopDesc::monitor_value); // Inflated? jccb (Assembler::zero, Stacked); // It's inflated. +#if INCLUDE_RTM_OPT + if (use_rtm) { + Label L_regular_inflated_unlock; + // Clean monitor_value bit to get valid pointer + int owner_offset = ObjectMonitor::owner_offset_in_bytes() - markOopDesc::monitor_value; + movptr(boxReg, Address(tmpReg, owner_offset)); + testptr(boxReg, boxReg); + jccb(Assembler::notZero, L_regular_inflated_unlock); + xend(); + jmpb(DONE_LABEL); + bind(L_regular_inflated_unlock); + } +#endif + // Despite our balanced locking property we still check that m->_owner == Self // as java routines or native JNI code called by this thread might // have released the lock. @@ -2448,7 +2852,9 @@ Condition negated_cond = negate_condition(cond); Label L; jcc(negated_cond, L); + pushf(); // Preserve flags atomic_incl(counter_addr); + popf(); bind(L); }