graal-compiler: src/cpu/x86/vm/macroAssembler

comparison src/cpu/x86/vm/macroAssembler_x86.cpp @ 17780:606acabe7b5c

8031320: Use Intel RTM instructions for locks Summary: Use RTM for inflated locks and stack locks. Reviewed-by: iveresov, twisti, roland, dcubed

author	kvn
date	Thu, 20 Mar 2014 17:49:27 -0700
parents	4d4ea046d32a
children	526acaf3626f

comparison

equal deleted inserted replaced

-:a48e16541e6b
+:606acabe7b5c
 void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {
 mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 }
-void MacroAssembler::movptr(Register dst, AddressLiteral src) {
+void MacroAssembler::movptr(Register dst, AddressLiteral src, Register scratch) {
+// scratch register is not used,
+// it is defined to match parameters of 64-bit version of this method.
 if (src.is_lval()) {
 mov_literal32(dst, (intptr_t)src.target(), src.rspec());
 } else {
 movl(dst, as_Address(src));
 }
 if (value == 0) {                        ; return; }
 if (value == 1 && UseIncDec) { decq(dst) ; return; }
 /* else */      { subq(dst, value)       ; return; }
 }
+void MacroAssembler::incrementq(AddressLiteral dst) {
+if (reachable(dst)) {
+incrementq(as_Address(dst));
+} else {
+lea(rscratch1, dst);
+incrementq(Address(rscratch1, 0));
+}
+}
 void MacroAssembler::incrementq(Register reg, int value) {
 if (value == min_jint) { addq(reg, value); return; }
 if (value <  0) { decrementq(reg, -value); return; }
 if (value == 0) {                        ; return; }
 if (value == 1 && UseIncDec) { incq(reg) ; return; }
 void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {
 mov_literal64(rscratch1, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
 movq(dst, rscratch1);
 }
-void MacroAssembler::movptr(Register dst, AddressLiteral src) {
+void MacroAssembler::movptr(Register dst, AddressLiteral src, Register scratch) {
 if (src.is_lval()) {
 mov_literal64(dst, (intptr_t)src.target(), src.rspec());
 } else {
 if (reachable(src)) {
 movq(dst, as_Address(src));
 } else {
-lea(rscratch1, src);
+lea(scratch, src);
-movq(dst, Address(rscratch1,0));
+movq(dst, Address(scratch, 0));
 }
 }
 }
 void MacroAssembler::movptr(ArrayAddress dst, Register src) {
 void MacroAssembler::andptr(Register dst, int32_t imm32) {
 LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32));
 }
-void MacroAssembler::atomic_incl(AddressLiteral counter_addr) {
+void MacroAssembler::atomic_incl(Address counter_addr) {
-pushf();
+if (os::is_MP())
+lock();
+incrementl(counter_addr);
+}
+void MacroAssembler::atomic_incl(AddressLiteral counter_addr, Register scr) {
 if (reachable(counter_addr)) {
-if (os::is_MP())
+atomic_incl(as_Address(counter_addr));
-lock();
+} else {
-incrementl(as_Address(counter_addr));
+lea(scr, counter_addr);
-} else {
+atomic_incl(Address(scr, 0));
-lea(rscratch1, counter_addr);
+}
-if (os::is_MP())
+}
-lock();
-incrementl(Address(rscratch1, 0));
+#ifdef _LP64
-}
+void MacroAssembler::atomic_incq(Address counter_addr) {
-popf();
+if (os::is_MP())
-}
+lock();
+incrementq(counter_addr);
+}
+void MacroAssembler::atomic_incq(AddressLiteral counter_addr, Register scr) {
+if (reachable(counter_addr)) {
+atomic_incq(as_Address(counter_addr));
+} else {
+lea(scr, counter_addr);
+atomic_incq(Address(scr, 0));
+}
+}
+#endif
 // Writes to stack successive pages until offset reached to check for
 // stack overflow + shadow pages.  This clobbers tmp.
 void MacroAssembler::bang_stack_size(Register size, Register tmp) {
 movptr(tmp, rsp);
 cmpptr(temp_reg, markOopDesc::biased_lock_pattern);
 jcc(Assembler::equal, done);
 }
 #ifdef COMPILER2
+#if INCLUDE_RTM_OPT
+// Update rtm_counters based on abort status
+// input: abort_status
+//        rtm_counters (RTMLockingCounters*)
+// flags are killed
+void MacroAssembler::rtm_counters_update(Register abort_status, Register rtm_counters) {
+atomic_incptr(Address(rtm_counters, RTMLockingCounters::abort_count_offset()));
+if (PrintPreciseRTMLockingStatistics) {
+for (int i = 0; i < RTMLockingCounters::ABORT_STATUS_LIMIT; i++) {
+Label check_abort;
+testl(abort_status, (1<<i));
+jccb(Assembler::equal, check_abort);
+atomic_incptr(Address(rtm_counters, RTMLockingCounters::abortX_count_offset() + (i * sizeof(uintx))));
+bind(check_abort);
+}
+}
+}
+// Branch if (random & (count-1) != 0), count is 2^n
+// tmp, scr and flags are killed
+void MacroAssembler::branch_on_random_using_rdtsc(Register tmp, Register scr, int count, Label& brLabel) {
+assert(tmp == rax, "");
+assert(scr == rdx, "");
+rdtsc(); // modifies EDX:EAX
+andptr(tmp, count-1);
+jccb(Assembler::notZero, brLabel);
+}
+// Perform abort ratio calculation, set no_rtm bit if high ratio
+// input:  rtm_counters_Reg (RTMLockingCounters* address)
+// tmpReg, rtm_counters_Reg and flags are killed
+void MacroAssembler::rtm_abort_ratio_calculation(Register tmpReg,
+Register rtm_counters_Reg,
+RTMLockingCounters* rtm_counters,
+Metadata* method_data) {
+Label L_done, L_check_always_rtm1, L_check_always_rtm2;
+if (RTMLockingCalculationDelay > 0) {
+// Delay calculation
+movptr(tmpReg, ExternalAddress((address) RTMLockingCounters::rtm_calculation_flag_addr()), tmpReg);
+testptr(tmpReg, tmpReg);
+jccb(Assembler::equal, L_done);
+}
+// Abort ratio calculation only if abort_count > RTMAbortThreshold
+//   Aborted transactions = abort_count * 100
+//   All transactions = total_count *  RTMTotalCountIncrRate
+//   Set no_rtm bit if (Aborted transactions >= All transactions * RTMAbortRatio)
+movptr(tmpReg, Address(rtm_counters_Reg, RTMLockingCounters::abort_count_offset()));
+cmpptr(tmpReg, RTMAbortThreshold);
+jccb(Assembler::below, L_check_always_rtm2);
+imulptr(tmpReg, tmpReg, 100);
+Register scrReg = rtm_counters_Reg;
+movptr(scrReg, Address(rtm_counters_Reg, RTMLockingCounters::total_count_offset()));
+imulptr(scrReg, scrReg, RTMTotalCountIncrRate);
+imulptr(scrReg, scrReg, RTMAbortRatio);
+cmpptr(tmpReg, scrReg);
+jccb(Assembler::below, L_check_always_rtm1);
+if (method_data != NULL) {
+// set rtm_state to "no rtm" in MDO
+mov_metadata(tmpReg, method_data);
+if (os::is_MP()) {
+lock();
+}
+orl(Address(tmpReg, MethodData::rtm_state_offset_in_bytes()), NoRTM);
+}
+jmpb(L_done);
+bind(L_check_always_rtm1);
+// Reload RTMLockingCounters* address
+lea(rtm_counters_Reg, ExternalAddress((address)rtm_counters));
+bind(L_check_always_rtm2);
+movptr(tmpReg, Address(rtm_counters_Reg, RTMLockingCounters::total_count_offset()));
+cmpptr(tmpReg, RTMLockingThreshold / RTMTotalCountIncrRate);
+jccb(Assembler::below, L_done);
+if (method_data != NULL) {
+// set rtm_state to "always rtm" in MDO
+mov_metadata(tmpReg, method_data);
+if (os::is_MP()) {
+lock();
+}
+orl(Address(tmpReg, MethodData::rtm_state_offset_in_bytes()), UseRTM);
+}
+bind(L_done);
+}
+// Update counters and perform abort ratio calculation
+// input:  abort_status_Reg
+// rtm_counters_Reg, flags are killed
+void MacroAssembler::rtm_profiling(Register abort_status_Reg,
+Register rtm_counters_Reg,
+RTMLockingCounters* rtm_counters,
+Metadata* method_data,
+bool profile_rtm) {
+assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
+// update rtm counters based on rax value at abort
+// reads abort_status_Reg, updates flags
+lea(rtm_counters_Reg, ExternalAddress((address)rtm_counters));
+rtm_counters_update(abort_status_Reg, rtm_counters_Reg);
+if (profile_rtm) {
+// Save abort status because abort_status_Reg is used by following code.
+if (RTMRetryCount > 0) {
+push(abort_status_Reg);
+}
+assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
+rtm_abort_ratio_calculation(abort_status_Reg, rtm_counters_Reg, rtm_counters, method_data);
+// restore abort status
+if (RTMRetryCount > 0) {
+pop(abort_status_Reg);
+}
+}
+}
+// Retry on abort if abort's status is 0x6: can retry (0x2) | memory conflict (0x4)
+// inputs: retry_count_Reg
+//       : abort_status_Reg
+// output: retry_count_Reg decremented by 1
+// flags are killed
+void MacroAssembler::rtm_retry_lock_on_abort(Register retry_count_Reg, Register abort_status_Reg, Label& retryLabel) {
+Label doneRetry;
+assert(abort_status_Reg == rax, "");
+// The abort reason bits are in eax (see all states in rtmLocking.hpp)
+// 0x6 = conflict on which we can retry (0x2) | memory conflict (0x4)
+// if reason is in 0x6 and retry count != 0 then retry
+andptr(abort_status_Reg, 0x6);
+jccb(Assembler::zero, doneRetry);
+testl(retry_count_Reg, retry_count_Reg);
+jccb(Assembler::zero, doneRetry);
+pause();
+decrementl(retry_count_Reg);
+jmp(retryLabel);
+bind(doneRetry);
+}
+// Spin and retry if lock is busy,
+// inputs: box_Reg (monitor address)
+//       : retry_count_Reg
+// output: retry_count_Reg decremented by 1
+//       : clear z flag if retry count exceeded
+// tmp_Reg, scr_Reg, flags are killed
+void MacroAssembler::rtm_retry_lock_on_busy(Register retry_count_Reg, Register box_Reg,
+Register tmp_Reg, Register scr_Reg, Label& retryLabel) {
+Label SpinLoop, SpinExit, doneRetry;
+// Clean monitor_value bit to get valid pointer
+int owner_offset = ObjectMonitor::owner_offset_in_bytes() - markOopDesc::monitor_value;
+testl(retry_count_Reg, retry_count_Reg);
+jccb(Assembler::zero, doneRetry);
+decrementl(retry_count_Reg);
+movptr(scr_Reg, RTMSpinLoopCount);
+bind(SpinLoop);
+pause();
+decrementl(scr_Reg);
+jccb(Assembler::lessEqual, SpinExit);
+movptr(tmp_Reg, Address(box_Reg, owner_offset));
+testptr(tmp_Reg, tmp_Reg);
+jccb(Assembler::notZero, SpinLoop);
+bind(SpinExit);
+jmp(retryLabel);
+bind(doneRetry);
+incrementl(retry_count_Reg); // clear z flag
+}
+// Use RTM for normal stack locks
+// Input: objReg (object to lock)
+void MacroAssembler::rtm_stack_locking(Register objReg, Register tmpReg, Register scrReg,
+Register retry_on_abort_count_Reg,
+RTMLockingCounters* stack_rtm_counters,
+Metadata* method_data, bool profile_rtm,
+Label& DONE_LABEL, Label& IsInflated) {
+assert(UseRTMForStackLocks, "why call this otherwise?");
+assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking");
+assert(tmpReg == rax, "");
+assert(scrReg == rdx, "");
+Label L_rtm_retry, L_decrement_retry, L_on_abort;
+if (RTMRetryCount > 0) {
+movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort
+bind(L_rtm_retry);
+}
+if (!UseRTMXendForLockBusy) {
+movptr(tmpReg, Address(objReg, 0));
+testptr(tmpReg, markOopDesc::monitor_value);  // inflated vs stack-locked|neutral|biased
+jcc(Assembler::notZero, IsInflated);
+}
+if (PrintPreciseRTMLockingStatistics || profile_rtm) {
+Label L_noincrement;
+if (RTMTotalCountIncrRate > 1) {
+// tmpReg, scrReg and flags are killed
+branch_on_random_using_rdtsc(tmpReg, scrReg, (int)RTMTotalCountIncrRate, L_noincrement);
+}
+assert(stack_rtm_counters != NULL, "should not be NULL when profiling RTM");
+atomic_incptr(ExternalAddress((address)stack_rtm_counters->total_count_addr()), scrReg);
+bind(L_noincrement);
+}
+xbegin(L_on_abort);
+movptr(tmpReg, Address(objReg, 0));       // fetch markword
+andptr(tmpReg, markOopDesc::biased_lock_mask_in_place); // look at 3 lock bits
+cmpptr(tmpReg, markOopDesc::unlocked_value);            // bits = 001 unlocked
+jcc(Assembler::equal, DONE_LABEL);        // all done if unlocked
+Register abort_status_Reg = tmpReg; // status of abort is stored in RAX
+if (UseRTMXendForLockBusy) {
+xend();
+movptr(tmpReg, Address(objReg, 0));
+testptr(tmpReg, markOopDesc::monitor_value);  // inflated vs stack-locked|neutral|biased
+jcc(Assembler::notZero, IsInflated);
+movptr(abort_status_Reg, 0x1);                // Set the abort status to 1 (as xabort does)
+jmp(L_decrement_retry);
+}
+else {
+xabort(0);
+}
+bind(L_on_abort);
+if (PrintPreciseRTMLockingStatistics || profile_rtm) {
+rtm_profiling(abort_status_Reg, scrReg, stack_rtm_counters, method_data, profile_rtm);
+}
+bind(L_decrement_retry);
+if (RTMRetryCount > 0) {
+// retry on lock abort if abort status is 'can retry' (0x2) or 'memory conflict' (0x4)
+rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry);
+}
+}
+// Use RTM for inflating locks
+// inputs: objReg (object to lock)
+//         boxReg (on-stack box address (displaced header location) - KILLED)
+//         tmpReg (ObjectMonitor address + 2(monitor_value))
+void MacroAssembler::rtm_inflated_locking(Register objReg, Register boxReg, Register tmpReg,
+Register scrReg, Register retry_on_busy_count_Reg,
+Register retry_on_abort_count_Reg,
+RTMLockingCounters* rtm_counters,
+Metadata* method_data, bool profile_rtm,
+Label& DONE_LABEL) {
+assert(UseRTMLocking, "why call this otherwise?");
+assert(tmpReg == rax, "");
+assert(scrReg == rdx, "");
+Label L_rtm_retry, L_decrement_retry, L_on_abort;
+// Clean monitor_value bit to get valid pointer
+int owner_offset = ObjectMonitor::owner_offset_in_bytes() - markOopDesc::monitor_value;
+// Without cast to int32_t a movptr will destroy r10 which is typically obj
+movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
+movptr(boxReg, tmpReg); // Save ObjectMonitor address
+if (RTMRetryCount > 0) {
+movl(retry_on_busy_count_Reg, RTMRetryCount);  // Retry on lock busy
+movl(retry_on_abort_count_Reg, RTMRetryCount); // Retry on abort
+bind(L_rtm_retry);
+}
+if (PrintPreciseRTMLockingStatistics || profile_rtm) {
+Label L_noincrement;
+if (RTMTotalCountIncrRate > 1) {
+// tmpReg, scrReg and flags are killed
+branch_on_random_using_rdtsc(tmpReg, scrReg, (int)RTMTotalCountIncrRate, L_noincrement);
+}
+assert(rtm_counters != NULL, "should not be NULL when profiling RTM");
+atomic_incptr(ExternalAddress((address)rtm_counters->total_count_addr()), scrReg);
+bind(L_noincrement);
+}
+xbegin(L_on_abort);
+movptr(tmpReg, Address(objReg, 0));
+movptr(tmpReg, Address(tmpReg, owner_offset));
+testptr(tmpReg, tmpReg);
+jcc(Assembler::zero, DONE_LABEL);
+if (UseRTMXendForLockBusy) {
+xend();
+jmp(L_decrement_retry);
+}
+else {
+xabort(0);
+}
+bind(L_on_abort);
+Register abort_status_Reg = tmpReg; // status of abort is stored in RAX
+if (PrintPreciseRTMLockingStatistics || profile_rtm) {
+rtm_profiling(abort_status_Reg, scrReg, rtm_counters, method_data, profile_rtm);
+}
+if (RTMRetryCount > 0) {
+// retry on lock abort if abort status is 'can retry' (0x2) or 'memory conflict' (0x4)
+rtm_retry_lock_on_abort(retry_on_abort_count_Reg, abort_status_Reg, L_rtm_retry);
+}
+movptr(tmpReg, Address(boxReg, owner_offset)) ;
+testptr(tmpReg, tmpReg) ;
+jccb(Assembler::notZero, L_decrement_retry) ;
+// Appears unlocked - try to swing _owner from null to non-null.
+// Invariant: tmpReg == 0.  tmpReg is EAX which is the implicit cmpxchg comparand.
+#ifdef _LP64
+Register threadReg = r15_thread;
+#else
+get_thread(scrReg);
+Register threadReg = scrReg;
+#endif
+if (os::is_MP()) {
+lock();
+}
+cmpxchgptr(threadReg, Address(boxReg, owner_offset)); // Updates tmpReg
+if (RTMRetryCount > 0) {
+// success done else retry
+jccb(Assembler::equal, DONE_LABEL) ;
+bind(L_decrement_retry);
+// Spin and retry if lock is busy.
+rtm_retry_lock_on_busy(retry_on_busy_count_Reg, boxReg, tmpReg, scrReg, L_rtm_retry);
+}
+else {
+bind(L_decrement_retry);
+}
+}
+#endif //  INCLUDE_RTM_OPT
 // Fast_Lock and Fast_Unlock used by C2
 // Because the transitions from emitted code to the runtime
 // monitorenter/exit helper stubs are so slow it's critical that
 // we inline both the stack-locking fast-path and the inflated fast path.
 // obj: object to lock
 // box: on-stack box address (displaced header location) - KILLED
 // rax,: tmp -- KILLED
 // scr: tmp -- KILLED
-void MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmpReg, Register scrReg, BiasedLockingCounters* counters) {
+void MacroAssembler::fast_lock(Register objReg, Register boxReg, Register tmpReg,
+Register scrReg, Register cx1Reg, Register cx2Reg,
+BiasedLockingCounters* counters,
+RTMLockingCounters* rtm_counters,
+RTMLockingCounters* stack_rtm_counters,
+Metadata* method_data,
+bool use_rtm, bool profile_rtm) {
 // Ensure the register assignents are disjoint
-guarantee (objReg != boxReg, "");
+assert(tmpReg == rax, "");
-guarantee (objReg != tmpReg, "");
-guarantee (objReg != scrReg, "");
+if (use_rtm) {
-guarantee (boxReg != tmpReg, "");
+assert_different_registers(objReg, boxReg, tmpReg, scrReg, cx1Reg, cx2Reg);
-guarantee (boxReg != scrReg, "");
+} else {
-guarantee (tmpReg == rax, "");
+assert(cx1Reg == noreg, "");
+assert(cx2Reg == noreg, "");
+assert_different_registers(objReg, boxReg, tmpReg, scrReg);
+}
 if (counters != NULL) {
-atomic_incl(ExternalAddress((address)counters->total_entry_count_addr()));
+atomic_incl(ExternalAddress((address)counters->total_entry_count_addr()), scrReg);
 }
 if (EmitSync & 1) {
 // set box->dhw = unused_mark (3)
 // Force all sync thru slow-path: slow_enter() and slow_exit()
 movptr (Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
 // If this invariant is not held we risk exclusion (safety) failure.
 if (UseBiasedLocking && !UseOptoBiasInlining) {
 biased_locking_enter(boxReg, objReg, tmpReg, scrReg, true, DONE_LABEL, NULL, counters);
 }
+#if INCLUDE_RTM_OPT
+if (UseRTMForStackLocks && use_rtm) {
+rtm_stack_locking(objReg, tmpReg, scrReg, cx2Reg,
+stack_rtm_counters, method_data, profile_rtm,
+DONE_LABEL, IsInflated);
+}
+#endif // INCLUDE_RTM_OPT
 movptr(tmpReg, Address(objReg, 0));          // [FETCH]
-testl (tmpReg, markOopDesc::monitor_value);  // inflated vs stack-locked|neutral|biased
+testptr(tmpReg, markOopDesc::monitor_value); // inflated vs stack-locked|neutral|biased
-jccb  (Assembler::notZero, IsInflated);
+jccb(Assembler::notZero, IsInflated);
 // Attempt stack-locking ...
-orptr (tmpReg, 0x1);
+orptr (tmpReg, markOopDesc::unlocked_value);
 movptr(Address(boxReg, 0), tmpReg);          // Anticipate successful CAS
 if (os::is_MP()) {
 lock();
 }
 cmpxchgptr(boxReg, Address(objReg, 0));      // Updates tmpReg
 if (counters != NULL) {
 cond_inc32(Assembler::equal,
 ExternalAddress((address)counters->fast_path_entry_count_addr()));
 }
-jccb(Assembler::equal, DONE_LABEL);
+jcc(Assembler::equal, DONE_LABEL);           // Success
-// Recursive locking
+// Recursive locking.
+// The object is stack-locked: markword contains stack pointer to BasicLock.
+// Locked by current thread if difference with current SP is less than one page.
 subptr(tmpReg, rsp);
+// Next instruction set ZFlag == 1 (Success) if difference is less then one page.
 andptr(tmpReg, (int32_t) (NOT_LP64(0xFFFFF003) LP64_ONLY(7 - os::vm_page_size())) );
 movptr(Address(boxReg, 0), tmpReg);
 if (counters != NULL) {
 cond_inc32(Assembler::equal,
 ExternalAddress((address)counters->fast_path_entry_count_addr()));
 }
-jmpb(DONE_LABEL);
+jmp(DONE_LABEL);
 bind(IsInflated);
+// The object is inflated. tmpReg contains pointer to ObjectMonitor* + 2(monitor_value)
+#if INCLUDE_RTM_OPT
+// Use the same RTM locking code in 32- and 64-bit VM.
+if (use_rtm) {
+rtm_inflated_locking(objReg, boxReg, tmpReg, scrReg, cx1Reg, cx2Reg,
+rtm_counters, method_data, profile_rtm, DONE_LABEL);
+} else {
+#endif // INCLUDE_RTM_OPT
 #ifndef _LP64
 // The object is inflated.
 //
 // TODO-FIXME: eliminate the ugly use of manifest constants:
 //   Use markOopDesc::monitor_value instead of "2".
 // avoid an RTO->RTS upgrade on the $line.
 // Without cast to int32_t a movptr will destroy r10 which is typically obj
 movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark()));
-mov    (boxReg, tmpReg);
+movptr (boxReg, tmpReg);
 movptr (tmpReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2));
 testptr(tmpReg, tmpReg);
 jccb   (Assembler::notZero, DONE_LABEL);
 // It's inflated and appears unlocked
 if (os::is_MP()) {
 lock();
 }
 cmpxchgptr(r15_thread, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2));
 // Intentional fall-through into DONE_LABEL ...
+#endif // _LP64
+#if INCLUDE_RTM_OPT
+} // use_rtm()
 #endif
 // DONE_LABEL is a hot target - we'd really like to place it at the
 // start of cache line by padding with NOPs.
 // See the AMD and Intel software optimization manuals for the
 // most efficient "long" NOP encodings.
 // Unfortunately none of our alignment mechanisms suffice.
 // The only other source of unbalanced locking would be JNI.  The "Java Native Interface:
 // Programmer's Guide and Specification" claims that an object locked by jni_monitorenter
 // should not be unlocked by "normal" java-level locking and vice-versa.  The specification
 // doesn't specify what will occur if a program engages in such mixed-mode locking, however.
-void MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register tmpReg) {
+void MacroAssembler::fast_unlock(Register objReg, Register boxReg, Register tmpReg, bool use_rtm) {
-guarantee (objReg != boxReg, "");
+assert(boxReg == rax, "");
-guarantee (objReg != tmpReg, "");
+assert_different_registers(objReg, boxReg, tmpReg);
-guarantee (boxReg != tmpReg, "");
-guarantee (boxReg == rax, "");
 if (EmitSync & 4) {
 // Disable - inhibit all inlining.  Force control through the slow-path
 cmpptr (rsp, 0);
 } else
 // and appear before the (box->dhw == 0) recursive stack-lock test.
 if (UseBiasedLocking && !UseOptoBiasInlining) {
 biased_locking_exit(objReg, tmpReg, DONE_LABEL);
 }
+#if INCLUDE_RTM_OPT
+if (UseRTMForStackLocks && use_rtm) {
+assert(!UseBiasedLocking, "Biased locking is not supported with RTM locking");
+Label L_regular_unlock;
+movptr(tmpReg, Address(objReg, 0));           // fetch markword
+andptr(tmpReg, markOopDesc::biased_lock_mask_in_place); // look at 3 lock bits
+cmpptr(tmpReg, markOopDesc::unlocked_value);            // bits = 001 unlocked
+jccb(Assembler::notEqual, L_regular_unlock);  // if !HLE RegularLock
+xend();                                       // otherwise end...
+jmp(DONE_LABEL);                              // ... and we're done
+bind(L_regular_unlock);
+}
+#endif
 cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD); // Examine the displaced header
+jcc   (Assembler::zero, DONE_LABEL);            // 0 indicates recursive stack-lock
 movptr(tmpReg, Address(objReg, 0));             // Examine the object's markword
-jccb  (Assembler::zero, DONE_LABEL);            // 0 indicates recursive stack-lock
+testptr(tmpReg, markOopDesc::monitor_value);    // Inflated?
-testptr(tmpReg, 0x02);                          // Inflated?
 jccb  (Assembler::zero, Stacked);
 // It's inflated.
+#if INCLUDE_RTM_OPT
+if (use_rtm) {
+Label L_regular_inflated_unlock;
+// Clean monitor_value bit to get valid pointer
+int owner_offset = ObjectMonitor::owner_offset_in_bytes() - markOopDesc::monitor_value;
+movptr(boxReg, Address(tmpReg, owner_offset));
+testptr(boxReg, boxReg);
+jccb(Assembler::notZero, L_regular_inflated_unlock);
+xend();
+jmpb(DONE_LABEL);
+bind(L_regular_inflated_unlock);
+}
+#endif
 // Despite our balanced locking property we still check that m->_owner == Self
 // as java routines or native JNI code called by this thread might
 // have released the lock.
 // Refer to the comments in synchronizer.cpp for how we might encode extra
 // state in _succ so we can avoid fetching EntryList|cxq.
 void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr) {
 Condition negated_cond = negate_condition(cond);
 Label L;
 jcc(negated_cond, L);
+pushf(); // Preserve flags
 atomic_incl(counter_addr);
+popf();
 bind(L);
 }
 int MacroAssembler::corrected_idivl(Register reg) {
 // Full implementation of Java idiv and irem; checks for

Mercurial > hg > graal-compiler

comparison src/cpu/x86/vm/macroAssembler_x86.cpp @ 17780:606acabe7b5c