Mercurial > hg > truffle

comparison src/cpu/x86/vm/x86_64.ad @ 14494:5292439ef895

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler Summary: Consolidated C2 x86 locking code in one place in macroAssembler_x86.cpp. Reviewed-by: roland
author kvn
date Mon, 24 Feb 2014 15:12:26 -0800
parents 45467c53f178
children cd5d10655495
comparison
equal deleted inserted replaced
14472:80b39937b791 14494:5292439ef895
2597 __ movdbl(Address(rsp, 0), $src$$XMMRegister); 2597 __ movdbl(Address(rsp, 0), $src$$XMMRegister);
2598 __ fld_d(Address(rsp, 0)); 2598 __ fld_d(Address(rsp, 0));
2599 %} 2599 %}
2600 2600
2601 2601
2602 // obj: object to lock
2603 // box: box address (header location) -- killed
2604 // tmp: rax -- killed
2605 // scr: rbx -- killed
2606 //
2607 // What follows is a direct transliteration of fast_lock() and fast_unlock()
2608 // from i486.ad. See that file for comments.
2609 // TODO: where possible switch from movq (r, 0) to movl(r,0) and
2610 // use the shorter encoding. (Movl clears the high-order 32-bits).
2611
2612
2613 enc_class Fast_Lock(rRegP obj, rRegP box, rax_RegI tmp, rRegP scr)
2614 %{
2615 Register objReg = as_Register((int)$obj$$reg);
2616 Register boxReg = as_Register((int)$box$$reg);
2617 Register tmpReg = as_Register($tmp$$reg);
2618 Register scrReg = as_Register($scr$$reg);
2619 MacroAssembler masm(&cbuf);
2620
2621 // Verify uniqueness of register assignments -- necessary but not sufficient
2622 assert (objReg != boxReg && objReg != tmpReg &&
2623 objReg != scrReg && tmpReg != scrReg, "invariant") ;
2624
2625 if (_counters != NULL) {
2626 masm.atomic_incl(ExternalAddress((address) _counters->total_entry_count_addr()));
2627 }
2628 if (EmitSync & 1) {
2629 // Without cast to int32_t a movptr will destroy r10 which is typically obj
2630 masm.movptr (Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark())) ;
2631 masm.cmpptr(rsp, (int32_t)NULL_WORD) ;
2632 } else
2633 if (EmitSync & 2) {
2634 Label DONE_LABEL;
2635 if (UseBiasedLocking) {
2636 // Note: tmpReg maps to the swap_reg argument and scrReg to the tmp_reg argument.
2637 masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, _counters);
2638 }
2639 // QQQ was movl...
2640 masm.movptr(tmpReg, 0x1);
2641 masm.orptr(tmpReg, Address(objReg, 0));
2642 masm.movptr(Address(boxReg, 0), tmpReg);
2643 if (os::is_MP()) {
2644 masm.lock();
2645 }
2646 masm.cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg
2647 masm.jcc(Assembler::equal, DONE_LABEL);
2648
2649 // Recursive locking
2650 masm.subptr(tmpReg, rsp);
2651 masm.andptr(tmpReg, 7 - os::vm_page_size());
2652 masm.movptr(Address(boxReg, 0), tmpReg);
2653
2654 masm.bind(DONE_LABEL);
2655 masm.nop(); // avoid branch to branch
2656 } else {
2657 Label DONE_LABEL, IsInflated, Egress;
2658
2659 masm.movptr(tmpReg, Address(objReg, 0)) ;
2660 masm.testl (tmpReg, 0x02) ; // inflated vs stack-locked|neutral|biased
2661 masm.jcc (Assembler::notZero, IsInflated) ;
2662
2663 // it's stack-locked, biased or neutral
2664 // TODO: optimize markword triage order to reduce the number of
2665 // conditional branches in the most common cases.
2666 // Beware -- there's a subtle invariant that fetch of the markword
2667 // at [FETCH], below, will never observe a biased encoding (*101b).
2668 // If this invariant is not held we'll suffer exclusion (safety) failure.
2669
2670 if (UseBiasedLocking && !UseOptoBiasInlining) {
2671 masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, true, DONE_LABEL, NULL, _counters);
2672 masm.movptr(tmpReg, Address(objReg, 0)) ; // [FETCH]
2673 }
2674
2675 // was q will it destroy high?
2676 masm.orl (tmpReg, 1) ;
2677 masm.movptr(Address(boxReg, 0), tmpReg) ;
2678 if (os::is_MP()) { masm.lock(); }
2679 masm.cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg
2680 if (_counters != NULL) {
2681 masm.cond_inc32(Assembler::equal,
2682 ExternalAddress((address) _counters->fast_path_entry_count_addr()));
2683 }
2684 masm.jcc (Assembler::equal, DONE_LABEL);
2685
2686 // Recursive locking
2687 masm.subptr(tmpReg, rsp);
2688 masm.andptr(tmpReg, 7 - os::vm_page_size());
2689 masm.movptr(Address(boxReg, 0), tmpReg);
2690 if (_counters != NULL) {
2691 masm.cond_inc32(Assembler::equal,
2692 ExternalAddress((address) _counters->fast_path_entry_count_addr()));
2693 }
2694 masm.jmp (DONE_LABEL) ;
2695
2696 masm.bind (IsInflated) ;
2697 // It's inflated
2698
2699 // TODO: someday avoid the ST-before-CAS penalty by
2700 // relocating (deferring) the following ST.
2701 // We should also think about trying a CAS without having
2702 // fetched _owner. If the CAS is successful we may
2703 // avoid an RTO->RTS upgrade on the $line.
2704 // Without cast to int32_t a movptr will destroy r10 which is typically obj
2705 masm.movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark())) ;
2706
2707 masm.mov (boxReg, tmpReg) ;
2708 masm.movptr (tmpReg, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
2709 masm.testptr(tmpReg, tmpReg) ;
2710 masm.jcc (Assembler::notZero, DONE_LABEL) ;
2711
2712 // It's inflated and appears unlocked
2713 if (os::is_MP()) { masm.lock(); }
2714 masm.cmpxchgptr(r15_thread, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
2715 // Intentional fall-through into DONE_LABEL ...
2716
2717 masm.bind (DONE_LABEL) ;
2718 masm.nop () ; // avoid jmp to jmp
2719 }
2720 %}
2721
2722 // obj: object to unlock
2723 // box: box address (displaced header location), killed
2724 // RBX: killed tmp; cannot be obj nor box
2725 enc_class Fast_Unlock(rRegP obj, rax_RegP box, rRegP tmp)
2726 %{
2727
2728 Register objReg = as_Register($obj$$reg);
2729 Register boxReg = as_Register($box$$reg);
2730 Register tmpReg = as_Register($tmp$$reg);
2731 MacroAssembler masm(&cbuf);
2732
2733 if (EmitSync & 4) {
2734 masm.cmpptr(rsp, 0) ;
2735 } else
2736 if (EmitSync & 8) {
2737 Label DONE_LABEL;
2738 if (UseBiasedLocking) {
2739 masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL);
2740 }
2741
2742 // Check whether the displaced header is 0
2743 //(=> recursive unlock)
2744 masm.movptr(tmpReg, Address(boxReg, 0));
2745 masm.testptr(tmpReg, tmpReg);
2746 masm.jcc(Assembler::zero, DONE_LABEL);
2747
2748 // If not recursive lock, reset the header to displaced header
2749 if (os::is_MP()) {
2750 masm.lock();
2751 }
2752 masm.cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box
2753 masm.bind(DONE_LABEL);
2754 masm.nop(); // avoid branch to branch
2755 } else {
2756 Label DONE_LABEL, Stacked, CheckSucc ;
2757
2758 if (UseBiasedLocking && !UseOptoBiasInlining) {
2759 masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL);
2760 }
2761
2762 masm.movptr(tmpReg, Address(objReg, 0)) ;
2763 masm.cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD) ;
2764 masm.jcc (Assembler::zero, DONE_LABEL) ;
2765 masm.testl (tmpReg, 0x02) ;
2766 masm.jcc (Assembler::zero, Stacked) ;
2767
2768 // It's inflated
2769 masm.movptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
2770 masm.xorptr(boxReg, r15_thread) ;
2771 masm.orptr (boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2)) ;
2772 masm.jcc (Assembler::notZero, DONE_LABEL) ;
2773 masm.movptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2)) ;
2774 masm.orptr (boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2)) ;
2775 masm.jcc (Assembler::notZero, CheckSucc) ;
2776 masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
2777 masm.jmp (DONE_LABEL) ;
2778
2779 if ((EmitSync & 65536) == 0) {
2780 Label LSuccess, LGoSlowPath ;
2781 masm.bind (CheckSucc) ;
2782 masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
2783 masm.jcc (Assembler::zero, LGoSlowPath) ;
2784
2785 // I'd much rather use lock:andl m->_owner, 0 as it's faster than the
2786 // the explicit ST;MEMBAR combination, but masm doesn't currently support
2787 // "ANDQ M,IMM". Don't use MFENCE here. lock:add to TOS, xchg, etc
2788 // are all faster when the write buffer is populated.
2789 masm.movptr (Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
2790 if (os::is_MP()) {
2791 masm.lock () ; masm.addl (Address(rsp, 0), 0) ;
2792 }
2793 masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
2794 masm.jcc (Assembler::notZero, LSuccess) ;
2795
2796 masm.movptr (boxReg, (int32_t)NULL_WORD) ; // box is really EAX
2797 if (os::is_MP()) { masm.lock(); }
2798 masm.cmpxchgptr(r15_thread, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));
2799 masm.jcc (Assembler::notEqual, LSuccess) ;
2800 // Intentional fall-through into slow-path
2801
2802 masm.bind (LGoSlowPath) ;
2803 masm.orl (boxReg, 1) ; // set ICC.ZF=0 to indicate failure
2804 masm.jmp (DONE_LABEL) ;
2805
2806 masm.bind (LSuccess) ;
2807 masm.testl (boxReg, 0) ; // set ICC.ZF=1 to indicate success
2808 masm.jmp (DONE_LABEL) ;
2809 }
2810
2811 masm.bind (Stacked) ;
2812 masm.movptr(tmpReg, Address (boxReg, 0)) ; // re-fetch
2813 if (os::is_MP()) { masm.lock(); }
2814 masm.cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box
2815
2816 if (EmitSync & 65536) {
2817 masm.bind (CheckSucc) ;
2818 }
2819 masm.bind(DONE_LABEL);
2820 if (EmitSync & 32768) {
2821 masm.nop(); // avoid branch to branch
2822 }
2823 }
2824 %}
2825
2826
2827 enc_class enc_rethrow() 2602 enc_class enc_rethrow()
2828 %{ 2603 %{
2829 cbuf.set_insts_mark(); 2604 cbuf.set_insts_mark();
2830 emit_opcode(cbuf, 0xE9); // jmp entry 2605 emit_opcode(cbuf, 0xE9); // jmp entry
2831 emit_d32_reloc(cbuf, 2606 emit_d32_reloc(cbuf,
11451 %} 11226 %}
11452 11227
11453 // ============================================================================ 11228 // ============================================================================
11454 // inlined locking and unlocking 11229 // inlined locking and unlocking
11455 11230
11456 instruct cmpFastLock(rFlagsReg cr, 11231 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{
11457 rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr)
11458 %{
11459 match(Set cr (FastLock object box)); 11232 match(Set cr (FastLock object box));
11460 effect(TEMP tmp, TEMP scr, USE_KILL box); 11233 effect(TEMP tmp, TEMP scr, USE_KILL box);
11461
11462 ins_cost(300); 11234 ins_cost(300);
11463 format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %} 11235 format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
11464 ins_encode(Fast_Lock(object, box, tmp, scr)); 11236 ins_encode %{
11237 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $scr$$Register, _counters);
11238 %}
11465 ins_pipe(pipe_slow); 11239 ins_pipe(pipe_slow);
11466 %} 11240 %}
11467 11241
11468 instruct cmpFastUnlock(rFlagsReg cr, 11242 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{
11469 rRegP object, rax_RegP box, rRegP tmp)
11470 %{
11471 match(Set cr (FastUnlock object box)); 11243 match(Set cr (FastUnlock object box));
11472 effect(TEMP tmp, USE_KILL box); 11244 effect(TEMP tmp, USE_KILL box);
11473
11474 ins_cost(300); 11245 ins_cost(300);
11475 format %{ "fastunlock $object,$box\t! kills $box,$tmp" %} 11246 format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
11476 ins_encode(Fast_Unlock(object, box, tmp)); 11247 ins_encode %{
11248 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register);
11249 %}
11477 ins_pipe(pipe_slow); 11250 ins_pipe(pipe_slow);
11478 %} 11251 %}
11479 11252
11480 11253
11481 // ============================================================================ 11254 // ============================================================================