Mercurial > hg > graal-jvmci-8
comparison src/cpu/x86/vm/x86_64.ad @ 17714:4d4ea046d32a
8033805: Move Fast_Lock/Fast_Unlock code from .ad files to macroassembler
Summary: Consolidated C2 x86 locking code in one place in macroAssembler_x86.cpp.
Reviewed-by: roland
author | kvn |
---|---|
date | Mon, 24 Feb 2014 15:12:26 -0800 |
parents | 984401824c5e |
children | 085b304a1cc5 |
comparison
equal
deleted
inserted
replaced
17713:e7cf34c87433 | 17714:4d4ea046d32a |
---|---|
2589 __ movdbl(Address(rsp, 0), $src$$XMMRegister); | 2589 __ movdbl(Address(rsp, 0), $src$$XMMRegister); |
2590 __ fld_d(Address(rsp, 0)); | 2590 __ fld_d(Address(rsp, 0)); |
2591 %} | 2591 %} |
2592 | 2592 |
2593 | 2593 |
2594 // obj: object to lock | |
2595 // box: box address (header location) -- killed | |
2596 // tmp: rax -- killed | |
2597 // scr: rbx -- killed | |
2598 // | |
2599 // What follows is a direct transliteration of fast_lock() and fast_unlock() | |
2600 // from i486.ad. See that file for comments. | |
2601 // TODO: where possible switch from movq (r, 0) to movl(r,0) and | |
2602 // use the shorter encoding. (Movl clears the high-order 32-bits). | |
2603 | |
2604 | |
2605 enc_class Fast_Lock(rRegP obj, rRegP box, rax_RegI tmp, rRegP scr) | |
2606 %{ | |
2607 Register objReg = as_Register((int)$obj$$reg); | |
2608 Register boxReg = as_Register((int)$box$$reg); | |
2609 Register tmpReg = as_Register($tmp$$reg); | |
2610 Register scrReg = as_Register($scr$$reg); | |
2611 MacroAssembler masm(&cbuf); | |
2612 | |
2613 // Verify uniqueness of register assignments -- necessary but not sufficient | |
2614 assert (objReg != boxReg && objReg != tmpReg && | |
2615 objReg != scrReg && tmpReg != scrReg, "invariant") ; | |
2616 | |
2617 if (_counters != NULL) { | |
2618 masm.atomic_incl(ExternalAddress((address) _counters->total_entry_count_addr())); | |
2619 } | |
2620 if (EmitSync & 1) { | |
2621 // Without cast to int32_t a movptr will destroy r10 which is typically obj | |
2622 masm.movptr (Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark())) ; | |
2623 masm.cmpptr(rsp, (int32_t)NULL_WORD) ; | |
2624 } else | |
2625 if (EmitSync & 2) { | |
2626 Label DONE_LABEL; | |
2627 if (UseBiasedLocking) { | |
2628 // Note: tmpReg maps to the swap_reg argument and scrReg to the tmp_reg argument. | |
2629 masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, _counters); | |
2630 } | |
2631 // QQQ was movl... | |
2632 masm.movptr(tmpReg, 0x1); | |
2633 masm.orptr(tmpReg, Address(objReg, 0)); | |
2634 masm.movptr(Address(boxReg, 0), tmpReg); | |
2635 if (os::is_MP()) { | |
2636 masm.lock(); | |
2637 } | |
2638 masm.cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg | |
2639 masm.jcc(Assembler::equal, DONE_LABEL); | |
2640 | |
2641 // Recursive locking | |
2642 masm.subptr(tmpReg, rsp); | |
2643 masm.andptr(tmpReg, 7 - os::vm_page_size()); | |
2644 masm.movptr(Address(boxReg, 0), tmpReg); | |
2645 | |
2646 masm.bind(DONE_LABEL); | |
2647 masm.nop(); // avoid branch to branch | |
2648 } else { | |
2649 Label DONE_LABEL, IsInflated, Egress; | |
2650 | |
2651 masm.movptr(tmpReg, Address(objReg, 0)) ; | |
2652 masm.testl (tmpReg, 0x02) ; // inflated vs stack-locked|neutral|biased | |
2653 masm.jcc (Assembler::notZero, IsInflated) ; | |
2654 | |
2655 // it's stack-locked, biased or neutral | |
2656 // TODO: optimize markword triage order to reduce the number of | |
2657 // conditional branches in the most common cases. | |
2658 // Beware -- there's a subtle invariant that fetch of the markword | |
2659 // at [FETCH], below, will never observe a biased encoding (*101b). | |
2660 // If this invariant is not held we'll suffer exclusion (safety) failure. | |
2661 | |
2662 if (UseBiasedLocking && !UseOptoBiasInlining) { | |
2663 masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, true, DONE_LABEL, NULL, _counters); | |
2664 masm.movptr(tmpReg, Address(objReg, 0)) ; // [FETCH] | |
2665 } | |
2666 | |
2667 // was q will it destroy high? | |
2668 masm.orl (tmpReg, 1) ; | |
2669 masm.movptr(Address(boxReg, 0), tmpReg) ; | |
2670 if (os::is_MP()) { masm.lock(); } | |
2671 masm.cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg | |
2672 if (_counters != NULL) { | |
2673 masm.cond_inc32(Assembler::equal, | |
2674 ExternalAddress((address) _counters->fast_path_entry_count_addr())); | |
2675 } | |
2676 masm.jcc (Assembler::equal, DONE_LABEL); | |
2677 | |
2678 // Recursive locking | |
2679 masm.subptr(tmpReg, rsp); | |
2680 masm.andptr(tmpReg, 7 - os::vm_page_size()); | |
2681 masm.movptr(Address(boxReg, 0), tmpReg); | |
2682 if (_counters != NULL) { | |
2683 masm.cond_inc32(Assembler::equal, | |
2684 ExternalAddress((address) _counters->fast_path_entry_count_addr())); | |
2685 } | |
2686 masm.jmp (DONE_LABEL) ; | |
2687 | |
2688 masm.bind (IsInflated) ; | |
2689 // It's inflated | |
2690 | |
2691 // TODO: someday avoid the ST-before-CAS penalty by | |
2692 // relocating (deferring) the following ST. | |
2693 // We should also think about trying a CAS without having | |
2694 // fetched _owner. If the CAS is successful we may | |
2695 // avoid an RTO->RTS upgrade on the $line. | |
2696 // Without cast to int32_t a movptr will destroy r10 which is typically obj | |
2697 masm.movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark())) ; | |
2698 | |
2699 masm.mov (boxReg, tmpReg) ; | |
2700 masm.movptr (tmpReg, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; | |
2701 masm.testptr(tmpReg, tmpReg) ; | |
2702 masm.jcc (Assembler::notZero, DONE_LABEL) ; | |
2703 | |
2704 // It's inflated and appears unlocked | |
2705 if (os::is_MP()) { masm.lock(); } | |
2706 masm.cmpxchgptr(r15_thread, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; | |
2707 // Intentional fall-through into DONE_LABEL ... | |
2708 | |
2709 masm.bind (DONE_LABEL) ; | |
2710 masm.nop () ; // avoid jmp to jmp | |
2711 } | |
2712 %} | |
2713 | |
2714 // obj: object to unlock | |
2715 // box: box address (displaced header location), killed | |
2716 // RBX: killed tmp; cannot be obj nor box | |
2717 enc_class Fast_Unlock(rRegP obj, rax_RegP box, rRegP tmp) | |
2718 %{ | |
2719 | |
2720 Register objReg = as_Register($obj$$reg); | |
2721 Register boxReg = as_Register($box$$reg); | |
2722 Register tmpReg = as_Register($tmp$$reg); | |
2723 MacroAssembler masm(&cbuf); | |
2724 | |
2725 if (EmitSync & 4) { | |
2726 masm.cmpptr(rsp, 0) ; | |
2727 } else | |
2728 if (EmitSync & 8) { | |
2729 Label DONE_LABEL; | |
2730 if (UseBiasedLocking) { | |
2731 masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL); | |
2732 } | |
2733 | |
2734 // Check whether the displaced header is 0 | |
2735 //(=> recursive unlock) | |
2736 masm.movptr(tmpReg, Address(boxReg, 0)); | |
2737 masm.testptr(tmpReg, tmpReg); | |
2738 masm.jcc(Assembler::zero, DONE_LABEL); | |
2739 | |
2740 // If not recursive lock, reset the header to displaced header | |
2741 if (os::is_MP()) { | |
2742 masm.lock(); | |
2743 } | |
2744 masm.cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box | |
2745 masm.bind(DONE_LABEL); | |
2746 masm.nop(); // avoid branch to branch | |
2747 } else { | |
2748 Label DONE_LABEL, Stacked, CheckSucc ; | |
2749 | |
2750 if (UseBiasedLocking && !UseOptoBiasInlining) { | |
2751 masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL); | |
2752 } | |
2753 | |
2754 masm.movptr(tmpReg, Address(objReg, 0)) ; | |
2755 masm.cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD) ; | |
2756 masm.jcc (Assembler::zero, DONE_LABEL) ; | |
2757 masm.testl (tmpReg, 0x02) ; | |
2758 masm.jcc (Assembler::zero, Stacked) ; | |
2759 | |
2760 // It's inflated | |
2761 masm.movptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; | |
2762 masm.xorptr(boxReg, r15_thread) ; | |
2763 masm.orptr (boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2)) ; | |
2764 masm.jcc (Assembler::notZero, DONE_LABEL) ; | |
2765 masm.movptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2)) ; | |
2766 masm.orptr (boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2)) ; | |
2767 masm.jcc (Assembler::notZero, CheckSucc) ; | |
2768 masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int32_t)NULL_WORD) ; | |
2769 masm.jmp (DONE_LABEL) ; | |
2770 | |
2771 if ((EmitSync & 65536) == 0) { | |
2772 Label LSuccess, LGoSlowPath ; | |
2773 masm.bind (CheckSucc) ; | |
2774 masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD) ; | |
2775 masm.jcc (Assembler::zero, LGoSlowPath) ; | |
2776 | |
2777 // I'd much rather use lock:andl m->_owner, 0 as it's faster than the | |
2778 // the explicit ST;MEMBAR combination, but masm doesn't currently support | |
2779 // "ANDQ M,IMM". Don't use MFENCE here. lock:add to TOS, xchg, etc | |
2780 // are all faster when the write buffer is populated. | |
2781 masm.movptr (Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int32_t)NULL_WORD) ; | |
2782 if (os::is_MP()) { | |
2783 masm.lock () ; masm.addl (Address(rsp, 0), 0) ; | |
2784 } | |
2785 masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD) ; | |
2786 masm.jcc (Assembler::notZero, LSuccess) ; | |
2787 | |
2788 masm.movptr (boxReg, (int32_t)NULL_WORD) ; // box is really EAX | |
2789 if (os::is_MP()) { masm.lock(); } | |
2790 masm.cmpxchgptr(r15_thread, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)); | |
2791 masm.jcc (Assembler::notEqual, LSuccess) ; | |
2792 // Intentional fall-through into slow-path | |
2793 | |
2794 masm.bind (LGoSlowPath) ; | |
2795 masm.orl (boxReg, 1) ; // set ICC.ZF=0 to indicate failure | |
2796 masm.jmp (DONE_LABEL) ; | |
2797 | |
2798 masm.bind (LSuccess) ; | |
2799 masm.testl (boxReg, 0) ; // set ICC.ZF=1 to indicate success | |
2800 masm.jmp (DONE_LABEL) ; | |
2801 } | |
2802 | |
2803 masm.bind (Stacked) ; | |
2804 masm.movptr(tmpReg, Address (boxReg, 0)) ; // re-fetch | |
2805 if (os::is_MP()) { masm.lock(); } | |
2806 masm.cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box | |
2807 | |
2808 if (EmitSync & 65536) { | |
2809 masm.bind (CheckSucc) ; | |
2810 } | |
2811 masm.bind(DONE_LABEL); | |
2812 if (EmitSync & 32768) { | |
2813 masm.nop(); // avoid branch to branch | |
2814 } | |
2815 } | |
2816 %} | |
2817 | |
2818 | |
2819 enc_class enc_rethrow() | 2594 enc_class enc_rethrow() |
2820 %{ | 2595 %{ |
2821 cbuf.set_insts_mark(); | 2596 cbuf.set_insts_mark(); |
2822 emit_opcode(cbuf, 0xE9); // jmp entry | 2597 emit_opcode(cbuf, 0xE9); // jmp entry |
2823 emit_d32_reloc(cbuf, | 2598 emit_d32_reloc(cbuf, |
11441 %} | 11216 %} |
11442 | 11217 |
11443 // ============================================================================ | 11218 // ============================================================================ |
11444 // inlined locking and unlocking | 11219 // inlined locking and unlocking |
11445 | 11220 |
11446 instruct cmpFastLock(rFlagsReg cr, | 11221 instruct cmpFastLock(rFlagsReg cr, rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) %{ |
11447 rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr) | |
11448 %{ | |
11449 match(Set cr (FastLock object box)); | 11222 match(Set cr (FastLock object box)); |
11450 effect(TEMP tmp, TEMP scr, USE_KILL box); | 11223 effect(TEMP tmp, TEMP scr, USE_KILL box); |
11451 | |
11452 ins_cost(300); | 11224 ins_cost(300); |
11453 format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %} | 11225 format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %} |
11454 ins_encode(Fast_Lock(object, box, tmp, scr)); | 11226 ins_encode %{ |
11227 __ fast_lock($object$$Register, $box$$Register, $tmp$$Register, $scr$$Register, _counters); | |
11228 %} | |
11455 ins_pipe(pipe_slow); | 11229 ins_pipe(pipe_slow); |
11456 %} | 11230 %} |
11457 | 11231 |
11458 instruct cmpFastUnlock(rFlagsReg cr, | 11232 instruct cmpFastUnlock(rFlagsReg cr, rRegP object, rax_RegP box, rRegP tmp) %{ |
11459 rRegP object, rax_RegP box, rRegP tmp) | |
11460 %{ | |
11461 match(Set cr (FastUnlock object box)); | 11233 match(Set cr (FastUnlock object box)); |
11462 effect(TEMP tmp, USE_KILL box); | 11234 effect(TEMP tmp, USE_KILL box); |
11463 | |
11464 ins_cost(300); | 11235 ins_cost(300); |
11465 format %{ "fastunlock $object,$box\t! kills $box,$tmp" %} | 11236 format %{ "fastunlock $object,$box\t! kills $box,$tmp" %} |
11466 ins_encode(Fast_Unlock(object, box, tmp)); | 11237 ins_encode %{ |
11238 __ fast_unlock($object$$Register, $box$$Register, $tmp$$Register); | |
11239 %} | |
11467 ins_pipe(pipe_slow); | 11240 ins_pipe(pipe_slow); |
11468 %} | 11241 %} |
11469 | 11242 |
11470 | 11243 |
11471 // ============================================================================ | 11244 // ============================================================================ |