Mercurial > hg > graal-compiler
comparison src/cpu/x86/vm/x86_64.ad @ 304:dc7f315e41f7
5108146: Merge i486 and amd64 cpu directories
6459804: Want client (c1) compiler for x86_64 (amd64) for faster start-up
Reviewed-by: kvn
author | never |
---|---|
date | Wed, 27 Aug 2008 00:21:55 -0700 |
parents | 9c2ecc2ffb12 |
children | b744678d4d71 |
comparison
equal
deleted
inserted
replaced
303:fa4d1d240383 | 304:dc7f315e41f7 |
---|---|
476 R13, | 476 R13, |
477 R14); | 477 R14); |
478 | 478 |
479 // Class for all int registers except RAX, RDX (and RSP) | 479 // Class for all int registers except RAX, RDX (and RSP) |
480 reg_class int_no_rax_rdx_reg(RBP, | 480 reg_class int_no_rax_rdx_reg(RBP, |
481 RDI | 481 RDI, |
482 RSI, | 482 RSI, |
483 RCX, | 483 RCX, |
484 RBX, | 484 RBX, |
485 R8, | 485 R8, |
486 R9, | 486 R9, |
550 | 550 |
551 //----------SOURCE BLOCK------------------------------------------------------- | 551 //----------SOURCE BLOCK------------------------------------------------------- |
552 // This is a block of C++ code which provides values, functions, and | 552 // This is a block of C++ code which provides values, functions, and |
553 // definitions necessary in the rest of the architecture description | 553 // definitions necessary in the rest of the architecture description |
554 source %{ | 554 source %{ |
555 #define RELOC_IMM64 Assembler::imm64_operand | 555 #define RELOC_IMM64 Assembler::imm_operand |
556 #define RELOC_DISP32 Assembler::disp32_operand | 556 #define RELOC_DISP32 Assembler::disp32_operand |
557 | 557 |
558 #define __ _masm. | 558 #define __ _masm. |
559 | 559 |
560 // !!!!! Special hack to get all types of calls to specify the byte offset | 560 // !!!!! Special hack to get all types of calls to specify the byte offset |
960 | 960 |
961 #ifdef ASSERT | 961 #ifdef ASSERT |
962 if (VerifyStackAtCalls) { | 962 if (VerifyStackAtCalls) { |
963 Label L; | 963 Label L; |
964 MacroAssembler masm(&cbuf); | 964 MacroAssembler masm(&cbuf); |
965 masm.pushq(rax); | 965 masm.push(rax); |
966 masm.movq(rax, rsp); | 966 masm.mov(rax, rsp); |
967 masm.andq(rax, StackAlignmentInBytes-1); | 967 masm.andptr(rax, StackAlignmentInBytes-1); |
968 masm.cmpq(rax, StackAlignmentInBytes-wordSize); | 968 masm.cmpptr(rax, StackAlignmentInBytes-wordSize); |
969 masm.popq(rax); | 969 masm.pop(rax); |
970 masm.jcc(Assembler::equal, L); | 970 masm.jcc(Assembler::equal, L); |
971 masm.stop("Stack is not properly aligned!"); | 971 masm.stop("Stack is not properly aligned!"); |
972 masm.bind(L); | 972 masm.bind(L); |
973 } | 973 } |
974 #endif | 974 #endif |
1815 if (base == NULL) return; // CodeBuffer::expand failed | 1815 if (base == NULL) return; // CodeBuffer::expand failed |
1816 // static stub relocation stores the instruction address of the call | 1816 // static stub relocation stores the instruction address of the call |
1817 __ relocate(static_stub_Relocation::spec(mark), RELOC_IMM64); | 1817 __ relocate(static_stub_Relocation::spec(mark), RELOC_IMM64); |
1818 // static stub relocation also tags the methodOop in the code-stream. | 1818 // static stub relocation also tags the methodOop in the code-stream. |
1819 __ movoop(rbx, (jobject) NULL); // method is zapped till fixup time | 1819 __ movoop(rbx, (jobject) NULL); // method is zapped till fixup time |
1820 // This is recognized as unresolved by relocs/nativeinst/ic code | |
1820 __ jump(RuntimeAddress(__ pc())); | 1821 __ jump(RuntimeAddress(__ pc())); |
1821 | 1822 |
1822 // Update current stubs pointer and restore code_end. | 1823 // Update current stubs pointer and restore code_end. |
1823 __ end_a_stub(); | 1824 __ end_a_stub(); |
1824 } | 1825 } |
1861 #ifdef ASSERT | 1862 #ifdef ASSERT |
1862 uint code_size = cbuf.code_size(); | 1863 uint code_size = cbuf.code_size(); |
1863 #endif | 1864 #endif |
1864 if (UseCompressedOops) { | 1865 if (UseCompressedOops) { |
1865 masm.load_klass(rscratch1, j_rarg0); | 1866 masm.load_klass(rscratch1, j_rarg0); |
1866 masm.cmpq(rax, rscratch1); | 1867 masm.cmpptr(rax, rscratch1); |
1867 } else { | 1868 } else { |
1868 masm.cmpq(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes())); | 1869 masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes())); |
1869 } | 1870 } |
1870 | 1871 |
1871 masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub())); | 1872 masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub())); |
1872 | 1873 |
1873 /* WARNING these NOPs are critical so that verified entry point is properly | 1874 /* WARNING these NOPs are critical so that verified entry point is properly |
1947 | 1948 |
1948 // push address of "next" | 1949 // push address of "next" |
1949 __ call(next, relocInfo::none); // reloc none is fine since it is a disp32 | 1950 __ call(next, relocInfo::none); // reloc none is fine since it is a disp32 |
1950 __ bind(next); | 1951 __ bind(next); |
1951 // adjust it so it matches "the_pc" | 1952 // adjust it so it matches "the_pc" |
1952 __ subq(Address(rsp, 0), __ offset() - offset); | 1953 __ subptr(Address(rsp, 0), __ offset() - offset); |
1953 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); | 1954 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); |
1954 assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow"); | 1955 assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow"); |
1955 __ end_a_stub(); | 1956 __ end_a_stub(); |
1956 return offset; | 1957 return offset; |
1957 } | 1958 } |
2575 | 2576 |
2576 MacroAssembler _masm(&cbuf); | 2577 MacroAssembler _masm(&cbuf); |
2577 // Compare super with sub directly, since super is not in its own SSA. | 2578 // Compare super with sub directly, since super is not in its own SSA. |
2578 // The compiler used to emit this test, but we fold it in here, | 2579 // The compiler used to emit this test, but we fold it in here, |
2579 // to allow platform-specific tweaking on sparc. | 2580 // to allow platform-specific tweaking on sparc. |
2580 __ cmpq(Rrax, Rrsi); | 2581 __ cmpptr(Rrax, Rrsi); |
2581 __ jcc(Assembler::equal, hit); | 2582 __ jcc(Assembler::equal, hit); |
2582 #ifndef PRODUCT | 2583 #ifndef PRODUCT |
2583 __ lea(Rrcx, ExternalAddress((address)&SharedRuntime::_partial_subtype_ctr)); | 2584 __ lea(Rrcx, ExternalAddress((address)&SharedRuntime::_partial_subtype_ctr)); |
2584 __ incrementl(Address(Rrcx, 0)); | 2585 __ incrementl(Address(Rrcx, 0)); |
2585 #endif //PRODUCT | 2586 #endif //PRODUCT |
2586 __ movq(Rrdi, Address(Rrsi, | 2587 __ movptr(Rrdi, Address(Rrsi, |
2587 sizeof(oopDesc) + | 2588 sizeof(oopDesc) + |
2588 Klass::secondary_supers_offset_in_bytes())); | 2589 Klass::secondary_supers_offset_in_bytes())); |
2589 __ movl(Rrcx, Address(Rrdi, arrayOopDesc::length_offset_in_bytes())); | 2590 __ movl(Rrcx, Address(Rrdi, arrayOopDesc::length_offset_in_bytes())); |
2590 __ addq(Rrdi, arrayOopDesc::base_offset_in_bytes(T_OBJECT)); | 2591 __ addptr(Rrdi, arrayOopDesc::base_offset_in_bytes(T_OBJECT)); |
2591 if (UseCompressedOops) { | 2592 if (UseCompressedOops) { |
2592 __ encode_heap_oop(Rrax); | 2593 __ encode_heap_oop(Rrax); |
2593 __ repne_scanl(); | 2594 __ repne_scanl(); |
2594 __ jcc(Assembler::notEqual, cmiss); | 2595 __ jcc(Assembler::notEqual, cmiss); |
2595 __ decode_heap_oop(Rrax); | 2596 __ decode_heap_oop(Rrax); |
2596 __ movq(Address(Rrsi, | 2597 __ movptr(Address(Rrsi, |
2597 sizeof(oopDesc) + | 2598 sizeof(oopDesc) + |
2598 Klass::secondary_super_cache_offset_in_bytes()), | 2599 Klass::secondary_super_cache_offset_in_bytes()), |
2599 Rrax); | 2600 Rrax); |
2600 __ jmp(hit); | 2601 __ jmp(hit); |
2601 __ bind(cmiss); | 2602 __ bind(cmiss); |
2602 __ decode_heap_oop(Rrax); | 2603 __ decode_heap_oop(Rrax); |
2603 __ jmp(miss); | 2604 __ jmp(miss); |
2604 } else { | 2605 } else { |
2605 __ repne_scanq(); | 2606 __ repne_scan(); |
2606 __ jcc(Assembler::notEqual, miss); | 2607 __ jcc(Assembler::notEqual, miss); |
2607 __ movq(Address(Rrsi, | 2608 __ movptr(Address(Rrsi, |
2608 sizeof(oopDesc) + | 2609 sizeof(oopDesc) + |
2609 Klass::secondary_super_cache_offset_in_bytes()), | 2610 Klass::secondary_super_cache_offset_in_bytes()), |
2610 Rrax); | 2611 Rrax); |
2611 } | 2612 } |
2612 __ bind(hit); | 2613 __ bind(hit); |
2613 if ($primary) { | 2614 if ($primary) { |
2614 __ xorq(Rrdi, Rrdi); | 2615 __ xorptr(Rrdi, Rrdi); |
2615 } | 2616 } |
2616 __ bind(miss); | 2617 __ bind(miss); |
2617 %} | 2618 %} |
2618 | 2619 |
2619 enc_class Java_To_Interpreter(method meth) | 2620 enc_class Java_To_Interpreter(method meth) |
3525 | 3526 |
3526 if (_counters != NULL) { | 3527 if (_counters != NULL) { |
3527 masm.atomic_incl(ExternalAddress((address) _counters->total_entry_count_addr())); | 3528 masm.atomic_incl(ExternalAddress((address) _counters->total_entry_count_addr())); |
3528 } | 3529 } |
3529 if (EmitSync & 1) { | 3530 if (EmitSync & 1) { |
3530 masm.movptr (Address(boxReg, 0), intptr_t(markOopDesc::unused_mark())) ; | 3531 // Without cast to int32_t a movptr will destroy r10 which is typically obj |
3531 masm.cmpq (rsp, 0) ; | 3532 masm.movptr (Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark())) ; |
3533 masm.cmpptr(rsp, (int32_t)NULL_WORD) ; | |
3532 } else | 3534 } else |
3533 if (EmitSync & 2) { | 3535 if (EmitSync & 2) { |
3534 Label DONE_LABEL; | 3536 Label DONE_LABEL; |
3535 if (UseBiasedLocking) { | 3537 if (UseBiasedLocking) { |
3536 // Note: tmpReg maps to the swap_reg argument and scrReg to the tmp_reg argument. | 3538 // Note: tmpReg maps to the swap_reg argument and scrReg to the tmp_reg argument. |
3537 masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, _counters); | 3539 masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, _counters); |
3538 } | 3540 } |
3539 masm.movl(tmpReg, 0x1); | 3541 // QQQ was movl... |
3540 masm.orq(tmpReg, Address(objReg, 0)); | 3542 masm.movptr(tmpReg, 0x1); |
3541 masm.movq(Address(boxReg, 0), tmpReg); | 3543 masm.orptr(tmpReg, Address(objReg, 0)); |
3544 masm.movptr(Address(boxReg, 0), tmpReg); | |
3542 if (os::is_MP()) { | 3545 if (os::is_MP()) { |
3543 masm.lock(); | 3546 masm.lock(); |
3544 } | 3547 } |
3545 masm.cmpxchgq(boxReg, Address(objReg, 0)); // Updates tmpReg | 3548 masm.cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg |
3546 masm.jcc(Assembler::equal, DONE_LABEL); | 3549 masm.jcc(Assembler::equal, DONE_LABEL); |
3547 | 3550 |
3548 // Recursive locking | 3551 // Recursive locking |
3549 masm.subq(tmpReg, rsp); | 3552 masm.subptr(tmpReg, rsp); |
3550 masm.andq(tmpReg, 7 - os::vm_page_size()); | 3553 masm.andptr(tmpReg, 7 - os::vm_page_size()); |
3551 masm.movq(Address(boxReg, 0), tmpReg); | 3554 masm.movptr(Address(boxReg, 0), tmpReg); |
3552 | 3555 |
3553 masm.bind(DONE_LABEL); | 3556 masm.bind(DONE_LABEL); |
3554 masm.nop(); // avoid branch to branch | 3557 masm.nop(); // avoid branch to branch |
3555 } else { | 3558 } else { |
3556 Label DONE_LABEL, IsInflated, Egress; | 3559 Label DONE_LABEL, IsInflated, Egress; |
3557 | 3560 |
3558 masm.movq (tmpReg, Address(objReg, 0)) ; | 3561 masm.movptr(tmpReg, Address(objReg, 0)) ; |
3559 masm.testq (tmpReg, 0x02) ; // inflated vs stack-locked|neutral|biased | 3562 masm.testl (tmpReg, 0x02) ; // inflated vs stack-locked|neutral|biased |
3560 masm.jcc (Assembler::notZero, IsInflated) ; | 3563 masm.jcc (Assembler::notZero, IsInflated) ; |
3561 | 3564 |
3562 // it's stack-locked, biased or neutral | 3565 // it's stack-locked, biased or neutral |
3563 // TODO: optimize markword triage order to reduce the number of | 3566 // TODO: optimize markword triage order to reduce the number of |
3564 // conditional branches in the most common cases. | 3567 // conditional branches in the most common cases. |
3565 // Beware -- there's a subtle invariant that fetch of the markword | 3568 // Beware -- there's a subtle invariant that fetch of the markword |
3566 // at [FETCH], below, will never observe a biased encoding (*101b). | 3569 // at [FETCH], below, will never observe a biased encoding (*101b). |
3567 // If this invariant is not held we'll suffer exclusion (safety) failure. | 3570 // If this invariant is not held we'll suffer exclusion (safety) failure. |
3568 | 3571 |
3569 if (UseBiasedLocking) { | 3572 if (UseBiasedLocking) { |
3570 masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, true, DONE_LABEL, NULL, _counters); | 3573 masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, true, DONE_LABEL, NULL, _counters); |
3571 masm.movq (tmpReg, Address(objReg, 0)) ; // [FETCH] | 3574 masm.movptr(tmpReg, Address(objReg, 0)) ; // [FETCH] |
3572 } | 3575 } |
3573 | 3576 |
3574 masm.orq (tmpReg, 1) ; | 3577 // was q will it destroy high? |
3575 masm.movq (Address(boxReg, 0), tmpReg) ; | 3578 masm.orl (tmpReg, 1) ; |
3576 if (os::is_MP()) { masm.lock(); } | 3579 masm.movptr(Address(boxReg, 0), tmpReg) ; |
3577 masm.cmpxchgq(boxReg, Address(objReg, 0)); // Updates tmpReg | 3580 if (os::is_MP()) { masm.lock(); } |
3581 masm.cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg | |
3578 if (_counters != NULL) { | 3582 if (_counters != NULL) { |
3579 masm.cond_inc32(Assembler::equal, | 3583 masm.cond_inc32(Assembler::equal, |
3580 ExternalAddress((address) _counters->fast_path_entry_count_addr())); | 3584 ExternalAddress((address) _counters->fast_path_entry_count_addr())); |
3581 } | 3585 } |
3582 masm.jcc (Assembler::equal, DONE_LABEL); | 3586 masm.jcc (Assembler::equal, DONE_LABEL); |
3583 | 3587 |
3584 // Recursive locking | 3588 // Recursive locking |
3585 masm.subq (tmpReg, rsp); | 3589 masm.subptr(tmpReg, rsp); |
3586 masm.andq (tmpReg, 7 - os::vm_page_size()); | 3590 masm.andptr(tmpReg, 7 - os::vm_page_size()); |
3587 masm.movq (Address(boxReg, 0), tmpReg); | 3591 masm.movptr(Address(boxReg, 0), tmpReg); |
3588 if (_counters != NULL) { | 3592 if (_counters != NULL) { |
3589 masm.cond_inc32(Assembler::equal, | 3593 masm.cond_inc32(Assembler::equal, |
3590 ExternalAddress((address) _counters->fast_path_entry_count_addr())); | 3594 ExternalAddress((address) _counters->fast_path_entry_count_addr())); |
3591 } | 3595 } |
3592 masm.jmp (DONE_LABEL) ; | 3596 masm.jmp (DONE_LABEL) ; |
3597 // TODO: someday avoid the ST-before-CAS penalty by | 3601 // TODO: someday avoid the ST-before-CAS penalty by |
3598 // relocating (deferring) the following ST. | 3602 // relocating (deferring) the following ST. |
3599 // We should also think about trying a CAS without having | 3603 // We should also think about trying a CAS without having |
3600 // fetched _owner. If the CAS is successful we may | 3604 // fetched _owner. If the CAS is successful we may |
3601 // avoid an RTO->RTS upgrade on the $line. | 3605 // avoid an RTO->RTS upgrade on the $line. |
3602 masm.movptr(Address(boxReg, 0), intptr_t(markOopDesc::unused_mark())) ; | 3606 // Without cast to int32_t a movptr will destroy r10 which is typically obj |
3603 | 3607 masm.movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark())) ; |
3604 masm.movq (boxReg, tmpReg) ; | 3608 |
3605 masm.movq (tmpReg, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; | 3609 masm.mov (boxReg, tmpReg) ; |
3606 masm.testq (tmpReg, tmpReg) ; | 3610 masm.movptr (tmpReg, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; |
3607 masm.jcc (Assembler::notZero, DONE_LABEL) ; | 3611 masm.testptr(tmpReg, tmpReg) ; |
3612 masm.jcc (Assembler::notZero, DONE_LABEL) ; | |
3608 | 3613 |
3609 // It's inflated and appears unlocked | 3614 // It's inflated and appears unlocked |
3610 if (os::is_MP()) { masm.lock(); } | 3615 if (os::is_MP()) { masm.lock(); } |
3611 masm.cmpxchgq(r15_thread, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; | 3616 masm.cmpxchgptr(r15_thread, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; |
3612 // Intentional fall-through into DONE_LABEL ... | 3617 // Intentional fall-through into DONE_LABEL ... |
3613 | 3618 |
3614 masm.bind (DONE_LABEL) ; | 3619 masm.bind (DONE_LABEL) ; |
3615 masm.nop () ; // avoid jmp to jmp | 3620 masm.nop () ; // avoid jmp to jmp |
3616 } | 3621 } |
3625 Register objReg = as_Register($obj$$reg); | 3630 Register objReg = as_Register($obj$$reg); |
3626 Register boxReg = as_Register($box$$reg); | 3631 Register boxReg = as_Register($box$$reg); |
3627 Register tmpReg = as_Register($tmp$$reg); | 3632 Register tmpReg = as_Register($tmp$$reg); |
3628 MacroAssembler masm(&cbuf); | 3633 MacroAssembler masm(&cbuf); |
3629 | 3634 |
3630 if (EmitSync & 4) { | 3635 if (EmitSync & 4) { |
3631 masm.cmpq (rsp, 0) ; | 3636 masm.cmpptr(rsp, 0) ; |
3632 } else | 3637 } else |
3633 if (EmitSync & 8) { | 3638 if (EmitSync & 8) { |
3634 Label DONE_LABEL; | 3639 Label DONE_LABEL; |
3635 if (UseBiasedLocking) { | 3640 if (UseBiasedLocking) { |
3636 masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL); | 3641 masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL); |
3637 } | 3642 } |
3638 | 3643 |
3639 // Check whether the displaced header is 0 | 3644 // Check whether the displaced header is 0 |
3640 //(=> recursive unlock) | 3645 //(=> recursive unlock) |
3641 masm.movq(tmpReg, Address(boxReg, 0)); | 3646 masm.movptr(tmpReg, Address(boxReg, 0)); |
3642 masm.testq(tmpReg, tmpReg); | 3647 masm.testptr(tmpReg, tmpReg); |
3643 masm.jcc(Assembler::zero, DONE_LABEL); | 3648 masm.jcc(Assembler::zero, DONE_LABEL); |
3644 | 3649 |
3645 // If not recursive lock, reset the header to displaced header | 3650 // If not recursive lock, reset the header to displaced header |
3646 if (os::is_MP()) { | 3651 if (os::is_MP()) { |
3647 masm.lock(); | 3652 masm.lock(); |
3648 } | 3653 } |
3649 masm.cmpxchgq(tmpReg, Address(objReg, 0)); // Uses RAX which is box | 3654 masm.cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box |
3650 masm.bind(DONE_LABEL); | 3655 masm.bind(DONE_LABEL); |
3651 masm.nop(); // avoid branch to branch | 3656 masm.nop(); // avoid branch to branch |
3652 } else { | 3657 } else { |
3653 Label DONE_LABEL, Stacked, CheckSucc ; | 3658 Label DONE_LABEL, Stacked, CheckSucc ; |
3654 | 3659 |
3655 if (UseBiasedLocking) { | 3660 if (UseBiasedLocking) { |
3656 masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL); | 3661 masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL); |
3657 } | 3662 } |
3658 | 3663 |
3659 masm.movq (tmpReg, Address(objReg, 0)) ; | 3664 masm.movptr(tmpReg, Address(objReg, 0)) ; |
3660 masm.cmpq (Address(boxReg, 0), (int)NULL_WORD) ; | 3665 masm.cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD) ; |
3661 masm.jcc (Assembler::zero, DONE_LABEL) ; | 3666 masm.jcc (Assembler::zero, DONE_LABEL) ; |
3662 masm.testq (tmpReg, 0x02) ; | 3667 masm.testl (tmpReg, 0x02) ; |
3663 masm.jcc (Assembler::zero, Stacked) ; | 3668 masm.jcc (Assembler::zero, Stacked) ; |
3664 | 3669 |
3665 // It's inflated | 3670 // It's inflated |
3666 masm.movq (boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; | 3671 masm.movptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; |
3667 masm.xorq (boxReg, r15_thread) ; | 3672 masm.xorptr(boxReg, r15_thread) ; |
3668 masm.orq (boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2)) ; | 3673 masm.orptr (boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2)) ; |
3669 masm.jcc (Assembler::notZero, DONE_LABEL) ; | 3674 masm.jcc (Assembler::notZero, DONE_LABEL) ; |
3670 masm.movq (boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2)) ; | 3675 masm.movptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2)) ; |
3671 masm.orq (boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2)) ; | 3676 masm.orptr (boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2)) ; |
3672 masm.jcc (Assembler::notZero, CheckSucc) ; | 3677 masm.jcc (Assembler::notZero, CheckSucc) ; |
3673 masm.mov64 (Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int)NULL_WORD) ; | 3678 masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int32_t)NULL_WORD) ; |
3674 masm.jmp (DONE_LABEL) ; | 3679 masm.jmp (DONE_LABEL) ; |
3675 | 3680 |
3676 if ((EmitSync & 65536) == 0) { | 3681 if ((EmitSync & 65536) == 0) { |
3677 Label LSuccess, LGoSlowPath ; | 3682 Label LSuccess, LGoSlowPath ; |
3678 masm.bind (CheckSucc) ; | 3683 masm.bind (CheckSucc) ; |
3679 masm.cmpq (Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int)NULL_WORD) ; | 3684 masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD) ; |
3680 masm.jcc (Assembler::zero, LGoSlowPath) ; | 3685 masm.jcc (Assembler::zero, LGoSlowPath) ; |
3681 | 3686 |
3682 // I'd much rather use lock:andl m->_owner, 0 as it's faster than the | 3687 // I'd much rather use lock:andl m->_owner, 0 as it's faster than the |
3683 // the explicit ST;MEMBAR combination, but masm doesn't currently support | 3688 // the explicit ST;MEMBAR combination, but masm doesn't currently support |
3684 // "ANDQ M,IMM". Don't use MFENCE here. lock:add to TOS, xchg, etc | 3689 // "ANDQ M,IMM". Don't use MFENCE here. lock:add to TOS, xchg, etc |
3685 // are all faster when the write buffer is populated. | 3690 // are all faster when the write buffer is populated. |
3686 masm.movptr (Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int)NULL_WORD) ; | 3691 masm.movptr (Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int32_t)NULL_WORD) ; |
3687 if (os::is_MP()) { | 3692 if (os::is_MP()) { |
3688 masm.lock () ; masm.addq (Address(rsp, 0), 0) ; | 3693 masm.lock () ; masm.addl (Address(rsp, 0), 0) ; |
3689 } | 3694 } |
3690 masm.cmpq (Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int)NULL_WORD) ; | 3695 masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD) ; |
3691 masm.jcc (Assembler::notZero, LSuccess) ; | 3696 masm.jcc (Assembler::notZero, LSuccess) ; |
3692 | 3697 |
3693 masm.movptr (boxReg, (int)NULL_WORD) ; // box is really EAX | 3698 masm.movptr (boxReg, (int32_t)NULL_WORD) ; // box is really EAX |
3694 if (os::is_MP()) { masm.lock(); } | 3699 if (os::is_MP()) { masm.lock(); } |
3695 masm.cmpxchgq (r15_thread, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)); | 3700 masm.cmpxchgptr(r15_thread, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)); |
3696 masm.jcc (Assembler::notEqual, LSuccess) ; | 3701 masm.jcc (Assembler::notEqual, LSuccess) ; |
3697 // Intentional fall-through into slow-path | 3702 // Intentional fall-through into slow-path |
3698 | 3703 |
3699 masm.bind (LGoSlowPath) ; | 3704 masm.bind (LGoSlowPath) ; |
3700 masm.orl (boxReg, 1) ; // set ICC.ZF=0 to indicate failure | 3705 masm.orl (boxReg, 1) ; // set ICC.ZF=0 to indicate failure |
3703 masm.bind (LSuccess) ; | 3708 masm.bind (LSuccess) ; |
3704 masm.testl (boxReg, 0) ; // set ICC.ZF=1 to indicate success | 3709 masm.testl (boxReg, 0) ; // set ICC.ZF=1 to indicate success |
3705 masm.jmp (DONE_LABEL) ; | 3710 masm.jmp (DONE_LABEL) ; |
3706 } | 3711 } |
3707 | 3712 |
3708 masm.bind (Stacked) ; | 3713 masm.bind (Stacked) ; |
3709 masm.movq (tmpReg, Address (boxReg, 0)) ; // re-fetch | 3714 masm.movptr(tmpReg, Address (boxReg, 0)) ; // re-fetch |
3710 if (os::is_MP()) { masm.lock(); } | 3715 if (os::is_MP()) { masm.lock(); } |
3711 masm.cmpxchgq(tmpReg, Address(objReg, 0)); // Uses RAX which is box | 3716 masm.cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box |
3712 | 3717 |
3713 if (EmitSync & 65536) { | 3718 if (EmitSync & 65536) { |
3714 masm.bind (CheckSucc) ; | 3719 masm.bind (CheckSucc) ; |
3715 } | 3720 } |
3716 masm.bind(DONE_LABEL); | 3721 masm.bind(DONE_LABEL); |
3734 int count_offset = java_lang_String::count_offset_in_bytes(); | 3739 int count_offset = java_lang_String::count_offset_in_bytes(); |
3735 int base_offset = arrayOopDesc::base_offset_in_bytes(T_CHAR); | 3740 int base_offset = arrayOopDesc::base_offset_in_bytes(T_CHAR); |
3736 | 3741 |
3737 masm.load_heap_oop(rax, Address(rsi, value_offset)); | 3742 masm.load_heap_oop(rax, Address(rsi, value_offset)); |
3738 masm.movl(rcx, Address(rsi, offset_offset)); | 3743 masm.movl(rcx, Address(rsi, offset_offset)); |
3739 masm.leaq(rax, Address(rax, rcx, Address::times_2, base_offset)); | 3744 masm.lea(rax, Address(rax, rcx, Address::times_2, base_offset)); |
3740 masm.load_heap_oop(rbx, Address(rdi, value_offset)); | 3745 masm.load_heap_oop(rbx, Address(rdi, value_offset)); |
3741 masm.movl(rcx, Address(rdi, offset_offset)); | 3746 masm.movl(rcx, Address(rdi, offset_offset)); |
3742 masm.leaq(rbx, Address(rbx, rcx, Address::times_2, base_offset)); | 3747 masm.lea(rbx, Address(rbx, rcx, Address::times_2, base_offset)); |
3743 | 3748 |
3744 // Compute the minimum of the string lengths(rsi) and the | 3749 // Compute the minimum of the string lengths(rsi) and the |
3745 // difference of the string lengths (stack) | 3750 // difference of the string lengths (stack) |
3746 | 3751 |
3747 masm.movl(rdi, Address(rdi, count_offset)); | 3752 masm.movl(rdi, Address(rdi, count_offset)); |
3748 masm.movl(rsi, Address(rsi, count_offset)); | 3753 masm.movl(rsi, Address(rsi, count_offset)); |
3749 masm.movl(rcx, rdi); | 3754 masm.movl(rcx, rdi); |
3750 masm.subl(rdi, rsi); | 3755 masm.subl(rdi, rsi); |
3751 masm.pushq(rdi); | 3756 masm.push(rdi); |
3752 masm.cmovl(Assembler::lessEqual, rsi, rcx); | 3757 masm.cmov(Assembler::lessEqual, rsi, rcx); |
3753 | 3758 |
3754 // Is the minimum length zero? | 3759 // Is the minimum length zero? |
3755 masm.bind(RCX_GOOD_LABEL); | 3760 masm.bind(RCX_GOOD_LABEL); |
3756 masm.testl(rsi, rsi); | 3761 masm.testl(rsi, rsi); |
3757 masm.jcc(Assembler::zero, LENGTH_DIFF_LABEL); | 3762 masm.jcc(Assembler::zero, LENGTH_DIFF_LABEL); |
3768 | 3773 |
3769 { | 3774 { |
3770 // Check after comparing first character to see if strings are equivalent | 3775 // Check after comparing first character to see if strings are equivalent |
3771 Label LSkip2; | 3776 Label LSkip2; |
3772 // Check if the strings start at same location | 3777 // Check if the strings start at same location |
3773 masm.cmpq(rbx, rax); | 3778 masm.cmpptr(rbx, rax); |
3774 masm.jcc(Assembler::notEqual, LSkip2); | 3779 masm.jcc(Assembler::notEqual, LSkip2); |
3775 | 3780 |
3776 // Check if the length difference is zero (from stack) | 3781 // Check if the length difference is zero (from stack) |
3777 masm.cmpl(Address(rsp, 0), 0x0); | 3782 masm.cmpl(Address(rsp, 0), 0x0); |
3778 masm.jcc(Assembler::equal, LENGTH_DIFF_LABEL); | 3783 masm.jcc(Assembler::equal, LENGTH_DIFF_LABEL); |
3780 // Strings might not be equivalent | 3785 // Strings might not be equivalent |
3781 masm.bind(LSkip2); | 3786 masm.bind(LSkip2); |
3782 } | 3787 } |
3783 | 3788 |
3784 // Shift RAX and RBX to the end of the arrays, negate min | 3789 // Shift RAX and RBX to the end of the arrays, negate min |
3785 masm.leaq(rax, Address(rax, rsi, Address::times_2, 2)); | 3790 masm.lea(rax, Address(rax, rsi, Address::times_2, 2)); |
3786 masm.leaq(rbx, Address(rbx, rsi, Address::times_2, 2)); | 3791 masm.lea(rbx, Address(rbx, rsi, Address::times_2, 2)); |
3787 masm.negq(rsi); | 3792 masm.negptr(rsi); |
3788 | 3793 |
3789 // Compare the rest of the characters | 3794 // Compare the rest of the characters |
3790 masm.bind(WHILE_HEAD_LABEL); | 3795 masm.bind(WHILE_HEAD_LABEL); |
3791 masm.load_unsigned_word(rcx, Address(rbx, rsi, Address::times_2, 0)); | 3796 masm.load_unsigned_word(rcx, Address(rbx, rsi, Address::times_2, 0)); |
3792 masm.load_unsigned_word(rdi, Address(rax, rsi, Address::times_2, 0)); | 3797 masm.load_unsigned_word(rdi, Address(rax, rsi, Address::times_2, 0)); |
3793 masm.subl(rcx, rdi); | 3798 masm.subl(rcx, rdi); |
3794 masm.jcc(Assembler::notZero, POP_LABEL); | 3799 masm.jcc(Assembler::notZero, POP_LABEL); |
3795 masm.incrementq(rsi); | 3800 masm.increment(rsi); |
3796 masm.jcc(Assembler::notZero, WHILE_HEAD_LABEL); | 3801 masm.jcc(Assembler::notZero, WHILE_HEAD_LABEL); |
3797 | 3802 |
3798 // Strings are equal up to min length. Return the length difference. | 3803 // Strings are equal up to min length. Return the length difference. |
3799 masm.bind(LENGTH_DIFF_LABEL); | 3804 masm.bind(LENGTH_DIFF_LABEL); |
3800 masm.popq(rcx); | 3805 masm.pop(rcx); |
3801 masm.jmp(DONE_LABEL); | 3806 masm.jmp(DONE_LABEL); |
3802 | 3807 |
3803 // Discard the stored length difference | 3808 // Discard the stored length difference |
3804 masm.bind(POP_LABEL); | 3809 masm.bind(POP_LABEL); |
3805 masm.addq(rsp, 8); | 3810 masm.addptr(rsp, 8); |
3806 | 3811 |
3807 // That's it | 3812 // That's it |
3808 masm.bind(DONE_LABEL); | 3813 masm.bind(DONE_LABEL); |
3809 %} | 3814 %} |
3810 | 3815 |
3811 enc_class enc_Array_Equals(rdi_RegP ary1, rsi_RegP ary2, rax_RegI tmp1, rbx_RegI tmp2, rcx_RegI result) %{ | 3816 enc_class enc_Array_Equals(rdi_RegP ary1, rsi_RegP ary2, rax_RegI tmp1, rbx_RegI tmp2, rcx_RegI result) %{ |
3891 %} | 3896 %} |
3892 | 3897 |
3893 enc_class absF_encoding(regF dst) | 3898 enc_class absF_encoding(regF dst) |
3894 %{ | 3899 %{ |
3895 int dstenc = $dst$$reg; | 3900 int dstenc = $dst$$reg; |
3896 address signmask_address = (address) StubRoutines::amd64::float_sign_mask(); | 3901 address signmask_address = (address) StubRoutines::x86::float_sign_mask(); |
3897 | 3902 |
3898 cbuf.set_inst_mark(); | 3903 cbuf.set_inst_mark(); |
3899 if (dstenc >= 8) { | 3904 if (dstenc >= 8) { |
3900 emit_opcode(cbuf, Assembler::REX_R); | 3905 emit_opcode(cbuf, Assembler::REX_R); |
3901 dstenc -= 8; | 3906 dstenc -= 8; |
3908 %} | 3913 %} |
3909 | 3914 |
3910 enc_class absD_encoding(regD dst) | 3915 enc_class absD_encoding(regD dst) |
3911 %{ | 3916 %{ |
3912 int dstenc = $dst$$reg; | 3917 int dstenc = $dst$$reg; |
3913 address signmask_address = (address) StubRoutines::amd64::double_sign_mask(); | 3918 address signmask_address = (address) StubRoutines::x86::double_sign_mask(); |
3914 | 3919 |
3915 cbuf.set_inst_mark(); | 3920 cbuf.set_inst_mark(); |
3916 emit_opcode(cbuf, 0x66); | 3921 emit_opcode(cbuf, 0x66); |
3917 if (dstenc >= 8) { | 3922 if (dstenc >= 8) { |
3918 emit_opcode(cbuf, Assembler::REX_R); | 3923 emit_opcode(cbuf, Assembler::REX_R); |
3926 %} | 3931 %} |
3927 | 3932 |
3928 enc_class negF_encoding(regF dst) | 3933 enc_class negF_encoding(regF dst) |
3929 %{ | 3934 %{ |
3930 int dstenc = $dst$$reg; | 3935 int dstenc = $dst$$reg; |
3931 address signflip_address = (address) StubRoutines::amd64::float_sign_flip(); | 3936 address signflip_address = (address) StubRoutines::x86::float_sign_flip(); |
3932 | 3937 |
3933 cbuf.set_inst_mark(); | 3938 cbuf.set_inst_mark(); |
3934 if (dstenc >= 8) { | 3939 if (dstenc >= 8) { |
3935 emit_opcode(cbuf, Assembler::REX_R); | 3940 emit_opcode(cbuf, Assembler::REX_R); |
3936 dstenc -= 8; | 3941 dstenc -= 8; |
3943 %} | 3948 %} |
3944 | 3949 |
3945 enc_class negD_encoding(regD dst) | 3950 enc_class negD_encoding(regD dst) |
3946 %{ | 3951 %{ |
3947 int dstenc = $dst$$reg; | 3952 int dstenc = $dst$$reg; |
3948 address signflip_address = (address) StubRoutines::amd64::double_sign_flip(); | 3953 address signflip_address = (address) StubRoutines::x86::double_sign_flip(); |
3949 | 3954 |
3950 cbuf.set_inst_mark(); | 3955 cbuf.set_inst_mark(); |
3951 emit_opcode(cbuf, 0x66); | 3956 emit_opcode(cbuf, 0x66); |
3952 if (dstenc >= 8) { | 3957 if (dstenc >= 8) { |
3953 emit_opcode(cbuf, Assembler::REX_R); | 3958 emit_opcode(cbuf, Assembler::REX_R); |
4001 // call f2i_fixup | 4006 // call f2i_fixup |
4002 cbuf.set_inst_mark(); | 4007 cbuf.set_inst_mark(); |
4003 emit_opcode(cbuf, 0xE8); | 4008 emit_opcode(cbuf, 0xE8); |
4004 emit_d32_reloc(cbuf, | 4009 emit_d32_reloc(cbuf, |
4005 (int) | 4010 (int) |
4006 (StubRoutines::amd64::f2i_fixup() - cbuf.code_end() - 4), | 4011 (StubRoutines::x86::f2i_fixup() - cbuf.code_end() - 4), |
4007 runtime_call_Relocation::spec(), | 4012 runtime_call_Relocation::spec(), |
4008 RELOC_DISP32); | 4013 RELOC_DISP32); |
4009 | 4014 |
4010 // popq $dst | 4015 // popq $dst |
4011 if (dstenc >= 8) { | 4016 if (dstenc >= 8) { |
4018 | 4023 |
4019 enc_class f2l_fixup(rRegL dst, regF src) | 4024 enc_class f2l_fixup(rRegL dst, regF src) |
4020 %{ | 4025 %{ |
4021 int dstenc = $dst$$reg; | 4026 int dstenc = $dst$$reg; |
4022 int srcenc = $src$$reg; | 4027 int srcenc = $src$$reg; |
4023 address const_address = (address) StubRoutines::amd64::double_sign_flip(); | 4028 address const_address = (address) StubRoutines::x86::double_sign_flip(); |
4024 | 4029 |
4025 // cmpq $dst, [0x8000000000000000] | 4030 // cmpq $dst, [0x8000000000000000] |
4026 cbuf.set_inst_mark(); | 4031 cbuf.set_inst_mark(); |
4027 emit_opcode(cbuf, dstenc < 8 ? Assembler::REX_W : Assembler::REX_WR); | 4032 emit_opcode(cbuf, dstenc < 8 ? Assembler::REX_W : Assembler::REX_WR); |
4028 emit_opcode(cbuf, 0x39); | 4033 emit_opcode(cbuf, 0x39); |
4059 // call f2l_fixup | 4064 // call f2l_fixup |
4060 cbuf.set_inst_mark(); | 4065 cbuf.set_inst_mark(); |
4061 emit_opcode(cbuf, 0xE8); | 4066 emit_opcode(cbuf, 0xE8); |
4062 emit_d32_reloc(cbuf, | 4067 emit_d32_reloc(cbuf, |
4063 (int) | 4068 (int) |
4064 (StubRoutines::amd64::f2l_fixup() - cbuf.code_end() - 4), | 4069 (StubRoutines::x86::f2l_fixup() - cbuf.code_end() - 4), |
4065 runtime_call_Relocation::spec(), | 4070 runtime_call_Relocation::spec(), |
4066 RELOC_DISP32); | 4071 RELOC_DISP32); |
4067 | 4072 |
4068 // popq $dst | 4073 // popq $dst |
4069 if (dstenc >= 8) { | 4074 if (dstenc >= 8) { |
4115 // call d2i_fixup | 4120 // call d2i_fixup |
4116 cbuf.set_inst_mark(); | 4121 cbuf.set_inst_mark(); |
4117 emit_opcode(cbuf, 0xE8); | 4122 emit_opcode(cbuf, 0xE8); |
4118 emit_d32_reloc(cbuf, | 4123 emit_d32_reloc(cbuf, |
4119 (int) | 4124 (int) |
4120 (StubRoutines::amd64::d2i_fixup() - cbuf.code_end() - 4), | 4125 (StubRoutines::x86::d2i_fixup() - cbuf.code_end() - 4), |
4121 runtime_call_Relocation::spec(), | 4126 runtime_call_Relocation::spec(), |
4122 RELOC_DISP32); | 4127 RELOC_DISP32); |
4123 | 4128 |
4124 // popq $dst | 4129 // popq $dst |
4125 if (dstenc >= 8) { | 4130 if (dstenc >= 8) { |
4132 | 4137 |
4133 enc_class d2l_fixup(rRegL dst, regD src) | 4138 enc_class d2l_fixup(rRegL dst, regD src) |
4134 %{ | 4139 %{ |
4135 int dstenc = $dst$$reg; | 4140 int dstenc = $dst$$reg; |
4136 int srcenc = $src$$reg; | 4141 int srcenc = $src$$reg; |
4137 address const_address = (address) StubRoutines::amd64::double_sign_flip(); | 4142 address const_address = (address) StubRoutines::x86::double_sign_flip(); |
4138 | 4143 |
4139 // cmpq $dst, [0x8000000000000000] | 4144 // cmpq $dst, [0x8000000000000000] |
4140 cbuf.set_inst_mark(); | 4145 cbuf.set_inst_mark(); |
4141 emit_opcode(cbuf, dstenc < 8 ? Assembler::REX_W : Assembler::REX_WR); | 4146 emit_opcode(cbuf, dstenc < 8 ? Assembler::REX_W : Assembler::REX_WR); |
4142 emit_opcode(cbuf, 0x39); | 4147 emit_opcode(cbuf, 0x39); |
4173 // call d2l_fixup | 4178 // call d2l_fixup |
4174 cbuf.set_inst_mark(); | 4179 cbuf.set_inst_mark(); |
4175 emit_opcode(cbuf, 0xE8); | 4180 emit_opcode(cbuf, 0xE8); |
4176 emit_d32_reloc(cbuf, | 4181 emit_d32_reloc(cbuf, |
4177 (int) | 4182 (int) |
4178 (StubRoutines::amd64::d2l_fixup() - cbuf.code_end() - 4), | 4183 (StubRoutines::x86::d2l_fixup() - cbuf.code_end() - 4), |
4179 runtime_call_Relocation::spec(), | 4184 runtime_call_Relocation::spec(), |
4180 RELOC_DISP32); | 4185 RELOC_DISP32); |
4181 | 4186 |
4182 // popq $dst | 4187 // popq $dst |
4183 if (dstenc >= 8) { | 4188 if (dstenc >= 8) { |