Mercurial > hg > truffle

comparison src/cpu/x86/vm/x86_64.ad @ 304:dc7f315e41f7

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
5108146: Merge i486 and amd64 cpu directories 6459804: Want client (c1) compiler for x86_64 (amd64) for faster start-up Reviewed-by: kvn
author never
date Wed, 27 Aug 2008 00:21:55 -0700
parents 9c2ecc2ffb12
children b744678d4d71
comparison
equal deleted inserted replaced
303:fa4d1d240383 304:dc7f315e41f7
476 R13, 476 R13,
477 R14); 477 R14);
478 478
479 // Class for all int registers except RAX, RDX (and RSP) 479 // Class for all int registers except RAX, RDX (and RSP)
480 reg_class int_no_rax_rdx_reg(RBP, 480 reg_class int_no_rax_rdx_reg(RBP,
481 RDI 481 RDI,
482 RSI, 482 RSI,
483 RCX, 483 RCX,
484 RBX, 484 RBX,
485 R8, 485 R8,
486 R9, 486 R9,
550 550
551 //----------SOURCE BLOCK------------------------------------------------------- 551 //----------SOURCE BLOCK-------------------------------------------------------
552 // This is a block of C++ code which provides values, functions, and 552 // This is a block of C++ code which provides values, functions, and
553 // definitions necessary in the rest of the architecture description 553 // definitions necessary in the rest of the architecture description
554 source %{ 554 source %{
555 #define RELOC_IMM64 Assembler::imm64_operand 555 #define RELOC_IMM64 Assembler::imm_operand
556 #define RELOC_DISP32 Assembler::disp32_operand 556 #define RELOC_DISP32 Assembler::disp32_operand
557 557
558 #define __ _masm. 558 #define __ _masm.
559 559
560 // !!!!! Special hack to get all types of calls to specify the byte offset 560 // !!!!! Special hack to get all types of calls to specify the byte offset
960 960
961 #ifdef ASSERT 961 #ifdef ASSERT
962 if (VerifyStackAtCalls) { 962 if (VerifyStackAtCalls) {
963 Label L; 963 Label L;
964 MacroAssembler masm(&cbuf); 964 MacroAssembler masm(&cbuf);
965 masm.pushq(rax); 965 masm.push(rax);
966 masm.movq(rax, rsp); 966 masm.mov(rax, rsp);
967 masm.andq(rax, StackAlignmentInBytes-1); 967 masm.andptr(rax, StackAlignmentInBytes-1);
968 masm.cmpq(rax, StackAlignmentInBytes-wordSize); 968 masm.cmpptr(rax, StackAlignmentInBytes-wordSize);
969 masm.popq(rax); 969 masm.pop(rax);
970 masm.jcc(Assembler::equal, L); 970 masm.jcc(Assembler::equal, L);
971 masm.stop("Stack is not properly aligned!"); 971 masm.stop("Stack is not properly aligned!");
972 masm.bind(L); 972 masm.bind(L);
973 } 973 }
974 #endif 974 #endif
1815 if (base == NULL) return; // CodeBuffer::expand failed 1815 if (base == NULL) return; // CodeBuffer::expand failed
1816 // static stub relocation stores the instruction address of the call 1816 // static stub relocation stores the instruction address of the call
1817 __ relocate(static_stub_Relocation::spec(mark), RELOC_IMM64); 1817 __ relocate(static_stub_Relocation::spec(mark), RELOC_IMM64);
1818 // static stub relocation also tags the methodOop in the code-stream. 1818 // static stub relocation also tags the methodOop in the code-stream.
1819 __ movoop(rbx, (jobject) NULL); // method is zapped till fixup time 1819 __ movoop(rbx, (jobject) NULL); // method is zapped till fixup time
1820 // This is recognized as unresolved by relocs/nativeinst/ic code
1820 __ jump(RuntimeAddress(__ pc())); 1821 __ jump(RuntimeAddress(__ pc()));
1821 1822
1822 // Update current stubs pointer and restore code_end. 1823 // Update current stubs pointer and restore code_end.
1823 __ end_a_stub(); 1824 __ end_a_stub();
1824 } 1825 }
1861 #ifdef ASSERT 1862 #ifdef ASSERT
1862 uint code_size = cbuf.code_size(); 1863 uint code_size = cbuf.code_size();
1863 #endif 1864 #endif
1864 if (UseCompressedOops) { 1865 if (UseCompressedOops) {
1865 masm.load_klass(rscratch1, j_rarg0); 1866 masm.load_klass(rscratch1, j_rarg0);
1866 masm.cmpq(rax, rscratch1); 1867 masm.cmpptr(rax, rscratch1);
1867 } else { 1868 } else {
1868 masm.cmpq(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes())); 1869 masm.cmpptr(rax, Address(j_rarg0, oopDesc::klass_offset_in_bytes()));
1869 } 1870 }
1870 1871
1871 masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub())); 1872 masm.jump_cc(Assembler::notEqual, RuntimeAddress(SharedRuntime::get_ic_miss_stub()));
1872 1873
1873 /* WARNING these NOPs are critical so that verified entry point is properly 1874 /* WARNING these NOPs are critical so that verified entry point is properly
1947 1948
1948 // push address of "next" 1949 // push address of "next"
1949 __ call(next, relocInfo::none); // reloc none is fine since it is a disp32 1950 __ call(next, relocInfo::none); // reloc none is fine since it is a disp32
1950 __ bind(next); 1951 __ bind(next);
1951 // adjust it so it matches "the_pc" 1952 // adjust it so it matches "the_pc"
1952 __ subq(Address(rsp, 0), __ offset() - offset); 1953 __ subptr(Address(rsp, 0), __ offset() - offset);
1953 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack())); 1954 __ jump(RuntimeAddress(SharedRuntime::deopt_blob()->unpack()));
1954 assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow"); 1955 assert(__ offset() - offset <= (int) size_deopt_handler(), "overflow");
1955 __ end_a_stub(); 1956 __ end_a_stub();
1956 return offset; 1957 return offset;
1957 } 1958 }
2575 2576
2576 MacroAssembler _masm(&cbuf); 2577 MacroAssembler _masm(&cbuf);
2577 // Compare super with sub directly, since super is not in its own SSA. 2578 // Compare super with sub directly, since super is not in its own SSA.
2578 // The compiler used to emit this test, but we fold it in here, 2579 // The compiler used to emit this test, but we fold it in here,
2579 // to allow platform-specific tweaking on sparc. 2580 // to allow platform-specific tweaking on sparc.
2580 __ cmpq(Rrax, Rrsi); 2581 __ cmpptr(Rrax, Rrsi);
2581 __ jcc(Assembler::equal, hit); 2582 __ jcc(Assembler::equal, hit);
2582 #ifndef PRODUCT 2583 #ifndef PRODUCT
2583 __ lea(Rrcx, ExternalAddress((address)&SharedRuntime::_partial_subtype_ctr)); 2584 __ lea(Rrcx, ExternalAddress((address)&SharedRuntime::_partial_subtype_ctr));
2584 __ incrementl(Address(Rrcx, 0)); 2585 __ incrementl(Address(Rrcx, 0));
2585 #endif //PRODUCT 2586 #endif //PRODUCT
2586 __ movq(Rrdi, Address(Rrsi, 2587 __ movptr(Rrdi, Address(Rrsi,
2587 sizeof(oopDesc) + 2588 sizeof(oopDesc) +
2588 Klass::secondary_supers_offset_in_bytes())); 2589 Klass::secondary_supers_offset_in_bytes()));
2589 __ movl(Rrcx, Address(Rrdi, arrayOopDesc::length_offset_in_bytes())); 2590 __ movl(Rrcx, Address(Rrdi, arrayOopDesc::length_offset_in_bytes()));
2590 __ addq(Rrdi, arrayOopDesc::base_offset_in_bytes(T_OBJECT)); 2591 __ addptr(Rrdi, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
2591 if (UseCompressedOops) { 2592 if (UseCompressedOops) {
2592 __ encode_heap_oop(Rrax); 2593 __ encode_heap_oop(Rrax);
2593 __ repne_scanl(); 2594 __ repne_scanl();
2594 __ jcc(Assembler::notEqual, cmiss); 2595 __ jcc(Assembler::notEqual, cmiss);
2595 __ decode_heap_oop(Rrax); 2596 __ decode_heap_oop(Rrax);
2596 __ movq(Address(Rrsi, 2597 __ movptr(Address(Rrsi,
2597 sizeof(oopDesc) + 2598 sizeof(oopDesc) +
2598 Klass::secondary_super_cache_offset_in_bytes()), 2599 Klass::secondary_super_cache_offset_in_bytes()),
2599 Rrax); 2600 Rrax);
2600 __ jmp(hit); 2601 __ jmp(hit);
2601 __ bind(cmiss); 2602 __ bind(cmiss);
2602 __ decode_heap_oop(Rrax); 2603 __ decode_heap_oop(Rrax);
2603 __ jmp(miss); 2604 __ jmp(miss);
2604 } else { 2605 } else {
2605 __ repne_scanq(); 2606 __ repne_scan();
2606 __ jcc(Assembler::notEqual, miss); 2607 __ jcc(Assembler::notEqual, miss);
2607 __ movq(Address(Rrsi, 2608 __ movptr(Address(Rrsi,
2608 sizeof(oopDesc) + 2609 sizeof(oopDesc) +
2609 Klass::secondary_super_cache_offset_in_bytes()), 2610 Klass::secondary_super_cache_offset_in_bytes()),
2610 Rrax); 2611 Rrax);
2611 } 2612 }
2612 __ bind(hit); 2613 __ bind(hit);
2613 if ($primary) { 2614 if ($primary) {
2614 __ xorq(Rrdi, Rrdi); 2615 __ xorptr(Rrdi, Rrdi);
2615 } 2616 }
2616 __ bind(miss); 2617 __ bind(miss);
2617 %} 2618 %}
2618 2619
2619 enc_class Java_To_Interpreter(method meth) 2620 enc_class Java_To_Interpreter(method meth)
3525 3526
3526 if (_counters != NULL) { 3527 if (_counters != NULL) {
3527 masm.atomic_incl(ExternalAddress((address) _counters->total_entry_count_addr())); 3528 masm.atomic_incl(ExternalAddress((address) _counters->total_entry_count_addr()));
3528 } 3529 }
3529 if (EmitSync & 1) { 3530 if (EmitSync & 1) {
3530 masm.movptr (Address(boxReg, 0), intptr_t(markOopDesc::unused_mark())) ; 3531 // Without cast to int32_t a movptr will destroy r10 which is typically obj
3531 masm.cmpq (rsp, 0) ; 3532 masm.movptr (Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark())) ;
3533 masm.cmpptr(rsp, (int32_t)NULL_WORD) ;
3532 } else 3534 } else
3533 if (EmitSync & 2) { 3535 if (EmitSync & 2) {
3534 Label DONE_LABEL; 3536 Label DONE_LABEL;
3535 if (UseBiasedLocking) { 3537 if (UseBiasedLocking) {
3536 // Note: tmpReg maps to the swap_reg argument and scrReg to the tmp_reg argument. 3538 // Note: tmpReg maps to the swap_reg argument and scrReg to the tmp_reg argument.
3537 masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, _counters); 3539 masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, _counters);
3538 } 3540 }
3539 masm.movl(tmpReg, 0x1); 3541 // QQQ was movl...
3540 masm.orq(tmpReg, Address(objReg, 0)); 3542 masm.movptr(tmpReg, 0x1);
3541 masm.movq(Address(boxReg, 0), tmpReg); 3543 masm.orptr(tmpReg, Address(objReg, 0));
3544 masm.movptr(Address(boxReg, 0), tmpReg);
3542 if (os::is_MP()) { 3545 if (os::is_MP()) {
3543 masm.lock(); 3546 masm.lock();
3544 } 3547 }
3545 masm.cmpxchgq(boxReg, Address(objReg, 0)); // Updates tmpReg 3548 masm.cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg
3546 masm.jcc(Assembler::equal, DONE_LABEL); 3549 masm.jcc(Assembler::equal, DONE_LABEL);
3547 3550
3548 // Recursive locking 3551 // Recursive locking
3549 masm.subq(tmpReg, rsp); 3552 masm.subptr(tmpReg, rsp);
3550 masm.andq(tmpReg, 7 - os::vm_page_size()); 3553 masm.andptr(tmpReg, 7 - os::vm_page_size());
3551 masm.movq(Address(boxReg, 0), tmpReg); 3554 masm.movptr(Address(boxReg, 0), tmpReg);
3552 3555
3553 masm.bind(DONE_LABEL); 3556 masm.bind(DONE_LABEL);
3554 masm.nop(); // avoid branch to branch 3557 masm.nop(); // avoid branch to branch
3555 } else { 3558 } else {
3556 Label DONE_LABEL, IsInflated, Egress; 3559 Label DONE_LABEL, IsInflated, Egress;
3557 3560
3558 masm.movq (tmpReg, Address(objReg, 0)) ; 3561 masm.movptr(tmpReg, Address(objReg, 0)) ;
3559 masm.testq (tmpReg, 0x02) ; // inflated vs stack-locked|neutral|biased 3562 masm.testl (tmpReg, 0x02) ; // inflated vs stack-locked|neutral|biased
3560 masm.jcc (Assembler::notZero, IsInflated) ; 3563 masm.jcc (Assembler::notZero, IsInflated) ;
3561 3564
3562 // it's stack-locked, biased or neutral 3565 // it's stack-locked, biased or neutral
3563 // TODO: optimize markword triage order to reduce the number of 3566 // TODO: optimize markword triage order to reduce the number of
3564 // conditional branches in the most common cases. 3567 // conditional branches in the most common cases.
3565 // Beware -- there's a subtle invariant that fetch of the markword 3568 // Beware -- there's a subtle invariant that fetch of the markword
3566 // at [FETCH], below, will never observe a biased encoding (*101b). 3569 // at [FETCH], below, will never observe a biased encoding (*101b).
3567 // If this invariant is not held we'll suffer exclusion (safety) failure. 3570 // If this invariant is not held we'll suffer exclusion (safety) failure.
3568 3571
3569 if (UseBiasedLocking) { 3572 if (UseBiasedLocking) {
3570 masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, true, DONE_LABEL, NULL, _counters); 3573 masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, true, DONE_LABEL, NULL, _counters);
3571 masm.movq (tmpReg, Address(objReg, 0)) ; // [FETCH] 3574 masm.movptr(tmpReg, Address(objReg, 0)) ; // [FETCH]
3572 } 3575 }
3573 3576
3574 masm.orq (tmpReg, 1) ; 3577 // was q will it destroy high?
3575 masm.movq (Address(boxReg, 0), tmpReg) ; 3578 masm.orl (tmpReg, 1) ;
3576 if (os::is_MP()) { masm.lock(); } 3579 masm.movptr(Address(boxReg, 0), tmpReg) ;
3577 masm.cmpxchgq(boxReg, Address(objReg, 0)); // Updates tmpReg 3580 if (os::is_MP()) { masm.lock(); }
3581 masm.cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg
3578 if (_counters != NULL) { 3582 if (_counters != NULL) {
3579 masm.cond_inc32(Assembler::equal, 3583 masm.cond_inc32(Assembler::equal,
3580 ExternalAddress((address) _counters->fast_path_entry_count_addr())); 3584 ExternalAddress((address) _counters->fast_path_entry_count_addr()));
3581 } 3585 }
3582 masm.jcc (Assembler::equal, DONE_LABEL); 3586 masm.jcc (Assembler::equal, DONE_LABEL);
3583 3587
3584 // Recursive locking 3588 // Recursive locking
3585 masm.subq (tmpReg, rsp); 3589 masm.subptr(tmpReg, rsp);
3586 masm.andq (tmpReg, 7 - os::vm_page_size()); 3590 masm.andptr(tmpReg, 7 - os::vm_page_size());
3587 masm.movq (Address(boxReg, 0), tmpReg); 3591 masm.movptr(Address(boxReg, 0), tmpReg);
3588 if (_counters != NULL) { 3592 if (_counters != NULL) {
3589 masm.cond_inc32(Assembler::equal, 3593 masm.cond_inc32(Assembler::equal,
3590 ExternalAddress((address) _counters->fast_path_entry_count_addr())); 3594 ExternalAddress((address) _counters->fast_path_entry_count_addr()));
3591 } 3595 }
3592 masm.jmp (DONE_LABEL) ; 3596 masm.jmp (DONE_LABEL) ;
3597 // TODO: someday avoid the ST-before-CAS penalty by 3601 // TODO: someday avoid the ST-before-CAS penalty by
3598 // relocating (deferring) the following ST. 3602 // relocating (deferring) the following ST.
3599 // We should also think about trying a CAS without having 3603 // We should also think about trying a CAS without having
3600 // fetched _owner. If the CAS is successful we may 3604 // fetched _owner. If the CAS is successful we may
3601 // avoid an RTO->RTS upgrade on the $line. 3605 // avoid an RTO->RTS upgrade on the $line.
3602 masm.movptr(Address(boxReg, 0), intptr_t(markOopDesc::unused_mark())) ; 3606 // Without cast to int32_t a movptr will destroy r10 which is typically obj
3603 3607 masm.movptr(Address(boxReg, 0), (int32_t)intptr_t(markOopDesc::unused_mark())) ;
3604 masm.movq (boxReg, tmpReg) ; 3608
3605 masm.movq (tmpReg, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; 3609 masm.mov (boxReg, tmpReg) ;
3606 masm.testq (tmpReg, tmpReg) ; 3610 masm.movptr (tmpReg, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
3607 masm.jcc (Assembler::notZero, DONE_LABEL) ; 3611 masm.testptr(tmpReg, tmpReg) ;
3612 masm.jcc (Assembler::notZero, DONE_LABEL) ;
3608 3613
3609 // It's inflated and appears unlocked 3614 // It's inflated and appears unlocked
3610 if (os::is_MP()) { masm.lock(); } 3615 if (os::is_MP()) { masm.lock(); }
3611 masm.cmpxchgq(r15_thread, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; 3616 masm.cmpxchgptr(r15_thread, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
3612 // Intentional fall-through into DONE_LABEL ... 3617 // Intentional fall-through into DONE_LABEL ...
3613 3618
3614 masm.bind (DONE_LABEL) ; 3619 masm.bind (DONE_LABEL) ;
3615 masm.nop () ; // avoid jmp to jmp 3620 masm.nop () ; // avoid jmp to jmp
3616 } 3621 }
3625 Register objReg = as_Register($obj$$reg); 3630 Register objReg = as_Register($obj$$reg);
3626 Register boxReg = as_Register($box$$reg); 3631 Register boxReg = as_Register($box$$reg);
3627 Register tmpReg = as_Register($tmp$$reg); 3632 Register tmpReg = as_Register($tmp$$reg);
3628 MacroAssembler masm(&cbuf); 3633 MacroAssembler masm(&cbuf);
3629 3634
3630 if (EmitSync & 4) { 3635 if (EmitSync & 4) {
3631 masm.cmpq (rsp, 0) ; 3636 masm.cmpptr(rsp, 0) ;
3632 } else 3637 } else
3633 if (EmitSync & 8) { 3638 if (EmitSync & 8) {
3634 Label DONE_LABEL; 3639 Label DONE_LABEL;
3635 if (UseBiasedLocking) { 3640 if (UseBiasedLocking) {
3636 masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL); 3641 masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL);
3637 } 3642 }
3638 3643
3639 // Check whether the displaced header is 0 3644 // Check whether the displaced header is 0
3640 //(=> recursive unlock) 3645 //(=> recursive unlock)
3641 masm.movq(tmpReg, Address(boxReg, 0)); 3646 masm.movptr(tmpReg, Address(boxReg, 0));
3642 masm.testq(tmpReg, tmpReg); 3647 masm.testptr(tmpReg, tmpReg);
3643 masm.jcc(Assembler::zero, DONE_LABEL); 3648 masm.jcc(Assembler::zero, DONE_LABEL);
3644 3649
3645 // If not recursive lock, reset the header to displaced header 3650 // If not recursive lock, reset the header to displaced header
3646 if (os::is_MP()) { 3651 if (os::is_MP()) {
3647 masm.lock(); 3652 masm.lock();
3648 } 3653 }
3649 masm.cmpxchgq(tmpReg, Address(objReg, 0)); // Uses RAX which is box 3654 masm.cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box
3650 masm.bind(DONE_LABEL); 3655 masm.bind(DONE_LABEL);
3651 masm.nop(); // avoid branch to branch 3656 masm.nop(); // avoid branch to branch
3652 } else { 3657 } else {
3653 Label DONE_LABEL, Stacked, CheckSucc ; 3658 Label DONE_LABEL, Stacked, CheckSucc ;
3654 3659
3655 if (UseBiasedLocking) { 3660 if (UseBiasedLocking) {
3656 masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL); 3661 masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL);
3657 } 3662 }
3658 3663
3659 masm.movq (tmpReg, Address(objReg, 0)) ; 3664 masm.movptr(tmpReg, Address(objReg, 0)) ;
3660 masm.cmpq (Address(boxReg, 0), (int)NULL_WORD) ; 3665 masm.cmpptr(Address(boxReg, 0), (int32_t)NULL_WORD) ;
3661 masm.jcc (Assembler::zero, DONE_LABEL) ; 3666 masm.jcc (Assembler::zero, DONE_LABEL) ;
3662 masm.testq (tmpReg, 0x02) ; 3667 masm.testl (tmpReg, 0x02) ;
3663 masm.jcc (Assembler::zero, Stacked) ; 3668 masm.jcc (Assembler::zero, Stacked) ;
3664 3669
3665 // It's inflated 3670 // It's inflated
3666 masm.movq (boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; 3671 masm.movptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ;
3667 masm.xorq (boxReg, r15_thread) ; 3672 masm.xorptr(boxReg, r15_thread) ;
3668 masm.orq (boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2)) ; 3673 masm.orptr (boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2)) ;
3669 masm.jcc (Assembler::notZero, DONE_LABEL) ; 3674 masm.jcc (Assembler::notZero, DONE_LABEL) ;
3670 masm.movq (boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2)) ; 3675 masm.movptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2)) ;
3671 masm.orq (boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2)) ; 3676 masm.orptr (boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2)) ;
3672 masm.jcc (Assembler::notZero, CheckSucc) ; 3677 masm.jcc (Assembler::notZero, CheckSucc) ;
3673 masm.mov64 (Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int)NULL_WORD) ; 3678 masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
3674 masm.jmp (DONE_LABEL) ; 3679 masm.jmp (DONE_LABEL) ;
3675 3680
3676 if ((EmitSync & 65536) == 0) { 3681 if ((EmitSync & 65536) == 0) {
3677 Label LSuccess, LGoSlowPath ; 3682 Label LSuccess, LGoSlowPath ;
3678 masm.bind (CheckSucc) ; 3683 masm.bind (CheckSucc) ;
3679 masm.cmpq (Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int)NULL_WORD) ; 3684 masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
3680 masm.jcc (Assembler::zero, LGoSlowPath) ; 3685 masm.jcc (Assembler::zero, LGoSlowPath) ;
3681 3686
3682 // I'd much rather use lock:andl m->_owner, 0 as it's faster than the 3687 // I'd much rather use lock:andl m->_owner, 0 as it's faster than the
3683 // the explicit ST;MEMBAR combination, but masm doesn't currently support 3688 // the explicit ST;MEMBAR combination, but masm doesn't currently support
3684 // "ANDQ M,IMM". Don't use MFENCE here. lock:add to TOS, xchg, etc 3689 // "ANDQ M,IMM". Don't use MFENCE here. lock:add to TOS, xchg, etc
3685 // are all faster when the write buffer is populated. 3690 // are all faster when the write buffer is populated.
3686 masm.movptr (Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int)NULL_WORD) ; 3691 masm.movptr (Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
3687 if (os::is_MP()) { 3692 if (os::is_MP()) {
3688 masm.lock () ; masm.addq (Address(rsp, 0), 0) ; 3693 masm.lock () ; masm.addl (Address(rsp, 0), 0) ;
3689 } 3694 }
3690 masm.cmpq (Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int)NULL_WORD) ; 3695 masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), (int32_t)NULL_WORD) ;
3691 masm.jcc (Assembler::notZero, LSuccess) ; 3696 masm.jcc (Assembler::notZero, LSuccess) ;
3692 3697
3693 masm.movptr (boxReg, (int)NULL_WORD) ; // box is really EAX 3698 masm.movptr (boxReg, (int32_t)NULL_WORD) ; // box is really EAX
3694 if (os::is_MP()) { masm.lock(); } 3699 if (os::is_MP()) { masm.lock(); }
3695 masm.cmpxchgq (r15_thread, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)); 3700 masm.cmpxchgptr(r15_thread, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2));
3696 masm.jcc (Assembler::notEqual, LSuccess) ; 3701 masm.jcc (Assembler::notEqual, LSuccess) ;
3697 // Intentional fall-through into slow-path 3702 // Intentional fall-through into slow-path
3698 3703
3699 masm.bind (LGoSlowPath) ; 3704 masm.bind (LGoSlowPath) ;
3700 masm.orl (boxReg, 1) ; // set ICC.ZF=0 to indicate failure 3705 masm.orl (boxReg, 1) ; // set ICC.ZF=0 to indicate failure
3703 masm.bind (LSuccess) ; 3708 masm.bind (LSuccess) ;
3704 masm.testl (boxReg, 0) ; // set ICC.ZF=1 to indicate success 3709 masm.testl (boxReg, 0) ; // set ICC.ZF=1 to indicate success
3705 masm.jmp (DONE_LABEL) ; 3710 masm.jmp (DONE_LABEL) ;
3706 } 3711 }
3707 3712
3708 masm.bind (Stacked) ; 3713 masm.bind (Stacked) ;
3709 masm.movq (tmpReg, Address (boxReg, 0)) ; // re-fetch 3714 masm.movptr(tmpReg, Address (boxReg, 0)) ; // re-fetch
3710 if (os::is_MP()) { masm.lock(); } 3715 if (os::is_MP()) { masm.lock(); }
3711 masm.cmpxchgq(tmpReg, Address(objReg, 0)); // Uses RAX which is box 3716 masm.cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses RAX which is box
3712 3717
3713 if (EmitSync & 65536) { 3718 if (EmitSync & 65536) {
3714 masm.bind (CheckSucc) ; 3719 masm.bind (CheckSucc) ;
3715 } 3720 }
3716 masm.bind(DONE_LABEL); 3721 masm.bind(DONE_LABEL);
3734 int count_offset = java_lang_String::count_offset_in_bytes(); 3739 int count_offset = java_lang_String::count_offset_in_bytes();
3735 int base_offset = arrayOopDesc::base_offset_in_bytes(T_CHAR); 3740 int base_offset = arrayOopDesc::base_offset_in_bytes(T_CHAR);
3736 3741
3737 masm.load_heap_oop(rax, Address(rsi, value_offset)); 3742 masm.load_heap_oop(rax, Address(rsi, value_offset));
3738 masm.movl(rcx, Address(rsi, offset_offset)); 3743 masm.movl(rcx, Address(rsi, offset_offset));
3739 masm.leaq(rax, Address(rax, rcx, Address::times_2, base_offset)); 3744 masm.lea(rax, Address(rax, rcx, Address::times_2, base_offset));
3740 masm.load_heap_oop(rbx, Address(rdi, value_offset)); 3745 masm.load_heap_oop(rbx, Address(rdi, value_offset));
3741 masm.movl(rcx, Address(rdi, offset_offset)); 3746 masm.movl(rcx, Address(rdi, offset_offset));
3742 masm.leaq(rbx, Address(rbx, rcx, Address::times_2, base_offset)); 3747 masm.lea(rbx, Address(rbx, rcx, Address::times_2, base_offset));
3743 3748
3744 // Compute the minimum of the string lengths(rsi) and the 3749 // Compute the minimum of the string lengths(rsi) and the
3745 // difference of the string lengths (stack) 3750 // difference of the string lengths (stack)
3746 3751
3747 masm.movl(rdi, Address(rdi, count_offset)); 3752 masm.movl(rdi, Address(rdi, count_offset));
3748 masm.movl(rsi, Address(rsi, count_offset)); 3753 masm.movl(rsi, Address(rsi, count_offset));
3749 masm.movl(rcx, rdi); 3754 masm.movl(rcx, rdi);
3750 masm.subl(rdi, rsi); 3755 masm.subl(rdi, rsi);
3751 masm.pushq(rdi); 3756 masm.push(rdi);
3752 masm.cmovl(Assembler::lessEqual, rsi, rcx); 3757 masm.cmov(Assembler::lessEqual, rsi, rcx);
3753 3758
3754 // Is the minimum length zero? 3759 // Is the minimum length zero?
3755 masm.bind(RCX_GOOD_LABEL); 3760 masm.bind(RCX_GOOD_LABEL);
3756 masm.testl(rsi, rsi); 3761 masm.testl(rsi, rsi);
3757 masm.jcc(Assembler::zero, LENGTH_DIFF_LABEL); 3762 masm.jcc(Assembler::zero, LENGTH_DIFF_LABEL);
3768 3773
3769 { 3774 {
3770 // Check after comparing first character to see if strings are equivalent 3775 // Check after comparing first character to see if strings are equivalent
3771 Label LSkip2; 3776 Label LSkip2;
3772 // Check if the strings start at same location 3777 // Check if the strings start at same location
3773 masm.cmpq(rbx, rax); 3778 masm.cmpptr(rbx, rax);
3774 masm.jcc(Assembler::notEqual, LSkip2); 3779 masm.jcc(Assembler::notEqual, LSkip2);
3775 3780
3776 // Check if the length difference is zero (from stack) 3781 // Check if the length difference is zero (from stack)
3777 masm.cmpl(Address(rsp, 0), 0x0); 3782 masm.cmpl(Address(rsp, 0), 0x0);
3778 masm.jcc(Assembler::equal, LENGTH_DIFF_LABEL); 3783 masm.jcc(Assembler::equal, LENGTH_DIFF_LABEL);
3780 // Strings might not be equivalent 3785 // Strings might not be equivalent
3781 masm.bind(LSkip2); 3786 masm.bind(LSkip2);
3782 } 3787 }
3783 3788
3784 // Shift RAX and RBX to the end of the arrays, negate min 3789 // Shift RAX and RBX to the end of the arrays, negate min
3785 masm.leaq(rax, Address(rax, rsi, Address::times_2, 2)); 3790 masm.lea(rax, Address(rax, rsi, Address::times_2, 2));
3786 masm.leaq(rbx, Address(rbx, rsi, Address::times_2, 2)); 3791 masm.lea(rbx, Address(rbx, rsi, Address::times_2, 2));
3787 masm.negq(rsi); 3792 masm.negptr(rsi);
3788 3793
3789 // Compare the rest of the characters 3794 // Compare the rest of the characters
3790 masm.bind(WHILE_HEAD_LABEL); 3795 masm.bind(WHILE_HEAD_LABEL);
3791 masm.load_unsigned_word(rcx, Address(rbx, rsi, Address::times_2, 0)); 3796 masm.load_unsigned_word(rcx, Address(rbx, rsi, Address::times_2, 0));
3792 masm.load_unsigned_word(rdi, Address(rax, rsi, Address::times_2, 0)); 3797 masm.load_unsigned_word(rdi, Address(rax, rsi, Address::times_2, 0));
3793 masm.subl(rcx, rdi); 3798 masm.subl(rcx, rdi);
3794 masm.jcc(Assembler::notZero, POP_LABEL); 3799 masm.jcc(Assembler::notZero, POP_LABEL);
3795 masm.incrementq(rsi); 3800 masm.increment(rsi);
3796 masm.jcc(Assembler::notZero, WHILE_HEAD_LABEL); 3801 masm.jcc(Assembler::notZero, WHILE_HEAD_LABEL);
3797 3802
3798 // Strings are equal up to min length. Return the length difference. 3803 // Strings are equal up to min length. Return the length difference.
3799 masm.bind(LENGTH_DIFF_LABEL); 3804 masm.bind(LENGTH_DIFF_LABEL);
3800 masm.popq(rcx); 3805 masm.pop(rcx);
3801 masm.jmp(DONE_LABEL); 3806 masm.jmp(DONE_LABEL);
3802 3807
3803 // Discard the stored length difference 3808 // Discard the stored length difference
3804 masm.bind(POP_LABEL); 3809 masm.bind(POP_LABEL);
3805 masm.addq(rsp, 8); 3810 masm.addptr(rsp, 8);
3806 3811
3807 // That's it 3812 // That's it
3808 masm.bind(DONE_LABEL); 3813 masm.bind(DONE_LABEL);
3809 %} 3814 %}
3810 3815
3811 enc_class enc_Array_Equals(rdi_RegP ary1, rsi_RegP ary2, rax_RegI tmp1, rbx_RegI tmp2, rcx_RegI result) %{ 3816 enc_class enc_Array_Equals(rdi_RegP ary1, rsi_RegP ary2, rax_RegI tmp1, rbx_RegI tmp2, rcx_RegI result) %{
3891 %} 3896 %}
3892 3897
3893 enc_class absF_encoding(regF dst) 3898 enc_class absF_encoding(regF dst)
3894 %{ 3899 %{
3895 int dstenc = $dst$$reg; 3900 int dstenc = $dst$$reg;
3896 address signmask_address = (address) StubRoutines::amd64::float_sign_mask(); 3901 address signmask_address = (address) StubRoutines::x86::float_sign_mask();
3897 3902
3898 cbuf.set_inst_mark(); 3903 cbuf.set_inst_mark();
3899 if (dstenc >= 8) { 3904 if (dstenc >= 8) {
3900 emit_opcode(cbuf, Assembler::REX_R); 3905 emit_opcode(cbuf, Assembler::REX_R);
3901 dstenc -= 8; 3906 dstenc -= 8;
3908 %} 3913 %}
3909 3914
3910 enc_class absD_encoding(regD dst) 3915 enc_class absD_encoding(regD dst)
3911 %{ 3916 %{
3912 int dstenc = $dst$$reg; 3917 int dstenc = $dst$$reg;
3913 address signmask_address = (address) StubRoutines::amd64::double_sign_mask(); 3918 address signmask_address = (address) StubRoutines::x86::double_sign_mask();
3914 3919
3915 cbuf.set_inst_mark(); 3920 cbuf.set_inst_mark();
3916 emit_opcode(cbuf, 0x66); 3921 emit_opcode(cbuf, 0x66);
3917 if (dstenc >= 8) { 3922 if (dstenc >= 8) {
3918 emit_opcode(cbuf, Assembler::REX_R); 3923 emit_opcode(cbuf, Assembler::REX_R);
3926 %} 3931 %}
3927 3932
3928 enc_class negF_encoding(regF dst) 3933 enc_class negF_encoding(regF dst)
3929 %{ 3934 %{
3930 int dstenc = $dst$$reg; 3935 int dstenc = $dst$$reg;
3931 address signflip_address = (address) StubRoutines::amd64::float_sign_flip(); 3936 address signflip_address = (address) StubRoutines::x86::float_sign_flip();
3932 3937
3933 cbuf.set_inst_mark(); 3938 cbuf.set_inst_mark();
3934 if (dstenc >= 8) { 3939 if (dstenc >= 8) {
3935 emit_opcode(cbuf, Assembler::REX_R); 3940 emit_opcode(cbuf, Assembler::REX_R);
3936 dstenc -= 8; 3941 dstenc -= 8;
3943 %} 3948 %}
3944 3949
3945 enc_class negD_encoding(regD dst) 3950 enc_class negD_encoding(regD dst)
3946 %{ 3951 %{
3947 int dstenc = $dst$$reg; 3952 int dstenc = $dst$$reg;
3948 address signflip_address = (address) StubRoutines::amd64::double_sign_flip(); 3953 address signflip_address = (address) StubRoutines::x86::double_sign_flip();
3949 3954
3950 cbuf.set_inst_mark(); 3955 cbuf.set_inst_mark();
3951 emit_opcode(cbuf, 0x66); 3956 emit_opcode(cbuf, 0x66);
3952 if (dstenc >= 8) { 3957 if (dstenc >= 8) {
3953 emit_opcode(cbuf, Assembler::REX_R); 3958 emit_opcode(cbuf, Assembler::REX_R);
4001 // call f2i_fixup 4006 // call f2i_fixup
4002 cbuf.set_inst_mark(); 4007 cbuf.set_inst_mark();
4003 emit_opcode(cbuf, 0xE8); 4008 emit_opcode(cbuf, 0xE8);
4004 emit_d32_reloc(cbuf, 4009 emit_d32_reloc(cbuf,
4005 (int) 4010 (int)
4006 (StubRoutines::amd64::f2i_fixup() - cbuf.code_end() - 4), 4011 (StubRoutines::x86::f2i_fixup() - cbuf.code_end() - 4),
4007 runtime_call_Relocation::spec(), 4012 runtime_call_Relocation::spec(),
4008 RELOC_DISP32); 4013 RELOC_DISP32);
4009 4014
4010 // popq $dst 4015 // popq $dst
4011 if (dstenc >= 8) { 4016 if (dstenc >= 8) {
4018 4023
4019 enc_class f2l_fixup(rRegL dst, regF src) 4024 enc_class f2l_fixup(rRegL dst, regF src)
4020 %{ 4025 %{
4021 int dstenc = $dst$$reg; 4026 int dstenc = $dst$$reg;
4022 int srcenc = $src$$reg; 4027 int srcenc = $src$$reg;
4023 address const_address = (address) StubRoutines::amd64::double_sign_flip(); 4028 address const_address = (address) StubRoutines::x86::double_sign_flip();
4024 4029
4025 // cmpq $dst, [0x8000000000000000] 4030 // cmpq $dst, [0x8000000000000000]
4026 cbuf.set_inst_mark(); 4031 cbuf.set_inst_mark();
4027 emit_opcode(cbuf, dstenc < 8 ? Assembler::REX_W : Assembler::REX_WR); 4032 emit_opcode(cbuf, dstenc < 8 ? Assembler::REX_W : Assembler::REX_WR);
4028 emit_opcode(cbuf, 0x39); 4033 emit_opcode(cbuf, 0x39);
4059 // call f2l_fixup 4064 // call f2l_fixup
4060 cbuf.set_inst_mark(); 4065 cbuf.set_inst_mark();
4061 emit_opcode(cbuf, 0xE8); 4066 emit_opcode(cbuf, 0xE8);
4062 emit_d32_reloc(cbuf, 4067 emit_d32_reloc(cbuf,
4063 (int) 4068 (int)
4064 (StubRoutines::amd64::f2l_fixup() - cbuf.code_end() - 4), 4069 (StubRoutines::x86::f2l_fixup() - cbuf.code_end() - 4),
4065 runtime_call_Relocation::spec(), 4070 runtime_call_Relocation::spec(),
4066 RELOC_DISP32); 4071 RELOC_DISP32);
4067 4072
4068 // popq $dst 4073 // popq $dst
4069 if (dstenc >= 8) { 4074 if (dstenc >= 8) {
4115 // call d2i_fixup 4120 // call d2i_fixup
4116 cbuf.set_inst_mark(); 4121 cbuf.set_inst_mark();
4117 emit_opcode(cbuf, 0xE8); 4122 emit_opcode(cbuf, 0xE8);
4118 emit_d32_reloc(cbuf, 4123 emit_d32_reloc(cbuf,
4119 (int) 4124 (int)
4120 (StubRoutines::amd64::d2i_fixup() - cbuf.code_end() - 4), 4125 (StubRoutines::x86::d2i_fixup() - cbuf.code_end() - 4),
4121 runtime_call_Relocation::spec(), 4126 runtime_call_Relocation::spec(),
4122 RELOC_DISP32); 4127 RELOC_DISP32);
4123 4128
4124 // popq $dst 4129 // popq $dst
4125 if (dstenc >= 8) { 4130 if (dstenc >= 8) {
4132 4137
4133 enc_class d2l_fixup(rRegL dst, regD src) 4138 enc_class d2l_fixup(rRegL dst, regD src)
4134 %{ 4139 %{
4135 int dstenc = $dst$$reg; 4140 int dstenc = $dst$$reg;
4136 int srcenc = $src$$reg; 4141 int srcenc = $src$$reg;
4137 address const_address = (address) StubRoutines::amd64::double_sign_flip(); 4142 address const_address = (address) StubRoutines::x86::double_sign_flip();
4138 4143
4139 // cmpq $dst, [0x8000000000000000] 4144 // cmpq $dst, [0x8000000000000000]
4140 cbuf.set_inst_mark(); 4145 cbuf.set_inst_mark();
4141 emit_opcode(cbuf, dstenc < 8 ? Assembler::REX_W : Assembler::REX_WR); 4146 emit_opcode(cbuf, dstenc < 8 ? Assembler::REX_W : Assembler::REX_WR);
4142 emit_opcode(cbuf, 0x39); 4147 emit_opcode(cbuf, 0x39);
4173 // call d2l_fixup 4178 // call d2l_fixup
4174 cbuf.set_inst_mark(); 4179 cbuf.set_inst_mark();
4175 emit_opcode(cbuf, 0xE8); 4180 emit_opcode(cbuf, 0xE8);
4176 emit_d32_reloc(cbuf, 4181 emit_d32_reloc(cbuf,
4177 (int) 4182 (int)
4178 (StubRoutines::amd64::d2l_fixup() - cbuf.code_end() - 4), 4183 (StubRoutines::x86::d2l_fixup() - cbuf.code_end() - 4),
4179 runtime_call_Relocation::spec(), 4184 runtime_call_Relocation::spec(),
4180 RELOC_DISP32); 4185 RELOC_DISP32);
4181 4186
4182 // popq $dst 4187 // popq $dst
4183 if (dstenc >= 8) { 4188 if (dstenc >= 8) {