Mercurial > hg > truffle
comparison src/cpu/x86/vm/x86_32.ad @ 304:dc7f315e41f7
5108146: Merge i486 and amd64 cpu directories
6459804: Want client (c1) compiler for x86_64 (amd64) for faster start-up
Reviewed-by: kvn
author | never |
---|---|
date | Wed, 27 Aug 2008 00:21:55 -0700 |
parents | 9c2ecc2ffb12 |
children | b744678d4d71 |
comparison
equal
deleted
inserted
replaced
303:fa4d1d240383 | 304:dc7f315e41f7 |
---|---|
234 | 234 |
235 //----------SOURCE BLOCK------------------------------------------------------- | 235 //----------SOURCE BLOCK------------------------------------------------------- |
236 // This is a block of C++ code which provides values, functions, and | 236 // This is a block of C++ code which provides values, functions, and |
237 // definitions necessary in the rest of the architecture description | 237 // definitions necessary in the rest of the architecture description |
238 source %{ | 238 source %{ |
239 #define RELOC_IMM32 Assembler::imm32_operand | 239 #define RELOC_IMM32 Assembler::imm_operand |
240 #define RELOC_DISP32 Assembler::disp32_operand | 240 #define RELOC_DISP32 Assembler::disp32_operand |
241 | 241 |
242 #define __ _masm. | 242 #define __ _masm. |
243 | 243 |
244 // How to find the high register of a Long pair, given the low register | 244 // How to find the high register of a Long pair, given the low register |
591 | 591 |
592 #ifdef ASSERT | 592 #ifdef ASSERT |
593 if (VerifyStackAtCalls) { | 593 if (VerifyStackAtCalls) { |
594 Label L; | 594 Label L; |
595 MacroAssembler masm(&cbuf); | 595 MacroAssembler masm(&cbuf); |
596 masm.pushl(rax); | 596 masm.push(rax); |
597 masm.movl(rax, rsp); | 597 masm.mov(rax, rsp); |
598 masm.andl(rax, StackAlignmentInBytes-1); | 598 masm.andptr(rax, StackAlignmentInBytes-1); |
599 masm.cmpl(rax, StackAlignmentInBytes-wordSize); | 599 masm.cmpptr(rax, StackAlignmentInBytes-wordSize); |
600 masm.popl(rax); | 600 masm.pop(rax); |
601 masm.jcc(Assembler::equal, L); | 601 masm.jcc(Assembler::equal, L); |
602 masm.stop("Stack is not properly aligned!"); | 602 masm.stop("Stack is not properly aligned!"); |
603 masm.bind(L); | 603 masm.bind(L); |
604 } | 604 } |
605 #endif | 605 #endif |
1148 if (base == NULL) return; // CodeBuffer::expand failed | 1148 if (base == NULL) return; // CodeBuffer::expand failed |
1149 // static stub relocation stores the instruction address of the call | 1149 // static stub relocation stores the instruction address of the call |
1150 __ relocate(static_stub_Relocation::spec(mark), RELOC_IMM32); | 1150 __ relocate(static_stub_Relocation::spec(mark), RELOC_IMM32); |
1151 // static stub relocation also tags the methodOop in the code-stream. | 1151 // static stub relocation also tags the methodOop in the code-stream. |
1152 __ movoop(rbx, (jobject)NULL); // method is zapped till fixup time | 1152 __ movoop(rbx, (jobject)NULL); // method is zapped till fixup time |
1153 __ jump(RuntimeAddress((address)-1)); | 1153 // This is recognized as unresolved by relocs/nativeInst/ic code |
1154 __ jump(RuntimeAddress(__ pc())); | |
1154 | 1155 |
1155 __ end_a_stub(); | 1156 __ end_a_stub(); |
1156 // Update current stubs pointer and restore code_end. | 1157 // Update current stubs pointer and restore code_end. |
1157 } | 1158 } |
1158 // size of call stub, compiled java to interpretor | 1159 // size of call stub, compiled java to interpretor |
1179 void MachUEPNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const { | 1180 void MachUEPNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const { |
1180 MacroAssembler masm(&cbuf); | 1181 MacroAssembler masm(&cbuf); |
1181 #ifdef ASSERT | 1182 #ifdef ASSERT |
1182 uint code_size = cbuf.code_size(); | 1183 uint code_size = cbuf.code_size(); |
1183 #endif | 1184 #endif |
1184 masm.cmpl(rax, Address(rcx, oopDesc::klass_offset_in_bytes())); | 1185 masm.cmpptr(rax, Address(rcx, oopDesc::klass_offset_in_bytes())); |
1185 masm.jump_cc(Assembler::notEqual, | 1186 masm.jump_cc(Assembler::notEqual, |
1186 RuntimeAddress(SharedRuntime::get_ic_miss_stub())); | 1187 RuntimeAddress(SharedRuntime::get_ic_miss_stub())); |
1187 /* WARNING these NOPs are critical so that verified entry point is properly | 1188 /* WARNING these NOPs are critical so that verified entry point is properly |
1188 aligned for patching by NativeJump::patch_verified_entry() */ | 1189 aligned for patching by NativeJump::patch_verified_entry() */ |
1189 int nops_cnt = 2; | 1190 int nops_cnt = 2; |
1685 | 1686 |
1686 MacroAssembler _masm(&cbuf); | 1687 MacroAssembler _masm(&cbuf); |
1687 // Compare super with sub directly, since super is not in its own SSA. | 1688 // Compare super with sub directly, since super is not in its own SSA. |
1688 // The compiler used to emit this test, but we fold it in here, | 1689 // The compiler used to emit this test, but we fold it in here, |
1689 // to allow platform-specific tweaking on sparc. | 1690 // to allow platform-specific tweaking on sparc. |
1690 __ cmpl(Reax, Resi); | 1691 __ cmpptr(Reax, Resi); |
1691 __ jcc(Assembler::equal, hit); | 1692 __ jcc(Assembler::equal, hit); |
1692 #ifndef PRODUCT | 1693 #ifndef PRODUCT |
1693 __ increment(ExternalAddress((address)&SharedRuntime::_partial_subtype_ctr)); | 1694 __ incrementl(ExternalAddress((address)&SharedRuntime::_partial_subtype_ctr)); |
1694 #endif //PRODUCT | 1695 #endif //PRODUCT |
1695 __ movl(Redi,Address(Resi,sizeof(oopDesc) + Klass::secondary_supers_offset_in_bytes())); | 1696 __ movptr(Redi,Address(Resi,sizeof(oopDesc) + Klass::secondary_supers_offset_in_bytes())); |
1696 __ movl(Recx,Address(Redi,arrayOopDesc::length_offset_in_bytes())); | 1697 __ movl(Recx,Address(Redi,arrayOopDesc::length_offset_in_bytes())); |
1697 __ addl(Redi,arrayOopDesc::base_offset_in_bytes(T_OBJECT)); | 1698 __ addptr(Redi,arrayOopDesc::base_offset_in_bytes(T_OBJECT)); |
1698 __ repne_scan(); | 1699 __ repne_scan(); |
1699 __ jcc(Assembler::notEqual, miss); | 1700 __ jcc(Assembler::notEqual, miss); |
1700 __ movl(Address(Resi,sizeof(oopDesc) + Klass::secondary_super_cache_offset_in_bytes()),Reax); | 1701 __ movptr(Address(Resi,sizeof(oopDesc) + Klass::secondary_super_cache_offset_in_bytes()),Reax); |
1701 __ bind(hit); | 1702 __ bind(hit); |
1702 if( $primary ) | 1703 if( $primary ) |
1703 __ xorl(Redi,Redi); | 1704 __ xorptr(Redi,Redi); |
1704 __ bind(miss); | 1705 __ bind(miss); |
1705 %} | 1706 %} |
1706 | 1707 |
1707 enc_class FFree_Float_Stack_All %{ // Free_Float_Stack_All | 1708 enc_class FFree_Float_Stack_All %{ // Free_Float_Stack_All |
1708 MacroAssembler masm(&cbuf); | 1709 MacroAssembler masm(&cbuf); |
1747 // mode the result needs to be removed from the FPU stack. It's | 1748 // mode the result needs to be removed from the FPU stack. It's |
1748 // likely that this function call could be removed by the | 1749 // likely that this function call could be removed by the |
1749 // optimizer if the C function is a pure function. | 1750 // optimizer if the C function is a pure function. |
1750 __ ffree(0); | 1751 __ ffree(0); |
1751 } else if (rt == T_FLOAT) { | 1752 } else if (rt == T_FLOAT) { |
1752 __ leal(rsp, Address(rsp, -4)); | 1753 __ lea(rsp, Address(rsp, -4)); |
1753 __ fstp_s(Address(rsp, 0)); | 1754 __ fstp_s(Address(rsp, 0)); |
1754 __ movflt(xmm0, Address(rsp, 0)); | 1755 __ movflt(xmm0, Address(rsp, 0)); |
1755 __ leal(rsp, Address(rsp, 4)); | 1756 __ lea(rsp, Address(rsp, 4)); |
1756 } else if (rt == T_DOUBLE) { | 1757 } else if (rt == T_DOUBLE) { |
1757 __ leal(rsp, Address(rsp, -8)); | 1758 __ lea(rsp, Address(rsp, -8)); |
1758 __ fstp_d(Address(rsp, 0)); | 1759 __ fstp_d(Address(rsp, 0)); |
1759 __ movdbl(xmm0, Address(rsp, 0)); | 1760 __ movdbl(xmm0, Address(rsp, 0)); |
1760 __ leal(rsp, Address(rsp, 8)); | 1761 __ lea(rsp, Address(rsp, 8)); |
1761 } | 1762 } |
1762 } | 1763 } |
1763 %} | 1764 %} |
1764 | 1765 |
1765 | 1766 |
2886 | 2887 |
2887 __ jccb(Assembler::parity, nan); | 2888 __ jccb(Assembler::parity, nan); |
2888 __ jccb(Assembler::equal, done); | 2889 __ jccb(Assembler::equal, done); |
2889 __ jccb(Assembler::above, inc); | 2890 __ jccb(Assembler::above, inc); |
2890 __ bind(nan); | 2891 __ bind(nan); |
2891 __ decrement(as_Register($dst$$reg)); | 2892 __ decrement(as_Register($dst$$reg)); // NO L qqq |
2892 __ jmpb(done); | 2893 __ jmpb(done); |
2893 __ bind(inc); | 2894 __ bind(inc); |
2894 __ increment(as_Register($dst$$reg)); | 2895 __ increment(as_Register($dst$$reg)); // NO L qqq |
2895 __ bind(done); | 2896 __ bind(done); |
2896 %} | 2897 %} |
2897 | 2898 |
2898 // Compare the longs and set flags | 2899 // Compare the longs and set flags |
2899 // BROKEN! Do Not use as-is | 2900 // BROKEN! Do Not use as-is |
3156 %} | 3157 %} |
3157 | 3158 |
3158 enc_class mov_i2x(regXD dst, eRegI src) %{ | 3159 enc_class mov_i2x(regXD dst, eRegI src) %{ |
3159 MacroAssembler _masm(&cbuf); | 3160 MacroAssembler _masm(&cbuf); |
3160 | 3161 |
3161 __ movd(as_XMMRegister($dst$$reg), as_Register($src$$reg)); | 3162 __ movdl(as_XMMRegister($dst$$reg), as_Register($src$$reg)); |
3162 %} | 3163 %} |
3163 | 3164 |
3164 | 3165 |
3165 // Because the transitions from emitted code to the runtime | 3166 // Because the transitions from emitted code to the runtime |
3166 // monitorenter/exit helper stubs are so slow it's critical that | 3167 // monitorenter/exit helper stubs are so slow it's critical that |
3256 if (_counters != NULL) { | 3257 if (_counters != NULL) { |
3257 masm.atomic_incl(ExternalAddress((address) _counters->total_entry_count_addr())); | 3258 masm.atomic_incl(ExternalAddress((address) _counters->total_entry_count_addr())); |
3258 } | 3259 } |
3259 if (EmitSync & 1) { | 3260 if (EmitSync & 1) { |
3260 // set box->dhw = unused_mark (3) | 3261 // set box->dhw = unused_mark (3) |
3261 // Force all sync thru slow-path: slow_enter() and slow_exit() | 3262 // Force all sync thru slow-path: slow_enter() and slow_exit() |
3262 masm.movl (Address(boxReg, 0), intptr_t(markOopDesc::unused_mark())) ; | 3263 masm.movptr (Address(boxReg, 0), int32_t(markOopDesc::unused_mark())) ; |
3263 masm.cmpl (rsp, 0) ; | 3264 masm.cmpptr (rsp, (int32_t)0) ; |
3264 } else | 3265 } else |
3265 if (EmitSync & 2) { | 3266 if (EmitSync & 2) { |
3266 Label DONE_LABEL ; | 3267 Label DONE_LABEL ; |
3267 if (UseBiasedLocking) { | 3268 if (UseBiasedLocking) { |
3268 // Note: tmpReg maps to the swap_reg argument and scrReg to the tmp_reg argument. | 3269 // Note: tmpReg maps to the swap_reg argument and scrReg to the tmp_reg argument. |
3269 masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, _counters); | 3270 masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, _counters); |
3270 } | 3271 } |
3271 | 3272 |
3272 masm.movl (tmpReg, Address(objReg, 0)) ; // fetch markword | 3273 masm.movptr(tmpReg, Address(objReg, 0)) ; // fetch markword |
3273 masm.orl (tmpReg, 0x1); | 3274 masm.orptr (tmpReg, 0x1); |
3274 masm.movl (Address(boxReg, 0), tmpReg); // Anticipate successful CAS | 3275 masm.movptr(Address(boxReg, 0), tmpReg); // Anticipate successful CAS |
3275 if (os::is_MP()) { masm.lock(); } | 3276 if (os::is_MP()) { masm.lock(); } |
3276 masm.cmpxchg(boxReg, Address(objReg, 0)); // Updates tmpReg | 3277 masm.cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg |
3277 masm.jcc(Assembler::equal, DONE_LABEL); | 3278 masm.jcc(Assembler::equal, DONE_LABEL); |
3278 // Recursive locking | 3279 // Recursive locking |
3279 masm.subl(tmpReg, rsp); | 3280 masm.subptr(tmpReg, rsp); |
3280 masm.andl(tmpReg, 0xFFFFF003 ); | 3281 masm.andptr(tmpReg, (int32_t) 0xFFFFF003 ); |
3281 masm.movl(Address(boxReg, 0), tmpReg); | 3282 masm.movptr(Address(boxReg, 0), tmpReg); |
3282 masm.bind(DONE_LABEL) ; | 3283 masm.bind(DONE_LABEL) ; |
3283 } else { | 3284 } else { |
3284 // Possible cases that we'll encounter in fast_lock | 3285 // Possible cases that we'll encounter in fast_lock |
3285 // ------------------------------------------------ | 3286 // ------------------------------------------------ |
3286 // * Inflated | 3287 // * Inflated |
3287 // -- unlocked | 3288 // -- unlocked |
3288 // -- Locked | 3289 // -- Locked |
3289 // = by self | 3290 // = by self |
3308 // If this invariant is not held we risk exclusion (safety) failure. | 3309 // If this invariant is not held we risk exclusion (safety) failure. |
3309 if (UseBiasedLocking) { | 3310 if (UseBiasedLocking) { |
3310 masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, _counters); | 3311 masm.biased_locking_enter(boxReg, objReg, tmpReg, scrReg, false, DONE_LABEL, NULL, _counters); |
3311 } | 3312 } |
3312 | 3313 |
3313 masm.movl (tmpReg, Address(objReg, 0)) ; // [FETCH] | 3314 masm.movptr(tmpReg, Address(objReg, 0)) ; // [FETCH] |
3314 masm.testl (tmpReg, 0x02) ; // Inflated v (Stack-locked or neutral) | 3315 masm.testptr(tmpReg, 0x02) ; // Inflated v (Stack-locked or neutral) |
3315 masm.jccb (Assembler::notZero, IsInflated) ; | 3316 masm.jccb (Assembler::notZero, IsInflated) ; |
3316 | 3317 |
3317 // Attempt stack-locking ... | 3318 // Attempt stack-locking ... |
3318 masm.orl (tmpReg, 0x1); | 3319 masm.orptr (tmpReg, 0x1); |
3319 masm.movl (Address(boxReg, 0), tmpReg); // Anticipate successful CAS | 3320 masm.movptr(Address(boxReg, 0), tmpReg); // Anticipate successful CAS |
3320 if (os::is_MP()) { masm.lock(); } | 3321 if (os::is_MP()) { masm.lock(); } |
3321 masm.cmpxchg(boxReg, Address(objReg, 0)); // Updates tmpReg | 3322 masm.cmpxchgptr(boxReg, Address(objReg, 0)); // Updates tmpReg |
3322 if (_counters != NULL) { | 3323 if (_counters != NULL) { |
3323 masm.cond_inc32(Assembler::equal, | 3324 masm.cond_inc32(Assembler::equal, |
3324 ExternalAddress((address)_counters->fast_path_entry_count_addr())); | 3325 ExternalAddress((address)_counters->fast_path_entry_count_addr())); |
3325 } | 3326 } |
3326 masm.jccb (Assembler::equal, DONE_LABEL); | 3327 masm.jccb (Assembler::equal, DONE_LABEL); |
3327 | 3328 |
3328 // Recursive locking | 3329 // Recursive locking |
3329 masm.subl(tmpReg, rsp); | 3330 masm.subptr(tmpReg, rsp); |
3330 masm.andl(tmpReg, 0xFFFFF003 ); | 3331 masm.andptr(tmpReg, 0xFFFFF003 ); |
3331 masm.movl(Address(boxReg, 0), tmpReg); | 3332 masm.movptr(Address(boxReg, 0), tmpReg); |
3332 if (_counters != NULL) { | 3333 if (_counters != NULL) { |
3333 masm.cond_inc32(Assembler::equal, | 3334 masm.cond_inc32(Assembler::equal, |
3334 ExternalAddress((address)_counters->fast_path_entry_count_addr())); | 3335 ExternalAddress((address)_counters->fast_path_entry_count_addr())); |
3335 } | 3336 } |
3336 masm.jmp (DONE_LABEL) ; | 3337 masm.jmp (DONE_LABEL) ; |
3358 // We'd like to write: | 3359 // We'd like to write: |
3359 // set box->_displaced_header = markOop::unused_mark(). Any non-0 value suffices. | 3360 // set box->_displaced_header = markOop::unused_mark(). Any non-0 value suffices. |
3360 // This is convenient but results a ST-before-CAS penalty. The following CAS suffers | 3361 // This is convenient but results a ST-before-CAS penalty. The following CAS suffers |
3361 // additional latency as we have another ST in the store buffer that must drain. | 3362 // additional latency as we have another ST in the store buffer that must drain. |
3362 | 3363 |
3363 if (EmitSync & 8192) { | 3364 if (EmitSync & 8192) { |
3364 masm.movl (Address(boxReg, 0), 3) ; // results in ST-before-CAS penalty | 3365 masm.movptr(Address(boxReg, 0), 3) ; // results in ST-before-CAS penalty |
3365 masm.get_thread (scrReg) ; | 3366 masm.get_thread (scrReg) ; |
3366 masm.movl (boxReg, tmpReg); // consider: LEA box, [tmp-2] | 3367 masm.movptr(boxReg, tmpReg); // consider: LEA box, [tmp-2] |
3367 masm.movl (tmpReg, 0); // consider: xor vs mov | 3368 masm.movptr(tmpReg, 0); // consider: xor vs mov |
3368 if (os::is_MP()) { masm.lock(); } | 3369 if (os::is_MP()) { masm.lock(); } |
3369 masm.cmpxchg (scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; | 3370 masm.cmpxchgptr(scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; |
3370 } else | 3371 } else |
3371 if ((EmitSync & 128) == 0) { // avoid ST-before-CAS | 3372 if ((EmitSync & 128) == 0) { // avoid ST-before-CAS |
3372 masm.movl (scrReg, boxReg) ; | 3373 masm.movptr(scrReg, boxReg) ; |
3373 masm.movl (boxReg, tmpReg); // consider: LEA box, [tmp-2] | 3374 masm.movptr(boxReg, tmpReg); // consider: LEA box, [tmp-2] |
3374 | 3375 |
3375 // Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes | 3376 // Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes |
3376 if ((EmitSync & 2048) && VM_Version::supports_3dnow() && os::is_MP()) { | 3377 if ((EmitSync & 2048) && VM_Version::supports_3dnow() && os::is_MP()) { |
3377 // prefetchw [eax + Offset(_owner)-2] | 3378 // prefetchw [eax + Offset(_owner)-2] |
3378 masm.emit_raw (0x0F) ; | 3379 masm.prefetchw(Address(rax, ObjectMonitor::owner_offset_in_bytes()-2)); |
3379 masm.emit_raw (0x0D) ; | |
3380 masm.emit_raw (0x48) ; | |
3381 masm.emit_raw (ObjectMonitor::owner_offset_in_bytes()-2) ; | |
3382 } | 3380 } |
3383 | 3381 |
3384 if ((EmitSync & 64) == 0) { | 3382 if ((EmitSync & 64) == 0) { |
3385 // Optimistic form: consider XORL tmpReg,tmpReg | 3383 // Optimistic form: consider XORL tmpReg,tmpReg |
3386 masm.movl (tmpReg, 0 ) ; | 3384 masm.movptr(tmpReg, 0 ) ; |
3387 } else { | 3385 } else { |
3388 // Can suffer RTS->RTO upgrades on shared or cold $ lines | 3386 // Can suffer RTS->RTO upgrades on shared or cold $ lines |
3389 // Test-And-CAS instead of CAS | 3387 // Test-And-CAS instead of CAS |
3390 masm.movl (tmpReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; // rax, = m->_owner | 3388 masm.movptr(tmpReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; // rax, = m->_owner |
3391 masm.testl (tmpReg, tmpReg) ; // Locked ? | 3389 masm.testptr(tmpReg, tmpReg) ; // Locked ? |
3392 masm.jccb (Assembler::notZero, DONE_LABEL) ; | 3390 masm.jccb (Assembler::notZero, DONE_LABEL) ; |
3393 } | 3391 } |
3394 | 3392 |
3395 // Appears unlocked - try to swing _owner from null to non-null. | 3393 // Appears unlocked - try to swing _owner from null to non-null. |
3396 // Ideally, I'd manifest "Self" with get_thread and then attempt | 3394 // Ideally, I'd manifest "Self" with get_thread and then attempt |
3397 // to CAS the register containing Self into m->Owner. | 3395 // to CAS the register containing Self into m->Owner. |
3399 // rsp or the address of the box (in scr) into &m->owner. If the CAS succeeds | 3397 // rsp or the address of the box (in scr) into &m->owner. If the CAS succeeds |
3400 // we later store "Self" into m->Owner. Transiently storing a stack address | 3398 // we later store "Self" into m->Owner. Transiently storing a stack address |
3401 // (rsp or the address of the box) into m->owner is harmless. | 3399 // (rsp or the address of the box) into m->owner is harmless. |
3402 // Invariant: tmpReg == 0. tmpReg is EAX which is the implicit cmpxchg comparand. | 3400 // Invariant: tmpReg == 0. tmpReg is EAX which is the implicit cmpxchg comparand. |
3403 if (os::is_MP()) { masm.lock(); } | 3401 if (os::is_MP()) { masm.lock(); } |
3404 masm.cmpxchg (scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; | 3402 masm.cmpxchgptr(scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; |
3405 masm.movl (Address(scrReg, 0), 3) ; // box->_displaced_header = 3 | 3403 masm.movptr(Address(scrReg, 0), 3) ; // box->_displaced_header = 3 |
3406 masm.jccb (Assembler::notZero, DONE_LABEL) ; | 3404 masm.jccb (Assembler::notZero, DONE_LABEL) ; |
3407 masm.get_thread (scrReg) ; // beware: clobbers ICCs | 3405 masm.get_thread (scrReg) ; // beware: clobbers ICCs |
3408 masm.movl (Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2), scrReg) ; | 3406 masm.movptr(Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2), scrReg) ; |
3409 masm.xorl (boxReg, boxReg) ; // set icc.ZFlag = 1 to indicate success | 3407 masm.xorptr(boxReg, boxReg) ; // set icc.ZFlag = 1 to indicate success |
3410 | 3408 |
3411 // If the CAS fails we can either retry or pass control to the slow-path. | 3409 // If the CAS fails we can either retry or pass control to the slow-path. |
3412 // We use the latter tactic. | 3410 // We use the latter tactic. |
3413 // Pass the CAS result in the icc.ZFlag into DONE_LABEL | 3411 // Pass the CAS result in the icc.ZFlag into DONE_LABEL |
3414 // If the CAS was successful ... | 3412 // If the CAS was successful ... |
3415 // Self has acquired the lock | 3413 // Self has acquired the lock |
3416 // Invariant: m->_recursions should already be 0, so we don't need to explicitly set it. | 3414 // Invariant: m->_recursions should already be 0, so we don't need to explicitly set it. |
3417 // Intentional fall-through into DONE_LABEL ... | 3415 // Intentional fall-through into DONE_LABEL ... |
3418 } else { | 3416 } else { |
3419 masm.movl (Address(boxReg, 0), 3) ; // results in ST-before-CAS penalty | 3417 masm.movptr(Address(boxReg, 0), 3) ; // results in ST-before-CAS penalty |
3420 masm.movl (boxReg, tmpReg) ; | 3418 masm.movptr(boxReg, tmpReg) ; |
3421 | 3419 |
3422 // Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes | 3420 // Using a prefetchw helps avoid later RTS->RTO upgrades and cache probes |
3423 if ((EmitSync & 2048) && VM_Version::supports_3dnow() && os::is_MP()) { | 3421 if ((EmitSync & 2048) && VM_Version::supports_3dnow() && os::is_MP()) { |
3424 // prefetchw [eax + Offset(_owner)-2] | 3422 // prefetchw [eax + Offset(_owner)-2] |
3425 masm.emit_raw (0x0F) ; | 3423 masm.prefetchw(Address(rax, ObjectMonitor::owner_offset_in_bytes()-2)); |
3426 masm.emit_raw (0x0D) ; | |
3427 masm.emit_raw (0x48) ; | |
3428 masm.emit_raw (ObjectMonitor::owner_offset_in_bytes()-2) ; | |
3429 } | 3424 } |
3430 | 3425 |
3431 if ((EmitSync & 64) == 0) { | 3426 if ((EmitSync & 64) == 0) { |
3432 // Optimistic form | 3427 // Optimistic form |
3433 masm.xorl (tmpReg, tmpReg) ; | 3428 masm.xorptr (tmpReg, tmpReg) ; |
3434 } else { | 3429 } else { |
3435 // Can suffer RTS->RTO upgrades on shared or cold $ lines | 3430 // Can suffer RTS->RTO upgrades on shared or cold $ lines |
3436 masm.movl (tmpReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; // rax, = m->_owner | 3431 masm.movptr(tmpReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; // rax, = m->_owner |
3437 masm.testl (tmpReg, tmpReg) ; // Locked ? | 3432 masm.testptr(tmpReg, tmpReg) ; // Locked ? |
3438 masm.jccb (Assembler::notZero, DONE_LABEL) ; | 3433 masm.jccb (Assembler::notZero, DONE_LABEL) ; |
3439 } | 3434 } |
3440 | 3435 |
3441 // Appears unlocked - try to swing _owner from null to non-null. | 3436 // Appears unlocked - try to swing _owner from null to non-null. |
3442 // Use either "Self" (in scr) or rsp as thread identity in _owner. | 3437 // Use either "Self" (in scr) or rsp as thread identity in _owner. |
3443 // Invariant: tmpReg == 0. tmpReg is EAX which is the implicit cmpxchg comparand. | 3438 // Invariant: tmpReg == 0. tmpReg is EAX which is the implicit cmpxchg comparand. |
3444 masm.get_thread (scrReg) ; | 3439 masm.get_thread (scrReg) ; |
3445 if (os::is_MP()) { masm.lock(); } | 3440 if (os::is_MP()) { masm.lock(); } |
3446 masm.cmpxchg (scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; | 3441 masm.cmpxchgptr(scrReg, Address(boxReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; |
3447 | 3442 |
3448 // If the CAS fails we can either retry or pass control to the slow-path. | 3443 // If the CAS fails we can either retry or pass control to the slow-path. |
3449 // We use the latter tactic. | 3444 // We use the latter tactic. |
3450 // Pass the CAS result in the icc.ZFlag into DONE_LABEL | 3445 // Pass the CAS result in the icc.ZFlag into DONE_LABEL |
3451 // If the CAS was successful ... | 3446 // If the CAS was successful ... |
3512 guarantee (boxReg == as_Register(EAX_enc), "") ; | 3507 guarantee (boxReg == as_Register(EAX_enc), "") ; |
3513 MacroAssembler masm(&cbuf); | 3508 MacroAssembler masm(&cbuf); |
3514 | 3509 |
3515 if (EmitSync & 4) { | 3510 if (EmitSync & 4) { |
3516 // Disable - inhibit all inlining. Force control through the slow-path | 3511 // Disable - inhibit all inlining. Force control through the slow-path |
3517 masm.cmpl (rsp, 0) ; | 3512 masm.cmpptr (rsp, 0) ; |
3518 } else | 3513 } else |
3519 if (EmitSync & 8) { | 3514 if (EmitSync & 8) { |
3520 Label DONE_LABEL ; | 3515 Label DONE_LABEL ; |
3521 if (UseBiasedLocking) { | 3516 if (UseBiasedLocking) { |
3522 masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL); | 3517 masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL); |
3523 } | 3518 } |
3524 // classic stack-locking code ... | 3519 // classic stack-locking code ... |
3525 masm.movl (tmpReg, Address(boxReg, 0)) ; | 3520 masm.movptr(tmpReg, Address(boxReg, 0)) ; |
3526 masm.testl (tmpReg, tmpReg) ; | 3521 masm.testptr(tmpReg, tmpReg) ; |
3527 masm.jcc (Assembler::zero, DONE_LABEL) ; | 3522 masm.jcc (Assembler::zero, DONE_LABEL) ; |
3528 if (os::is_MP()) { masm.lock(); } | 3523 if (os::is_MP()) { masm.lock(); } |
3529 masm.cmpxchg(tmpReg, Address(objReg, 0)); // Uses EAX which is box | 3524 masm.cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses EAX which is box |
3530 masm.bind(DONE_LABEL); | 3525 masm.bind(DONE_LABEL); |
3531 } else { | 3526 } else { |
3532 Label DONE_LABEL, Stacked, CheckSucc, Inflated ; | 3527 Label DONE_LABEL, Stacked, CheckSucc, Inflated ; |
3533 | 3528 |
3534 // Critically, the biased locking test must have precedence over | 3529 // Critically, the biased locking test must have precedence over |
3535 // and appear before the (box->dhw == 0) recursive stack-lock test. | 3530 // and appear before the (box->dhw == 0) recursive stack-lock test. |
3536 if (UseBiasedLocking) { | 3531 if (UseBiasedLocking) { |
3537 masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL); | 3532 masm.biased_locking_exit(objReg, tmpReg, DONE_LABEL); |
3538 } | 3533 } |
3539 | 3534 |
3540 masm.cmpl (Address(boxReg, 0), 0) ; // Examine the displaced header | 3535 masm.cmpptr(Address(boxReg, 0), 0) ; // Examine the displaced header |
3541 masm.movl (tmpReg, Address(objReg, 0)) ; // Examine the object's markword | 3536 masm.movptr(tmpReg, Address(objReg, 0)) ; // Examine the object's markword |
3542 masm.jccb (Assembler::zero, DONE_LABEL) ; // 0 indicates recursive stack-lock | 3537 masm.jccb (Assembler::zero, DONE_LABEL) ; // 0 indicates recursive stack-lock |
3543 | 3538 |
3544 masm.testl (tmpReg, 0x02) ; // Inflated? | 3539 masm.testptr(tmpReg, 0x02) ; // Inflated? |
3545 masm.jccb (Assembler::zero, Stacked) ; | 3540 masm.jccb (Assembler::zero, Stacked) ; |
3546 | 3541 |
3547 masm.bind (Inflated) ; | 3542 masm.bind (Inflated) ; |
3548 // It's inflated. | 3543 // It's inflated. |
3549 // Despite our balanced locking property we still check that m->_owner == Self | 3544 // Despite our balanced locking property we still check that m->_owner == Self |
3569 // each other and there's no need for an explicit barrier (fence). | 3564 // each other and there's no need for an explicit barrier (fence). |
3570 // See also http://gee.cs.oswego.edu/dl/jmm/cookbook.html. | 3565 // See also http://gee.cs.oswego.edu/dl/jmm/cookbook.html. |
3571 | 3566 |
3572 masm.get_thread (boxReg) ; | 3567 masm.get_thread (boxReg) ; |
3573 if ((EmitSync & 4096) && VM_Version::supports_3dnow() && os::is_MP()) { | 3568 if ((EmitSync & 4096) && VM_Version::supports_3dnow() && os::is_MP()) { |
3574 // prefetchw [ebx + Offset(_owner)-2] | 3569 // prefetchw [ebx + Offset(_owner)-2] |
3575 masm.emit_raw (0x0F) ; | 3570 masm.prefetchw(Address(rbx, ObjectMonitor::owner_offset_in_bytes()-2)); |
3576 masm.emit_raw (0x0D) ; | |
3577 masm.emit_raw (0x4B) ; | |
3578 masm.emit_raw (ObjectMonitor::owner_offset_in_bytes()-2) ; | |
3579 } | 3571 } |
3580 | 3572 |
3581 // Note that we could employ various encoding schemes to reduce | 3573 // Note that we could employ various encoding schemes to reduce |
3582 // the number of loads below (currently 4) to just 2 or 3. | 3574 // the number of loads below (currently 4) to just 2 or 3. |
3583 // Refer to the comments in synchronizer.cpp. | 3575 // Refer to the comments in synchronizer.cpp. |
3584 // In practice the chain of fetches doesn't seem to impact performance, however. | 3576 // In practice the chain of fetches doesn't seem to impact performance, however. |
3585 if ((EmitSync & 65536) == 0 && (EmitSync & 256)) { | 3577 if ((EmitSync & 65536) == 0 && (EmitSync & 256)) { |
3586 // Attempt to reduce branch density - AMD's branch predictor. | 3578 // Attempt to reduce branch density - AMD's branch predictor. |
3587 masm.xorl (boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; | 3579 masm.xorptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; |
3588 masm.orl (boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2)) ; | 3580 masm.orptr(boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2)) ; |
3589 masm.orl (boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2)) ; | 3581 masm.orptr(boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2)) ; |
3590 masm.orl (boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2)) ; | 3582 masm.orptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2)) ; |
3591 masm.jccb (Assembler::notZero, DONE_LABEL) ; | 3583 masm.jccb (Assembler::notZero, DONE_LABEL) ; |
3592 masm.movl (Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), 0) ; | 3584 masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), 0) ; |
3593 masm.jmpb (DONE_LABEL) ; | 3585 masm.jmpb (DONE_LABEL) ; |
3594 } else { | 3586 } else { |
3595 masm.xorl (boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; | 3587 masm.xorptr(boxReg, Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)) ; |
3596 masm.orl (boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2)) ; | 3588 masm.orptr(boxReg, Address (tmpReg, ObjectMonitor::recursions_offset_in_bytes()-2)) ; |
3597 masm.jccb (Assembler::notZero, DONE_LABEL) ; | 3589 masm.jccb (Assembler::notZero, DONE_LABEL) ; |
3598 masm.movl (boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2)) ; | 3590 masm.movptr(boxReg, Address (tmpReg, ObjectMonitor::EntryList_offset_in_bytes()-2)) ; |
3599 masm.orl (boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2)) ; | 3591 masm.orptr(boxReg, Address (tmpReg, ObjectMonitor::cxq_offset_in_bytes()-2)) ; |
3600 masm.jccb (Assembler::notZero, CheckSucc) ; | 3592 masm.jccb (Assembler::notZero, CheckSucc) ; |
3601 masm.movl (Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), 0) ; | 3593 masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), 0) ; |
3602 masm.jmpb (DONE_LABEL) ; | 3594 masm.jmpb (DONE_LABEL) ; |
3603 } | 3595 } |
3604 | 3596 |
3605 // The Following code fragment (EmitSync & 65536) improves the performance of | 3597 // The Following code fragment (EmitSync & 65536) improves the performance of |
3606 // contended applications and contended synchronization microbenchmarks. | 3598 // contended applications and contended synchronization microbenchmarks. |
3607 // Unfortunately the emission of the code - even though not executed - causes regressions | 3599 // Unfortunately the emission of the code - even though not executed - causes regressions |
3613 Label LSuccess, LGoSlowPath ; | 3605 Label LSuccess, LGoSlowPath ; |
3614 | 3606 |
3615 masm.bind (CheckSucc) ; | 3607 masm.bind (CheckSucc) ; |
3616 | 3608 |
3617 // Optional pre-test ... it's safe to elide this | 3609 // Optional pre-test ... it's safe to elide this |
3618 if ((EmitSync & 16) == 0) { | 3610 if ((EmitSync & 16) == 0) { |
3619 masm.cmpl (Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), 0) ; | 3611 masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), 0) ; |
3620 masm.jccb (Assembler::zero, LGoSlowPath) ; | 3612 masm.jccb (Assembler::zero, LGoSlowPath) ; |
3621 } | 3613 } |
3622 | 3614 |
3623 // We have a classic Dekker-style idiom: | 3615 // We have a classic Dekker-style idiom: |
3624 // ST m->_owner = 0 ; MEMBAR; LD m->_succ | 3616 // ST m->_owner = 0 ; MEMBAR; LD m->_succ |
3625 // There are a number of ways to implement the barrier: | 3617 // There are a number of ways to implement the barrier: |
3643 // we just stored into _owner, it's likely that the $line | 3635 // we just stored into _owner, it's likely that the $line |
3644 // remains in M-state for the lock:orl. | 3636 // remains in M-state for the lock:orl. |
3645 // | 3637 // |
3646 // We currently use (3), although it's likely that switching to (2) | 3638 // We currently use (3), although it's likely that switching to (2) |
3647 // is correct for the future. | 3639 // is correct for the future. |
3648 | 3640 |
3649 masm.movl (Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), 0) ; | 3641 masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), 0) ; |
3650 if (os::is_MP()) { | 3642 if (os::is_MP()) { |
3651 if (VM_Version::supports_sse2() && 1 == FenceInstruction) { | 3643 if (VM_Version::supports_sse2() && 1 == FenceInstruction) { |
3652 masm.emit_raw (0x0F) ; // MFENCE ... | 3644 masm.mfence(); |
3653 masm.emit_raw (0xAE) ; | 3645 } else { |
3654 masm.emit_raw (0xF0) ; | 3646 masm.lock () ; masm.addptr(Address(rsp, 0), 0) ; |
3655 } else { | |
3656 masm.lock () ; masm.addl (Address(rsp, 0), 0) ; | |
3657 } | 3647 } |
3658 } | 3648 } |
3659 // Ratify _succ remains non-null | 3649 // Ratify _succ remains non-null |
3660 masm.cmpl (Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), 0) ; | 3650 masm.cmpptr(Address (tmpReg, ObjectMonitor::succ_offset_in_bytes()-2), 0) ; |
3661 masm.jccb (Assembler::notZero, LSuccess) ; | 3651 masm.jccb (Assembler::notZero, LSuccess) ; |
3662 | 3652 |
3663 masm.xorl (boxReg, boxReg) ; // box is really EAX | 3653 masm.xorptr(boxReg, boxReg) ; // box is really EAX |
3664 if (os::is_MP()) { masm.lock(); } | 3654 if (os::is_MP()) { masm.lock(); } |
3665 masm.cmpxchg(rsp, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)); | 3655 masm.cmpxchgptr(rsp, Address(tmpReg, ObjectMonitor::owner_offset_in_bytes()-2)); |
3666 masm.jccb (Assembler::notEqual, LSuccess) ; | 3656 masm.jccb (Assembler::notEqual, LSuccess) ; |
3667 // Since we're low on registers we installed rsp as a placeholding in _owner. | 3657 // Since we're low on registers we installed rsp as a placeholding in _owner. |
3668 // Now install Self over rsp. This is safe as we're transitioning from | 3658 // Now install Self over rsp. This is safe as we're transitioning from |
3669 // non-null to non=null | 3659 // non-null to non=null |
3670 masm.get_thread (boxReg) ; | 3660 masm.get_thread (boxReg) ; |
3671 masm.movl (Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), boxReg) ; | 3661 masm.movptr(Address (tmpReg, ObjectMonitor::owner_offset_in_bytes()-2), boxReg) ; |
3672 // Intentional fall-through into LGoSlowPath ... | 3662 // Intentional fall-through into LGoSlowPath ... |
3673 | 3663 |
3674 masm.bind (LGoSlowPath) ; | 3664 masm.bind (LGoSlowPath) ; |
3675 masm.orl (boxReg, 1) ; // set ICC.ZF=0 to indicate failure | 3665 masm.orptr(boxReg, 1) ; // set ICC.ZF=0 to indicate failure |
3676 masm.jmpb (DONE_LABEL) ; | 3666 masm.jmpb (DONE_LABEL) ; |
3677 | 3667 |
3678 masm.bind (LSuccess) ; | 3668 masm.bind (LSuccess) ; |
3679 masm.xorl (boxReg, boxReg) ; // set ICC.ZF=1 to indicate success | 3669 masm.xorptr(boxReg, boxReg) ; // set ICC.ZF=1 to indicate success |
3680 masm.jmpb (DONE_LABEL) ; | 3670 masm.jmpb (DONE_LABEL) ; |
3681 } | 3671 } |
3682 | 3672 |
3683 masm.bind (Stacked) ; | 3673 masm.bind (Stacked) ; |
3684 // It's not inflated and it's not recursively stack-locked and it's not biased. | 3674 // It's not inflated and it's not recursively stack-locked and it's not biased. |
3685 // It must be stack-locked. | 3675 // It must be stack-locked. |
3686 // Try to reset the header to displaced header. | 3676 // Try to reset the header to displaced header. |
3687 // The "box" value on the stack is stable, so we can reload | 3677 // The "box" value on the stack is stable, so we can reload |
3688 // and be assured we observe the same value as above. | 3678 // and be assured we observe the same value as above. |
3689 masm.movl (tmpReg, Address(boxReg, 0)) ; | 3679 masm.movptr(tmpReg, Address(boxReg, 0)) ; |
3690 if (os::is_MP()) { masm.lock(); } | 3680 if (os::is_MP()) { masm.lock(); } |
3691 masm.cmpxchg(tmpReg, Address(objReg, 0)); // Uses EAX which is box | 3681 masm.cmpxchgptr(tmpReg, Address(objReg, 0)); // Uses EAX which is box |
3692 // Intention fall-thru into DONE_LABEL | 3682 // Intention fall-thru into DONE_LABEL |
3693 | 3683 |
3694 | 3684 |
3695 // DONE_LABEL is a hot target - we'd really like to place it at the | 3685 // DONE_LABEL is a hot target - we'd really like to place it at the |
3696 // start of cache line by padding with NOPs. | 3686 // start of cache line by padding with NOPs. |
3718 int value_offset = java_lang_String::value_offset_in_bytes(); | 3708 int value_offset = java_lang_String::value_offset_in_bytes(); |
3719 int offset_offset = java_lang_String::offset_offset_in_bytes(); | 3709 int offset_offset = java_lang_String::offset_offset_in_bytes(); |
3720 int count_offset = java_lang_String::count_offset_in_bytes(); | 3710 int count_offset = java_lang_String::count_offset_in_bytes(); |
3721 int base_offset = arrayOopDesc::base_offset_in_bytes(T_CHAR); | 3711 int base_offset = arrayOopDesc::base_offset_in_bytes(T_CHAR); |
3722 | 3712 |
3723 masm.movl(rax, Address(rsi, value_offset)); | 3713 masm.movptr(rax, Address(rsi, value_offset)); |
3724 masm.movl(rcx, Address(rsi, offset_offset)); | 3714 masm.movl(rcx, Address(rsi, offset_offset)); |
3725 masm.leal(rax, Address(rax, rcx, Address::times_2, base_offset)); | 3715 masm.lea(rax, Address(rax, rcx, Address::times_2, base_offset)); |
3726 masm.movl(rbx, Address(rdi, value_offset)); | 3716 masm.movptr(rbx, Address(rdi, value_offset)); |
3727 masm.movl(rcx, Address(rdi, offset_offset)); | 3717 masm.movl(rcx, Address(rdi, offset_offset)); |
3728 masm.leal(rbx, Address(rbx, rcx, Address::times_2, base_offset)); | 3718 masm.lea(rbx, Address(rbx, rcx, Address::times_2, base_offset)); |
3729 | 3719 |
3730 // Compute the minimum of the string lengths(rsi) and the | 3720 // Compute the minimum of the string lengths(rsi) and the |
3731 // difference of the string lengths (stack) | 3721 // difference of the string lengths (stack) |
3732 | 3722 |
3733 | 3723 |
3734 if (VM_Version::supports_cmov()) { | 3724 if (VM_Version::supports_cmov()) { |
3735 masm.movl(rdi, Address(rdi, count_offset)); | 3725 masm.movl(rdi, Address(rdi, count_offset)); |
3736 masm.movl(rsi, Address(rsi, count_offset)); | 3726 masm.movl(rsi, Address(rsi, count_offset)); |
3737 masm.movl(rcx, rdi); | 3727 masm.movl(rcx, rdi); |
3738 masm.subl(rdi, rsi); | 3728 masm.subl(rdi, rsi); |
3739 masm.pushl(rdi); | 3729 masm.push(rdi); |
3740 masm.cmovl(Assembler::lessEqual, rsi, rcx); | 3730 masm.cmovl(Assembler::lessEqual, rsi, rcx); |
3741 } else { | 3731 } else { |
3742 masm.movl(rdi, Address(rdi, count_offset)); | 3732 masm.movl(rdi, Address(rdi, count_offset)); |
3743 masm.movl(rcx, Address(rsi, count_offset)); | 3733 masm.movl(rcx, Address(rsi, count_offset)); |
3744 masm.movl(rsi, rdi); | 3734 masm.movl(rsi, rdi); |
3745 masm.subl(rdi, rcx); | 3735 masm.subl(rdi, rcx); |
3746 masm.pushl(rdi); | 3736 masm.push(rdi); |
3747 masm.jcc(Assembler::lessEqual, ECX_GOOD_LABEL); | 3737 masm.jcc(Assembler::lessEqual, ECX_GOOD_LABEL); |
3748 masm.movl(rsi, rcx); | 3738 masm.movl(rsi, rcx); |
3749 // rsi holds min, rcx is unused | 3739 // rsi holds min, rcx is unused |
3750 } | 3740 } |
3751 | 3741 |
3759 masm.load_unsigned_word(rdi, Address(rax, 0)); | 3749 masm.load_unsigned_word(rdi, Address(rax, 0)); |
3760 | 3750 |
3761 // Compare first characters | 3751 // Compare first characters |
3762 masm.subl(rcx, rdi); | 3752 masm.subl(rcx, rdi); |
3763 masm.jcc(Assembler::notZero, POP_LABEL); | 3753 masm.jcc(Assembler::notZero, POP_LABEL); |
3764 masm.decrement(rsi); | 3754 masm.decrementl(rsi); |
3765 masm.jcc(Assembler::zero, LENGTH_DIFF_LABEL); | 3755 masm.jcc(Assembler::zero, LENGTH_DIFF_LABEL); |
3766 | 3756 |
3767 { | 3757 { |
3768 // Check after comparing first character to see if strings are equivalent | 3758 // Check after comparing first character to see if strings are equivalent |
3769 Label LSkip2; | 3759 Label LSkip2; |
3770 // Check if the strings start at same location | 3760 // Check if the strings start at same location |
3771 masm.cmpl(rbx,rax); | 3761 masm.cmpptr(rbx,rax); |
3772 masm.jcc(Assembler::notEqual, LSkip2); | 3762 masm.jcc(Assembler::notEqual, LSkip2); |
3773 | 3763 |
3774 // Check if the length difference is zero (from stack) | 3764 // Check if the length difference is zero (from stack) |
3775 masm.cmpl(Address(rsp, 0), 0x0); | 3765 masm.cmpl(Address(rsp, 0), 0x0); |
3776 masm.jcc(Assembler::equal, LENGTH_DIFF_LABEL); | 3766 masm.jcc(Assembler::equal, LENGTH_DIFF_LABEL); |
3778 // Strings might not be equivalent | 3768 // Strings might not be equivalent |
3779 masm.bind(LSkip2); | 3769 masm.bind(LSkip2); |
3780 } | 3770 } |
3781 | 3771 |
3782 // Shift rax, and rbx, to the end of the arrays, negate min | 3772 // Shift rax, and rbx, to the end of the arrays, negate min |
3783 masm.leal(rax, Address(rax, rsi, Address::times_2, 2)); | 3773 masm.lea(rax, Address(rax, rsi, Address::times_2, 2)); |
3784 masm.leal(rbx, Address(rbx, rsi, Address::times_2, 2)); | 3774 masm.lea(rbx, Address(rbx, rsi, Address::times_2, 2)); |
3785 masm.negl(rsi); | 3775 masm.negl(rsi); |
3786 | 3776 |
3787 // Compare the rest of the characters | 3777 // Compare the rest of the characters |
3788 masm.bind(WHILE_HEAD_LABEL); | 3778 masm.bind(WHILE_HEAD_LABEL); |
3789 masm.load_unsigned_word(rcx, Address(rbx, rsi, Address::times_2, 0)); | 3779 masm.load_unsigned_word(rcx, Address(rbx, rsi, Address::times_2, 0)); |
3790 masm.load_unsigned_word(rdi, Address(rax, rsi, Address::times_2, 0)); | 3780 masm.load_unsigned_word(rdi, Address(rax, rsi, Address::times_2, 0)); |
3791 masm.subl(rcx, rdi); | 3781 masm.subl(rcx, rdi); |
3792 masm.jcc(Assembler::notZero, POP_LABEL); | 3782 masm.jcc(Assembler::notZero, POP_LABEL); |
3793 masm.increment(rsi); | 3783 masm.incrementl(rsi); |
3794 masm.jcc(Assembler::notZero, WHILE_HEAD_LABEL); | 3784 masm.jcc(Assembler::notZero, WHILE_HEAD_LABEL); |
3795 | 3785 |
3796 // Strings are equal up to min length. Return the length difference. | 3786 // Strings are equal up to min length. Return the length difference. |
3797 masm.bind(LENGTH_DIFF_LABEL); | 3787 masm.bind(LENGTH_DIFF_LABEL); |
3798 masm.popl(rcx); | 3788 masm.pop(rcx); |
3799 masm.jmp(DONE_LABEL); | 3789 masm.jmp(DONE_LABEL); |
3800 | 3790 |
3801 // Discard the stored length difference | 3791 // Discard the stored length difference |
3802 masm.bind(POP_LABEL); | 3792 masm.bind(POP_LABEL); |
3803 masm.addl(rsp, 4); | 3793 masm.addptr(rsp, 4); |
3804 | 3794 |
3805 // That's it | 3795 // That's it |
3806 masm.bind(DONE_LABEL); | 3796 masm.bind(DONE_LABEL); |
3807 %} | 3797 %} |
3808 | 3798 |
3809 enc_class enc_Array_Equals(eDIRegP ary1, eSIRegP ary2, eAXRegI tmp1, eBXRegI tmp2, eCXRegI result) %{ | 3799 enc_class enc_Array_Equals(eDIRegP ary1, eSIRegP ary2, eAXRegI tmp1, eBXRegI tmp2, eCXRegI result) %{ |
4313 // masm.membar(); | 4303 // masm.membar(); |
4314 %} | 4304 %} |
4315 | 4305 |
4316 enc_class enc_membar_volatile %{ | 4306 enc_class enc_membar_volatile %{ |
4317 MacroAssembler masm(&cbuf); | 4307 MacroAssembler masm(&cbuf); |
4318 masm.membar(); | 4308 masm.membar(Assembler::Membar_mask_bits(Assembler::StoreLoad | |
4309 Assembler::StoreStore)); | |
4319 %} | 4310 %} |
4320 | 4311 |
4321 // Atomically load the volatile long | 4312 // Atomically load the volatile long |
4322 enc_class enc_loadL_volatile( memory mem, stackSlotL dst ) %{ | 4313 enc_class enc_loadL_volatile( memory mem, stackSlotL dst ) %{ |
4323 emit_opcode(cbuf,0xDF); | 4314 emit_opcode(cbuf,0xDF); |
11149 match(Set dst (ConvI2D src)); | 11140 match(Set dst (ConvI2D src)); |
11150 | 11141 |
11151 format %{ "MOVD $dst,$src\n\t" | 11142 format %{ "MOVD $dst,$src\n\t" |
11152 "CVTDQ2PD $dst,$dst\t# i2d" %} | 11143 "CVTDQ2PD $dst,$dst\t# i2d" %} |
11153 ins_encode %{ | 11144 ins_encode %{ |
11154 __ movd($dst$$XMMRegister, $src$$Register); | 11145 __ movdl($dst$$XMMRegister, $src$$Register); |
11155 __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister); | 11146 __ cvtdq2pd($dst$$XMMRegister, $dst$$XMMRegister); |
11156 %} | 11147 %} |
11157 ins_pipe(pipe_slow); // XXX | 11148 ins_pipe(pipe_slow); // XXX |
11158 %} | 11149 %} |
11159 | 11150 |
11247 match(Set dst (ConvI2F src)); | 11238 match(Set dst (ConvI2F src)); |
11248 | 11239 |
11249 format %{ "MOVD $dst,$src\n\t" | 11240 format %{ "MOVD $dst,$src\n\t" |
11250 "CVTDQ2PS $dst,$dst\t# i2f" %} | 11241 "CVTDQ2PS $dst,$dst\t# i2f" %} |
11251 ins_encode %{ | 11242 ins_encode %{ |
11252 __ movd($dst$$XMMRegister, $src$$Register); | 11243 __ movdl($dst$$XMMRegister, $src$$Register); |
11253 __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister); | 11244 __ cvtdq2ps($dst$$XMMRegister, $dst$$XMMRegister); |
11254 %} | 11245 %} |
11255 ins_pipe(pipe_slow); // XXX | 11246 ins_pipe(pipe_slow); // XXX |
11256 %} | 11247 %} |
11257 | 11248 |
12260 "JMP,s done\n" | 12251 "JMP,s done\n" |
12261 "m_one:\tDEC $dst\n" | 12252 "m_one:\tDEC $dst\n" |
12262 "done:" %} | 12253 "done:" %} |
12263 ins_encode %{ | 12254 ins_encode %{ |
12264 Label p_one, m_one, done; | 12255 Label p_one, m_one, done; |
12265 __ xorl($dst$$Register, $dst$$Register); | 12256 __ xorptr($dst$$Register, $dst$$Register); |
12266 __ cmpl(HIGH_FROM_LOW($src1$$Register), HIGH_FROM_LOW($src2$$Register)); | 12257 __ cmpl(HIGH_FROM_LOW($src1$$Register), HIGH_FROM_LOW($src2$$Register)); |
12267 __ jccb(Assembler::less, m_one); | 12258 __ jccb(Assembler::less, m_one); |
12268 __ jccb(Assembler::greater, p_one); | 12259 __ jccb(Assembler::greater, p_one); |
12269 __ cmpl($src1$$Register, $src2$$Register); | 12260 __ cmpl($src1$$Register, $src2$$Register); |
12270 __ jccb(Assembler::below, m_one); | 12261 __ jccb(Assembler::below, m_one); |
12271 __ jccb(Assembler::equal, done); | 12262 __ jccb(Assembler::equal, done); |
12272 __ bind(p_one); | 12263 __ bind(p_one); |
12273 __ increment($dst$$Register); | 12264 __ incrementl($dst$$Register); |
12274 __ jmpb(done); | 12265 __ jmpb(done); |
12275 __ bind(m_one); | 12266 __ bind(m_one); |
12276 __ decrement($dst$$Register); | 12267 __ decrementl($dst$$Register); |
12277 __ bind(done); | 12268 __ bind(done); |
12278 %} | 12269 %} |
12279 ins_pipe( pipe_slow ); | 12270 ins_pipe( pipe_slow ); |
12280 %} | 12271 %} |
12281 | 12272 |