truffle: src/cpu/x86/vm/assembler

comparison src/cpu/x86/vm/assembler_x86.cpp @ 6948:e522a00b91aa

Merge with http://hg.openjdk.java.net/hsx/hsx25/hotspot/ after NPG - C++ build works

author	Doug Simon <doug.simon@oracle.com>
date	Mon, 12 Nov 2012 23:14:12 +0100
parents	c38f13903fdf a3ecd773a7b9
children	291ffc492eb6

comparison

equal deleted inserted replaced

-:ae13cc658b80
+:e522a00b91aa
 AddressLiteral::AddressLiteral(address target, relocInfo::relocType rtype) {
 _is_lval = false;
 _target = target;
 switch (rtype) {
 case relocInfo::oop_type:
+case relocInfo::metadata_type:
 // Oops are a special case. Normally they would be their own section
 // but in cases like icBuffer they are literals in the code stream that
 // we don't have a section for. We use none so that we get a literal address
 // which is always patchable.
 break;
 // Convert the raw encoding form into the form expected by the constructor for
 // Address.  An index of 4 (rsp) corresponds to having no index, so convert
 // that to noreg for the Address constructor.
-Address Address::make_raw(int base, int index, int scale, int disp, bool disp_is_oop) {
+Address Address::make_raw(int base, int index, int scale, int disp, relocInfo::relocType disp_reloc) {
 RelocationHolder rspec;
-if (disp_is_oop) {
+if (disp_reloc != relocInfo::none) {
-rspec = Relocation::spec_simple(relocInfo::oop_type);
+rspec = Relocation::spec_simple(disp_reloc);
 }
 bool valid_index = index != rsp->encoding();
 if (valid_index) {
 Address madr(as_Register(base), as_Register(index), (Address::ScaleFactor)scale, in_ByteSize(disp));
 madr._rspec = rspec;
 } else {
 emit_byte(op1);
 emit_operand(rm, adr, 4);
 emit_long(imm32);
 }
-}
-void Assembler::emit_arith(int op1, int op2, Register dst, jobject obj) {
-LP64_ONLY(ShouldNotReachHere());
-assert(isByte(op1) && isByte(op2), "wrong opcode");
-assert((op1 & 0x01) == 1, "should be 32bit operation");
-assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
-InstructionMark im(this);
-emit_byte(op1);
-emit_byte(op2 | encode(dst));
-emit_data((intptr_t)obj, relocInfo::oop_type, 0);
 }
 void Assembler::emit_arith(int op1, int op2, Register dst, Register src) {
 assert(isByte(op1) && isByte(op2), "wrong opcode");
 void Assembler::addss(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
 emit_simd_arith(0x58, dst, src, VEX_SIMD_F3);
 }
+void Assembler::aesdec(XMMRegister dst, Address src) {
+assert(VM_Version::supports_aes(), "");
+InstructionMark im(this);
+simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
+emit_byte(0xde);
+emit_operand(dst, src);
+}
+void Assembler::aesdec(XMMRegister dst, XMMRegister src) {
+assert(VM_Version::supports_aes(), "");
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
+emit_byte(0xde);
+emit_byte(0xC0 | encode);
+}
+void Assembler::aesdeclast(XMMRegister dst, Address src) {
+assert(VM_Version::supports_aes(), "");
+InstructionMark im(this);
+simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
+emit_byte(0xdf);
+emit_operand(dst, src);
+}
+void Assembler::aesdeclast(XMMRegister dst, XMMRegister src) {
+assert(VM_Version::supports_aes(), "");
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
+emit_byte(0xdf);
+emit_byte(0xC0 | encode);
+}
+void Assembler::aesenc(XMMRegister dst, Address src) {
+assert(VM_Version::supports_aes(), "");
+InstructionMark im(this);
+simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
+emit_byte(0xdc);
+emit_operand(dst, src);
+}
+void Assembler::aesenc(XMMRegister dst, XMMRegister src) {
+assert(VM_Version::supports_aes(), "");
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
+emit_byte(0xdc);
+emit_byte(0xC0 | encode);
+}
+void Assembler::aesenclast(XMMRegister dst, Address src) {
+assert(VM_Version::supports_aes(), "");
+InstructionMark im(this);
+simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
+emit_byte(0xdd);
+emit_operand(dst, src);
+}
+void Assembler::aesenclast(XMMRegister dst, XMMRegister src) {
+assert(VM_Version::supports_aes(), "");
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
+emit_byte(0xdd);
+emit_byte(0xC0 | encode);
+}
 void Assembler::andl(Address dst, int32_t imm32) {
 InstructionMark im(this);
 prefix(dst);
 emit_byte(0x81);
 emit_operand(rsp, dst, 4);
 // The 32-bit cmpxchg compares the value at adr with the contents of rax,
 // and stores reg into adr if so; otherwise, the value at adr is loaded into rax,.
 // The ZF is set if the compared values were equal, and cleared otherwise.
 void Assembler::cmpxchgl(Register reg, Address adr) { // cmpxchg
-if (Atomics & 2) {
+InstructionMark im(this);
-// caveat: no instructionmark, so this isn't relocatable.
+prefix(adr, reg);
-// Emit a synthetic, non-atomic, CAS equivalent.
+emit_byte(0x0F);
-// Beware.  The synthetic form sets all ICCs, not just ZF.
+emit_byte(0xB1);
-// cmpxchg r,[m] is equivalent to rax, = CAS (m, rax, r)
+emit_operand(reg, adr);
-cmpl(rax, adr);
-movl(rax, adr);
-if (reg != rax) {
-Label L ;
-jcc(Assembler::notEqual, L);
-movl(adr, reg);
-bind(L);
-}
-} else {
-InstructionMark im(this);
-prefix(adr, reg);
-emit_byte(0x0F);
-emit_byte(0xB1);
-emit_operand(reg, adr);
-}
 }
 void Assembler::comisd(XMMRegister dst, Address src) {
 // NOTE: dbx seems to decode this as comiss even though the
 // 0x66 is there. Strangly ucomisd comes out correct
 emit_byte(0x8D);
 emit_operand(dst, src);
 }
 void Assembler::lock() {
-if (Atomics & 1) {
+emit_byte(0xF0);
-// Emit either nothing, a NOP, or a NOP: prefix
-emit_byte(0x90) ;
-} else {
-emit_byte(0xF0);
-}
 }
 void Assembler::lzcntl(Register dst, Register src) {
 assert(VM_Version::supports_lzcnt(), "encoding is treated as BSR");
 emit_byte(0xF3);
 void Assembler::prefix(Prefix p) {
 a_byte(p);
 }
+void Assembler::pshufb(XMMRegister dst, XMMRegister src) {
+assert(VM_Version::supports_ssse3(), "");
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
+emit_byte(0x00);
+emit_byte(0xC0 | encode);
+}
+void Assembler::pshufb(XMMRegister dst, Address src) {
+assert(VM_Version::supports_ssse3(), "");
+assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
+InstructionMark im(this);
+simd_prefix(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
+emit_byte(0x00);
+emit_operand(dst, src);
+}
 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
 assert(isByte(mode), "invalid value");
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 emit_simd_arith_nonds(0x70, dst, src, VEX_SIMD_66);
 emit_byte(mode & 0xFF);
 // 0x00 - insert into lower 128 bits
 // 0x01 - insert into upper 128 bits
 emit_byte(0x01);
 }
+void Assembler::vinsertf128h(XMMRegister dst, Address src) {
+assert(VM_Version::supports_avx(), "");
+InstructionMark im(this);
+bool vector256 = true;
+assert(dst != xnoreg, "sanity");
+int dst_enc = dst->encoding();
+// swap src<->dst for encoding
+vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, false, vector256);
+emit_byte(0x18);
+emit_operand(dst, src);
+// 0x01 - insert into upper 128 bits
+emit_byte(0x01);
+}
+void Assembler::vextractf128h(Address dst, XMMRegister src) {
+assert(VM_Version::supports_avx(), "");
+InstructionMark im(this);
+bool vector256 = true;
+assert(src != xnoreg, "sanity");
+int src_enc = src->encoding();
+vex_prefix(dst, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, false, vector256);
+emit_byte(0x19);
+emit_operand(src, dst);
+// 0x01 - extract from upper 128 bits
+emit_byte(0x01);
+}
 void Assembler::vinserti128h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 assert(VM_Version::supports_avx2(), "");
 bool vector256 = true;
 int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector256, VEX_OPCODE_0F_3A);
 emit_byte(0x38);
 emit_byte(0xC0 | encode);
 // 0x00 - insert into lower 128 bits
 // 0x01 - insert into upper 128 bits
+emit_byte(0x01);
+}
+void Assembler::vinserti128h(XMMRegister dst, Address src) {
+assert(VM_Version::supports_avx2(), "");
+InstructionMark im(this);
+bool vector256 = true;
+assert(dst != xnoreg, "sanity");
+int dst_enc = dst->encoding();
+// swap src<->dst for encoding
+vex_prefix(src, dst_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, false, vector256);
+emit_byte(0x38);
+emit_operand(dst, src);
+// 0x01 - insert into upper 128 bits
+emit_byte(0x01);
+}
+void Assembler::vextracti128h(Address dst, XMMRegister src) {
+assert(VM_Version::supports_avx2(), "");
+InstructionMark im(this);
+bool vector256 = true;
+assert(src != xnoreg, "sanity");
+int src_enc = src->encoding();
+vex_prefix(dst, 0, src_enc, VEX_SIMD_66, VEX_OPCODE_0F_3A, false, vector256);
+emit_byte(0x39);
+emit_operand(src, dst);
+// 0x01 - extract from upper 128 bits
 emit_byte(0x01);
 }
 void Assembler::vzeroupper() {
 assert(VM_Version::supports_avx(), "");
 }
 void MacroAssembler::call_VM_leaf_base(address entry_point,
 int number_of_arguments) {
 call(RuntimeAddress(entry_point));
 increment(rsp, number_of_arguments * wordSize);
+}
+void MacroAssembler::cmpklass(Address src1, Metadata* obj) {
+cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
+}
+void MacroAssembler::cmpklass(Register src1, Metadata* obj) {
+cmp_literal32(src1, (int32_t)obj, metadata_Relocation::spec_for_immediate());
 }
 void MacroAssembler::cmpoop(Address src1, jobject obj) {
 cmp_literal32(src1, (int32_t)obj, oop_Relocation::spec_for_immediate());
 }
 void MacroAssembler::movoop(Address dst, jobject obj) {
 mov_literal32(dst, (int32_t)obj, oop_Relocation::spec_for_immediate());
 }
+void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
+mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
+}
+void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {
+mov_literal32(dst, (int32_t)obj, metadata_Relocation::spec_for_immediate());
+}
 void MacroAssembler::movptr(Register dst, AddressLiteral src) {
 if (src.is_lval()) {
 mov_literal32(dst, (intptr_t)src.target(), src.rspec());
 } else {
 movl(dst, as_Address(src));
 void MacroAssembler::pushoop(jobject obj) {
 push_literal32((int32_t)obj, oop_Relocation::spec_for_immediate());
 }
+void MacroAssembler::pushklass(Metadata* obj) {
+push_literal32((int32_t)obj, metadata_Relocation::spec_for_immediate());
+}
 void MacroAssembler::pushptr(AddressLiteral src) {
 if (src.is_lval()) {
 push_literal32((int32_t)src.target(), src.rspec());
 } else {
 // To see where a verify_oop failed, get $ebx+40/X for this frame.
 // This is the value of eip which points to where verify_oop will return.
 if (os::message_box(msg, "Execution stopped, print registers?")) {
 print_state32(rdi, rsi, rbp, rsp, rbx, rdx, rcx, rax, eip);
 BREAKPOINT;
-assert(false, "start up GDB");
 }
 } else {
 ttyLocker ttyl;
 ::tty->print_cr("=============== DEBUG MESSAGE: %s ================\n", msg);
+}
+// Don't assert holding the ttyLock
 assert(false, err_msg("DEBUG MESSAGE: %s", msg));
-}
 ThreadStateTransition::transition(thread, _thread_in_vm, saved_state);
 }
 void MacroAssembler::print_state32(int rdi, int rsi, int rbp, int rsp, int rbx, int rdx, int rcx, int rax, int eip) {
 ttyLocker ttyl;
 void MacroAssembler::movoop(Address dst, jobject obj) {
 mov_literal64(rscratch1, (intptr_t)obj, oop_Relocation::spec_for_immediate());
 movq(dst, rscratch1);
 }
+void MacroAssembler::mov_metadata(Register dst, Metadata* obj) {
+mov_literal64(dst, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
+}
+void MacroAssembler::mov_metadata(Address dst, Metadata* obj) {
+mov_literal64(rscratch1, (intptr_t)obj, metadata_Relocation::spec_for_immediate());
+movq(dst, rscratch1);
+}
 void MacroAssembler::movptr(Register dst, AddressLiteral src) {
 if (src.is_lval()) {
 mov_literal64(dst, (intptr_t)src.target(), src.rspec());
 } else {
 if (reachable(src)) {
 mov64(dst, (intptr_t)src);
 }
 void MacroAssembler::pushoop(jobject obj) {
 movoop(rscratch1, obj);
+push(rscratch1);
+}
+void MacroAssembler::pushklass(Metadata* obj) {
+mov_metadata(rscratch1, obj);
 push(rscratch1);
 }
 void MacroAssembler::pushptr(AddressLiteral src) {
 lea(rscratch1, src);
 lea(rscratch1, entry);
 Assembler::call(rscratch1);
 }
 }
+void MacroAssembler::ic_call(address entry) {
+RelocationHolder rh = virtual_call_Relocation::spec(pc());
+movptr(rax, (intptr_t)Universe::non_oop_word());
+call(AddressLiteral(entry, rh));
+}
 // Implementation of call_VM versions
 void MacroAssembler::call_VM(Register oop_result,
 address entry_point,
 bool check_exceptions) {
 assert(number_of_arguments >= 0   , "cannot have negative number of arguments");
 LP64_ONLY(assert(java_thread == r15_thread, "unexpected register"));
 #ifdef ASSERT
 // TraceBytecodes does not use r12 but saves it over the call, so don't verify
 // r12 is the heapbase.
-LP64_ONLY(if (UseCompressedOops && !TraceBytecodes) verify_heapbase("call_VM_base");)
+LP64_ONLY(if ((UseCompressedOops || UseCompressedKlassPointers) && !TraceBytecodes) verify_heapbase("call_VM_base: heap base corrupted?");)
 #endif // ASSERT
 assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");
 assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");
 #endif // LP64
 }
 // get oop result if there is one and reset the value in the thread
 if (oop_result->is_valid()) {
-movptr(oop_result, Address(java_thread, JavaThread::vm_result_offset()));
+get_vm_result(oop_result, java_thread);
-movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD);
-verify_oop(oop_result, "broken oop in call_VM_base");
 }
 }
 void MacroAssembler::call_VM_helper(Register oop_result, address entry_point, int number_of_arguments, bool check_exceptions) {
 pass_arg2(this, arg_2);
 LP64_ONLY(assert(arg_0 != c_rarg1, "smashed arg"));
 pass_arg1(this, arg_1);
 pass_arg0(this, arg_0);
 MacroAssembler::call_VM_leaf_base(entry_point, 4);
+}
+void MacroAssembler::get_vm_result(Register oop_result, Register java_thread) {
+movptr(oop_result, Address(java_thread, JavaThread::vm_result_offset()));
+movptr(Address(java_thread, JavaThread::vm_result_offset()), NULL_WORD);
+verify_oop(oop_result, "broken oop in call_VM_base");
+}
+void MacroAssembler::get_vm_result_2(Register metadata_result, Register java_thread) {
+movptr(metadata_result, Address(java_thread, JavaThread::vm_result_2_offset()));
+movptr(Address(java_thread, JavaThread::vm_result_2_offset()), NULL_WORD);
 }
 void MacroAssembler::check_and_handle_earlyret(Register java_thread) {
 }
 void MacroAssembler::movptr(Address dst, Register src) {
 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 }
+void MacroAssembler::movdqu(XMMRegister dst, AddressLiteral src) {
+if (reachable(src)) {
+Assembler::movdqu(dst, as_Address(src));
+} else {
+lea(rscratch1, src);
+Assembler::movdqu(dst, Address(rscratch1, 0));
+}
+}
 void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src) {
 if (reachable(src)) {
 Assembler::movsd(dst, as_Address(src));
 } else {
 lea(rscratch1, src);
 lea(rscratch1, src);
 Assembler::xorps(dst, Address(rscratch1, 0));
 }
 }
+void MacroAssembler::pshufb(XMMRegister dst, AddressLiteral src) {
+// Used in sign-bit flipping with aligned address.
+assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
+if (reachable(src)) {
+Assembler::pshufb(dst, as_Address(src));
+} else {
+lea(rscratch1, src);
+Assembler::pshufb(dst, Address(rscratch1, 0));
+}
+}
 // AVX 3-operands instructions
 void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
 if (reachable(src)) {
 vaddsd(dst, nds, as_Address(src));
 void MacroAssembler::fp_runtime_fallback(address runtime_entry, int nb_args, int num_fpu_regs_in_use) {
 pusha();
 // if we are coming from c1, xmm registers may be live
-if (UseSSE >= 1) {
-subptr(rsp, sizeof(jdouble)* LP64_ONLY(16) NOT_LP64(8));
-}
 int off = 0;
 if (UseSSE == 1)  {
+subptr(rsp, sizeof(jdouble)*8);
 movflt(Address(rsp,off++*sizeof(jdouble)),xmm0);
 movflt(Address(rsp,off++*sizeof(jdouble)),xmm1);
 movflt(Address(rsp,off++*sizeof(jdouble)),xmm2);
 movflt(Address(rsp,off++*sizeof(jdouble)),xmm3);
 movflt(Address(rsp,off++*sizeof(jdouble)),xmm4);
 movflt(Address(rsp,off++*sizeof(jdouble)),xmm5);
 movflt(Address(rsp,off++*sizeof(jdouble)),xmm6);
 movflt(Address(rsp,off++*sizeof(jdouble)),xmm7);
 } else if (UseSSE >= 2)  {
-movdbl(Address(rsp,off++*sizeof(jdouble)),xmm0);
+#ifdef COMPILER2
-movdbl(Address(rsp,off++*sizeof(jdouble)),xmm1);
+if (MaxVectorSize > 16) {
-movdbl(Address(rsp,off++*sizeof(jdouble)),xmm2);
+assert(UseAVX > 0, "256bit vectors are supported only with AVX");
-movdbl(Address(rsp,off++*sizeof(jdouble)),xmm3);
+// Save upper half of YMM registes
-movdbl(Address(rsp,off++*sizeof(jdouble)),xmm4);
+subptr(rsp, 16 * LP64_ONLY(16) NOT_LP64(8));
-movdbl(Address(rsp,off++*sizeof(jdouble)),xmm5);
+vextractf128h(Address(rsp,  0),xmm0);
-movdbl(Address(rsp,off++*sizeof(jdouble)),xmm6);
+vextractf128h(Address(rsp, 16),xmm1);
-movdbl(Address(rsp,off++*sizeof(jdouble)),xmm7);
+vextractf128h(Address(rsp, 32),xmm2);
+vextractf128h(Address(rsp, 48),xmm3);
+vextractf128h(Address(rsp, 64),xmm4);
+vextractf128h(Address(rsp, 80),xmm5);
+vextractf128h(Address(rsp, 96),xmm6);
+vextractf128h(Address(rsp,112),xmm7);
 #ifdef _LP64
-movdbl(Address(rsp,off++*sizeof(jdouble)),xmm8);
+vextractf128h(Address(rsp,128),xmm8);
-movdbl(Address(rsp,off++*sizeof(jdouble)),xmm9);
+vextractf128h(Address(rsp,144),xmm9);
-movdbl(Address(rsp,off++*sizeof(jdouble)),xmm10);
+vextractf128h(Address(rsp,160),xmm10);
-movdbl(Address(rsp,off++*sizeof(jdouble)),xmm11);
+vextractf128h(Address(rsp,176),xmm11);
-movdbl(Address(rsp,off++*sizeof(jdouble)),xmm12);
+vextractf128h(Address(rsp,192),xmm12);
-movdbl(Address(rsp,off++*sizeof(jdouble)),xmm13);
+vextractf128h(Address(rsp,208),xmm13);
-movdbl(Address(rsp,off++*sizeof(jdouble)),xmm14);
+vextractf128h(Address(rsp,224),xmm14);
-movdbl(Address(rsp,off++*sizeof(jdouble)),xmm15);
+vextractf128h(Address(rsp,240),xmm15);
+#endif
+}
+#endif
+// Save whole 128bit (16 bytes) XMM regiters
+subptr(rsp, 16 * LP64_ONLY(16) NOT_LP64(8));
+movdqu(Address(rsp,off++*16),xmm0);
+movdqu(Address(rsp,off++*16),xmm1);
+movdqu(Address(rsp,off++*16),xmm2);
+movdqu(Address(rsp,off++*16),xmm3);
+movdqu(Address(rsp,off++*16),xmm4);
+movdqu(Address(rsp,off++*16),xmm5);
+movdqu(Address(rsp,off++*16),xmm6);
+movdqu(Address(rsp,off++*16),xmm7);
+#ifdef _LP64
+movdqu(Address(rsp,off++*16),xmm8);
+movdqu(Address(rsp,off++*16),xmm9);
+movdqu(Address(rsp,off++*16),xmm10);
+movdqu(Address(rsp,off++*16),xmm11);
+movdqu(Address(rsp,off++*16),xmm12);
+movdqu(Address(rsp,off++*16),xmm13);
+movdqu(Address(rsp,off++*16),xmm14);
+movdqu(Address(rsp,off++*16),xmm15);
 #endif
 }
 // Preserve registers across runtime call
 int incoming_argument_and_return_value_offset = -1;
 movflt(xmm3, Address(rsp,off++*sizeof(jdouble)));
 movflt(xmm4, Address(rsp,off++*sizeof(jdouble)));
 movflt(xmm5, Address(rsp,off++*sizeof(jdouble)));
 movflt(xmm6, Address(rsp,off++*sizeof(jdouble)));
 movflt(xmm7, Address(rsp,off++*sizeof(jdouble)));
+addptr(rsp, sizeof(jdouble)*8);
 } else if (UseSSE >= 2)  {
-movdbl(xmm0, Address(rsp,off++*sizeof(jdouble)));
+// Restore whole 128bit (16 bytes) XMM regiters
-movdbl(xmm1, Address(rsp,off++*sizeof(jdouble)));
+movdqu(xmm0, Address(rsp,off++*16));
-movdbl(xmm2, Address(rsp,off++*sizeof(jdouble)));
+movdqu(xmm1, Address(rsp,off++*16));
-movdbl(xmm3, Address(rsp,off++*sizeof(jdouble)));
+movdqu(xmm2, Address(rsp,off++*16));
-movdbl(xmm4, Address(rsp,off++*sizeof(jdouble)));
+movdqu(xmm3, Address(rsp,off++*16));
-movdbl(xmm5, Address(rsp,off++*sizeof(jdouble)));
+movdqu(xmm4, Address(rsp,off++*16));
-movdbl(xmm6, Address(rsp,off++*sizeof(jdouble)));
+movdqu(xmm5, Address(rsp,off++*16));
-movdbl(xmm7, Address(rsp,off++*sizeof(jdouble)));
+movdqu(xmm6, Address(rsp,off++*16));
+movdqu(xmm7, Address(rsp,off++*16));
 #ifdef _LP64
-movdbl(xmm8, Address(rsp,off++*sizeof(jdouble)));
+movdqu(xmm8, Address(rsp,off++*16));
-movdbl(xmm9, Address(rsp,off++*sizeof(jdouble)));
+movdqu(xmm9, Address(rsp,off++*16));
-movdbl(xmm10, Address(rsp,off++*sizeof(jdouble)));
+movdqu(xmm10, Address(rsp,off++*16));
-movdbl(xmm11, Address(rsp,off++*sizeof(jdouble)));
+movdqu(xmm11, Address(rsp,off++*16));
-movdbl(xmm12, Address(rsp,off++*sizeof(jdouble)));
+movdqu(xmm12, Address(rsp,off++*16));
-movdbl(xmm13, Address(rsp,off++*sizeof(jdouble)));
+movdqu(xmm13, Address(rsp,off++*16));
-movdbl(xmm14, Address(rsp,off++*sizeof(jdouble)));
+movdqu(xmm14, Address(rsp,off++*16));
-movdbl(xmm15, Address(rsp,off++*sizeof(jdouble)));
+movdqu(xmm15, Address(rsp,off++*16));
 #endif
-}
+addptr(rsp, 16 * LP64_ONLY(16) NOT_LP64(8));
-if (UseSSE >= 1) {
+#ifdef COMPILER2
-addptr(rsp, sizeof(jdouble)* LP64_ONLY(16) NOT_LP64(8));
+if (MaxVectorSize > 16) {
+// Restore upper half of YMM registes.
+vinsertf128h(xmm0, Address(rsp,  0));
+vinsertf128h(xmm1, Address(rsp, 16));
+vinsertf128h(xmm2, Address(rsp, 32));
+vinsertf128h(xmm3, Address(rsp, 48));
+vinsertf128h(xmm4, Address(rsp, 64));
+vinsertf128h(xmm5, Address(rsp, 80));
+vinsertf128h(xmm6, Address(rsp, 96));
+vinsertf128h(xmm7, Address(rsp,112));
+#ifdef _LP64
+vinsertf128h(xmm8, Address(rsp,128));
+vinsertf128h(xmm9, Address(rsp,144));
+vinsertf128h(xmm10, Address(rsp,160));
+vinsertf128h(xmm11, Address(rsp,176));
+vinsertf128h(xmm12, Address(rsp,192));
+vinsertf128h(xmm13, Address(rsp,208));
+vinsertf128h(xmm14, Address(rsp,224));
+vinsertf128h(xmm15, Address(rsp,240));
+#endif
+addptr(rsp, 16 * LP64_ONLY(16) NOT_LP64(8));
+}
+#endif
 }
 popa();
 }
 static const double     pi_4 =  0.7853981633974483;
 assert_different_registers(recv_klass, intf_klass, method_result, scan_temp);
 assert(itable_index.is_constant() || itable_index.as_register() == method_result,
 "caller must use same register for non-constant itable index as for method");
 // Compute start of first itableOffsetEntry (which is at the end of the vtable)
-int vtable_base = instanceKlass::vtable_start_offset() * wordSize;
+int vtable_base = InstanceKlass::vtable_start_offset() * wordSize;
 int itentry_off = itableMethodEntry::method_offset_in_bytes();
 int scan_step   = itableOffsetEntry::size() * wordSize;
 int vte_size    = vtableEntry::size() * wordSize;
 Address::ScaleFactor times_vte_scale = Address::times_ptr;
 assert(vte_size == wordSize, "else adjust times_vte_scale");
-movl(scan_temp, Address(recv_klass, instanceKlass::vtable_length_offset() * wordSize));
+movl(scan_temp, Address(recv_klass, InstanceKlass::vtable_length_offset() * wordSize));
 // %%% Could store the aligned, prescaled offset in the klassoop.
 lea(scan_temp, Address(recv_klass, scan_temp, times_vte_scale, vtable_base));
 if (HeapWordsPerLong > 1) {
 // Round up to align_object_offset boundary
-// see code for instanceKlass::start_of_itable!
+// see code for InstanceKlass::start_of_itable!
 round_to(scan_temp, BytesPerLong);
 }
 // Adjust recv_klass by scaled itable_index, so we can free itable_index.
 assert(itableMethodEntry::size() * wordSize == wordSize, "adjust the scaling in the code below");
 // virtual method calling
 void MacroAssembler::lookup_virtual_method(Register recv_klass,
 RegisterOrConstant vtable_index,
 Register method_result) {
-const int base = instanceKlass::vtable_start_offset() * wordSize;
+const int base = InstanceKlass::vtable_start_offset() * wordSize;
 assert(vtableEntry::size() * wordSize == wordSize, "else adjust the scaling in the code below");
 Address vtable_entry_addr(recv_klass,
 vtable_index, Address::times_ptr,
 base + vtableEntry::method_offset_in_bytes());
 movptr(method_result, vtable_entry_addr);
 #endif //PRODUCT
 // We will consult the secondary-super array.
 movptr(rdi, secondary_supers_addr);
 // Load the array length.  (Positive movl does right thing on LP64.)
-movl(rcx, Address(rdi, arrayOopDesc::length_offset_in_bytes()));
+movl(rcx, Address(rdi, Array<Klass*>::length_offset_in_bytes()));
 // Skip to start of data.
-addptr(rdi, arrayOopDesc::base_offset_in_bytes(T_OBJECT));
+addptr(rdi, Array<Klass*>::base_offset_in_bytes());
 // Scan RCX words at [RDI] for an occurrence of RAX.
 // Set NZ/Z based on last compare.
 // Z flag value will not be set by 'repne' if RCX == 0 since 'repne' does
 // not change flags (only scas instruction which is repeated sets flags).
 // Set Z = 0 (not equal) before 'repne' to indicate that class was not found.
-#ifdef _LP64
-// This part is tricky, as values in supers array could be 32 or 64 bit wide
-// and we store values in objArrays always encoded, thus we need to encode
-// the value of rax before repne.  Note that rax is dead after the repne.
-if (UseCompressedOops) {
-encode_heap_oop_not_null(rax); // Changes flags.
-// The superclass is never null; it would be a basic system error if a null
-// pointer were to sneak in here.  Note that we have already loaded the
-// Klass::super_check_offset from the super_klass in the fast path,
-// so if there is a null in that register, we are already in the afterlife.
-testl(rax,rax); // Set Z = 0
-repne_scanl();
-} else
-#endif // _LP64
-{
 testptr(rax,rax); // Set Z = 0
 repne_scan();
-}
 // Unspill the temp. registers:
 if (pushed_rdi)  pop(rdi);
 if (pushed_rcx)  pop(rcx);
 if (pushed_rax)  pop(rax);
 pop_CPU_state();
 }
 void MacroAssembler::load_klass(Register dst, Register src) {
 #ifdef _LP64
-if (UseCompressedOops) {
+if (UseCompressedKlassPointers) {
 movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
-decode_heap_oop_not_null(dst);
+decode_klass_not_null(dst);
 } else
 #endif
 movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 }
 void MacroAssembler::load_prototype_header(Register dst, Register src) {
 #ifdef _LP64
-if (UseCompressedOops) {
+if (UseCompressedKlassPointers) {
 assert (Universe::heap() != NULL, "java heap should be initialized");
 movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
-if (Universe::narrow_oop_shift() != 0) {
+if (Universe::narrow_klass_shift() != 0) {
-assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
+assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
-if (LogMinObjAlignmentInBytes == Address::times_8) {
+assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?");
 movq(dst, Address(r12_heapbase, dst, Address::times_8, Klass::prototype_header_offset()));
-} else {
-// OK to use shift since we don't need to preserve flags.
-shlq(dst, LogMinObjAlignmentInBytes);
-movq(dst, Address(r12_heapbase, dst, Address::times_1, Klass::prototype_header_offset()));
-}
 } else {
 movq(dst, Address(dst, Klass::prototype_header_offset()));
 }
 } else
 #endif
 }
 }
 void MacroAssembler::store_klass(Register dst, Register src) {
 #ifdef _LP64
-if (UseCompressedOops) {
+if (UseCompressedKlassPointers) {
-encode_heap_oop_not_null(src);
+encode_klass_not_null(src);
 movl(Address(dst, oopDesc::klass_offset_in_bytes()), src);
 } else
 #endif
 movptr(Address(dst, oopDesc::klass_offset_in_bytes()), src);
 }
 void MacroAssembler::load_heap_oop(Register dst, Address src) {
 #ifdef _LP64
+// FIXME: Must change all places where we try to load the klass.
 if (UseCompressedOops) {
 movl(dst, src);
 decode_heap_oop(dst);
 } else
 #endif
 #endif
 }
 #ifdef _LP64
 void MacroAssembler::store_klass_gap(Register dst, Register src) {
-if (UseCompressedOops) {
+if (UseCompressedKlassPointers) {
 // Store to klass gap in destination
 movl(Address(dst, oopDesc::klass_gap_offset_in_bytes()), src);
 }
 }
 #ifdef ASSERT
 void MacroAssembler::verify_heapbase(const char* msg) {
-assert (UseCompressedOops, "should be compressed");
+assert (UseCompressedOops || UseCompressedKlassPointers, "should be compressed");
 assert (Universe::heap() != NULL, "java heap should be initialized");
 if (CheckCompressedOops) {
 Label ok;
 push(rscratch1); // cmpptr trashes rscratch1
-cmpptr(r12_heapbase, ExternalAddress((address)Universe::narrow_oop_base_addr()));
+cmpptr(r12_heapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr()));
 jcc(Assembler::equal, ok);
 STOP(msg);
 bind(ok);
 pop(rscratch1);
 }
 movq(dst, src);
 }
 }
 }
+void MacroAssembler::encode_klass_not_null(Register r) {
+assert(Metaspace::is_initialized(), "metaspace should be initialized");
+#ifdef ASSERT
+verify_heapbase("MacroAssembler::encode_klass_not_null: heap base corrupted?");
+#endif
+if (Universe::narrow_klass_base() != NULL) {
+subq(r, r12_heapbase);
+}
+if (Universe::narrow_klass_shift() != 0) {
+assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
+shrq(r, LogKlassAlignmentInBytes);
+}
+}
+void MacroAssembler::encode_klass_not_null(Register dst, Register src) {
+assert(Metaspace::is_initialized(), "metaspace should be initialized");
+#ifdef ASSERT
+verify_heapbase("MacroAssembler::encode_klass_not_null2: heap base corrupted?");
+#endif
+if (dst != src) {
+movq(dst, src);
+}
+if (Universe::narrow_klass_base() != NULL) {
+subq(dst, r12_heapbase);
+}
+if (Universe::narrow_klass_shift() != 0) {
+assert (LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
+shrq(dst, LogKlassAlignmentInBytes);
+}
+}
+void  MacroAssembler::decode_klass_not_null(Register r) {
+assert(Metaspace::is_initialized(), "metaspace should be initialized");
+// Note: it will change flags
+assert (UseCompressedKlassPointers, "should only be used for compressed headers");
+// Cannot assert, unverified entry point counts instructions (see .ad file)
+// vtableStubs also counts instructions in pd_code_size_limit.
+// Also do not verify_oop as this is called by verify_oop.
+if (Universe::narrow_klass_shift() != 0) {
+assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
+shlq(r, LogKlassAlignmentInBytes);
+if (Universe::narrow_klass_base() != NULL) {
+addq(r, r12_heapbase);
+}
+} else {
+assert (Universe::narrow_klass_base() == NULL, "sanity");
+}
+}
+void  MacroAssembler::decode_klass_not_null(Register dst, Register src) {
+assert(Metaspace::is_initialized(), "metaspace should be initialized");
+// Note: it will change flags
+assert (UseCompressedKlassPointers, "should only be used for compressed headers");
+// Cannot assert, unverified entry point counts instructions (see .ad file)
+// vtableStubs also counts instructions in pd_code_size_limit.
+// Also do not verify_oop as this is called by verify_oop.
+if (Universe::narrow_klass_shift() != 0) {
+assert(LogKlassAlignmentInBytes == Universe::narrow_klass_shift(), "decode alg wrong");
+assert(LogKlassAlignmentInBytes == Address::times_8, "klass not aligned on 64bits?");
+leaq(dst, Address(r12_heapbase, src, Address::times_8, 0));
+} else {
+assert (Universe::narrow_klass_base() == NULL, "sanity");
+if (dst != src) {
+movq(dst, src);
+}
+}
+}
 void  MacroAssembler::set_narrow_oop(Register dst, jobject obj) {
 assert (UseCompressedOops, "should only be used for compressed headers");
 assert (Universe::heap() != NULL, "java heap should be initialized");
 assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
 int oop_index = oop_recorder()->find_index(obj);
 int oop_index = oop_recorder()->find_index(obj);
 RelocationHolder rspec = oop_Relocation::spec(oop_index);
 mov_narrow_oop(dst, oop_index, rspec);
 }
+void  MacroAssembler::set_narrow_klass(Register dst, Klass* k) {
+assert (UseCompressedKlassPointers, "should only be used for compressed headers");
+assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
+int klass_index = oop_recorder()->find_index(k);
+RelocationHolder rspec = metadata_Relocation::spec(klass_index);
+mov_narrow_oop(dst, oopDesc::encode_klass(k), rspec);
+}
+void  MacroAssembler::set_narrow_klass(Address dst, Klass* k) {
+assert (UseCompressedKlassPointers, "should only be used for compressed headers");
+assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
+int klass_index = oop_recorder()->find_index(k);
+RelocationHolder rspec = metadata_Relocation::spec(klass_index);
+mov_narrow_oop(dst, oopDesc::encode_klass(k), rspec);
+}
 void  MacroAssembler::cmp_narrow_oop(Register dst, jobject obj) {
 assert (UseCompressedOops, "should only be used for compressed headers");
 assert (Universe::heap() != NULL, "java heap should be initialized");
 assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
 int oop_index = oop_recorder()->find_index(obj);
 int oop_index = oop_recorder()->find_index(obj);
 RelocationHolder rspec = oop_Relocation::spec(oop_index);
 Assembler::cmp_narrow_oop(dst, oop_index, rspec);
 }
+void  MacroAssembler::cmp_narrow_klass(Register dst, Klass* k) {
+assert (UseCompressedKlassPointers, "should only be used for compressed headers");
+assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
+int klass_index = oop_recorder()->find_index(k);
+RelocationHolder rspec = metadata_Relocation::spec(klass_index);
+Assembler::cmp_narrow_oop(dst, oopDesc::encode_klass(k), rspec);
+}
+void  MacroAssembler::cmp_narrow_klass(Address dst, Klass* k) {
+assert (UseCompressedKlassPointers, "should only be used for compressed headers");
+assert (oop_recorder() != NULL, "this assembler needs an OopRecorder");
+int klass_index = oop_recorder()->find_index(k);
+RelocationHolder rspec = metadata_Relocation::spec(klass_index);
+Assembler::cmp_narrow_oop(dst, oopDesc::encode_klass(k), rspec);
+}
 void MacroAssembler::reinit_heapbase() {
-if (UseCompressedOops) {
+if (UseCompressedOops || UseCompressedKlassPointers) {
-movptr(r12_heapbase, ExternalAddress((address)Universe::narrow_oop_base_addr()));
+movptr(r12_heapbase, ExternalAddress((address)Universe::narrow_ptrs_base_addr()));
 }
 }
 #endif // _LP64
 movq(vec, Address(str2, 0));  // move 64 bits
 } else { // cnt2 = { 3, 5, 6, 7 }
 // Array header size is 12 bytes in 32-bit VM
 // + 6 bytes for 3 chars == 18 bytes,
 // enough space to load vec and shift.
-assert(HeapWordSize*typeArrayKlass::header_size() >= 12,"sanity");
+assert(HeapWordSize*TypeArrayKlass::header_size() >= 12,"sanity");
 movdqu(vec, Address(str2, (int_cnt2*2)-16));
 psrldq(vec, 16-(int_cnt2*2));
 }
 } else { // not constant substring
 cmpl(cnt2, 8);

Mercurial > hg > truffle

comparison src/cpu/x86/vm/assembler_x86.cpp @ 6948:e522a00b91aa