graal-jvmci-8: src/cpu/x86/vm/assembler

comparison src/cpu/x86/vm/assembler_x86.cpp @ 4970:33df1aeaebbf

Merge with http://hg.openjdk.java.net/hsx/hsx24/hotspot/

author	Thomas Wuerthinger <thomas.wuerthinger@oracle.com>
date	Mon, 27 Feb 2012 13:10:13 +0100
parents	04b9a2566eec fd8114661503
children	957c266d8bc5

comparison

equal deleted inserted replaced

-:2cfb7fb2dce7
+:33df1aeaebbf
 /*
-* Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved.
+* Copyright (c) 1997, 2012, Oracle and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 only, as
 * published by the Free Software Foundation.
 } else {
 emit_byte(op1);
 emit_byte(op2 | encode(dst));
 emit_long(imm32);
 }
+}
+// Force generation of a 4 byte immediate value even if it fits into 8bit
+void Assembler::emit_arith_imm32(int op1, int op2, Register dst, int32_t imm32) {
+assert(isByte(op1) && isByte(op2), "wrong opcode");
+assert((op1 & 0x01) == 1, "should be 32bit operation");
+assert((op1 & 0x02) == 0, "sign-extension bit should not be set");
+emit_byte(op1);
+emit_byte(op2 | encode(dst));
+emit_long(imm32);
 }
 // immediate-to-memory forms
 void Assembler::emit_arith_operand(int op1, Register rm, Address adr, int32_t imm32) {
 assert((op1 & 0x01) == 1, "should be 32bit operation");
 debug_only(has_disp32 = true); // has both kinds of operands!
 break;
 case 0x0F: // movx..., etc.
 switch (0xFF & *ip++) {
+case 0x3A: // pcmpestri
+tail_size = 1;
+case 0x38: // ptest, pmovzxbw
+ip++; // skip opcode
+debug_only(has_disp32 = true); // has both kinds of operands!
+break;
+case 0x70: // pshufd r, r/a, #8
+debug_only(has_disp32 = true); // has both kinds of operands!
+case 0x73: // psrldq r, #8
+tail_size = 1;
+break;
 case 0x12: // movlps
 case 0x28: // movaps
 case 0x2E: // ucomiss
 case 0x2F: // comiss
 case 0x54: // andps
 case 0x55: // andnps
 case 0x56: // orps
 case 0x57: // xorps
 case 0x6E: // movd
 case 0x7E: // movd
-case 0xAE: // ldmxcsr   a
+case 0xAE: // ldmxcsr, stmxcsr, fxrstor, fxsave, clflush
-// 64bit side says it these have both operands but that doesn't
-// appear to be true
 debug_only(has_disp32 = true);
 break;
 case 0xAD: // shrd r, a, %cl
 case 0xAF: // imul r, a
 case REP16(0x90): // setcc a
 debug_only(has_disp32 = true);
 // fall out of the switch to decode the address
 break;
+case 0xC4: // pinsrw r, a, #8
+debug_only(has_disp32 = true);
+case 0xC5: // pextrw r, r, #8
+tail_size = 1;  // the imm8
+break;
 case 0xAC: // shrd r, a, #8
 debug_only(has_disp32 = true);
 tail_size = 1;  // the imm8
 break;
 case 0x6B: // imul r, a, #8
 debug_only(has_disp32 = true); // has both kinds of operands!
 tail_size = 1; // the imm8
 break;
-case 0xE8: // call rdisp32
+case 0xC4: // VEX_3bytes
-case 0xE9: // jmp  rdisp32
+case 0xC5: // VEX_2bytes
-if (which == end_pc_operand)  return ip + 4;
+assert((UseAVX > 0), "shouldn't have VEX prefix");
-assert(which == call32_operand, "call has no disp32 or imm");
+assert(ip == inst+1, "no prefixes allowed");
-return ip;
+// C4 and C5 are also used as opcodes for PINSRW and PEXTRW instructions
+// but they have prefix 0x0F and processed when 0x0F processed above.
+//
+// In 32-bit mode the VEX first byte C4 and C5 alias onto LDS and LES
+// instructions (these instructions are not supported in 64-bit mode).
+// To distinguish them bits [7:6] are set in the VEX second byte since
+// ModRM byte can not be of the form 11xxxxxx in 32-bit mode. To set
+// those VEX bits REX and vvvv bits are inverted.
+//
+// Fortunately C2 doesn't generate these instructions so we don't need
+// to check for them in product version.
+// Check second byte
+NOT_LP64(assert((0xC0 & *ip) == 0xC0, "shouldn't have LDS and LES instructions"));
+// First byte
+if ((0xFF & *inst) == VEX_3bytes) {
+ip++; // third byte
+is_64bit = ((VEX_W & *ip) == VEX_W);
+}
+ip++; // opcode
+// To find the end of instruction (which == end_pc_operand).
+switch (0xFF & *ip) {
+case 0x61: // pcmpestri r, r/a, #8
+case 0x70: // pshufd r, r/a, #8
+case 0x73: // psrldq r, #8
+tail_size = 1;  // the imm8
+break;
+default:
+break;
+}
+ip++; // skip opcode
+debug_only(has_disp32 = true); // has both kinds of operands!
+break;
 case 0xD1: // sal a, 1; sar a, 1; shl a, 1; shr a, 1
 case 0xD3: // sal a, %cl; sar a, %cl; shl a, %cl; shr a, %cl
 case 0xD9: // fld_s a; fst_s a; fstp_s a; fldcw a
 case 0xDD: // fld_d a; fst_d a; fstp_d a
 case 0xD8: // fadd_s a; fsubr_s a; fmul_s a; fdivr_s a; fcomp_s a
 case 0xDC: // fadd_d a; fsubr_d a; fmul_d a; fdivr_d a; fcomp_d a
 case 0xDE: // faddp_d a; fsubrp_d a; fmulp_d a; fdivrp_d a; fcompp_d a
 debug_only(has_disp32 = true);
 break;
+case 0xE8: // call rdisp32
+case 0xE9: // jmp  rdisp32
+if (which == end_pc_operand)  return ip + 4;
+assert(which == call32_operand, "call has no disp32 or imm");
+return ip;
 case 0xF0:                    // Lock
 assert(os::is_MP(), "only on MP");
 goto again_after_prefix;
 (void) prefix_and_encode(dst->encoding(), src->encoding());
 emit_arith(0x03, 0xC0, dst, src);
 }
 void Assembler::addr_nop_4() {
+assert(UseAddressNop, "no CPU support");
 // 4 bytes: NOP DWORD PTR [EAX+0]
 emit_byte(0x0F);
 emit_byte(0x1F);
 emit_byte(0x40); // emit_rm(cbuf, 0x1, EAX_enc, EAX_enc);
 emit_byte(0);    // 8-bits offset (1 byte)
 }
 void Assembler::addr_nop_5() {
+assert(UseAddressNop, "no CPU support");
 // 5 bytes: NOP DWORD PTR [EAX+EAX*0+0] 8-bits offset
 emit_byte(0x0F);
 emit_byte(0x1F);
 emit_byte(0x44); // emit_rm(cbuf, 0x1, EAX_enc, 0x4);
 emit_byte(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
 emit_byte(0);    // 8-bits offset (1 byte)
 }
 void Assembler::addr_nop_7() {
+assert(UseAddressNop, "no CPU support");
 // 7 bytes: NOP DWORD PTR [EAX+0] 32-bits offset
 emit_byte(0x0F);
 emit_byte(0x1F);
 emit_byte(0x80); // emit_rm(cbuf, 0x2, EAX_enc, EAX_enc);
 emit_long(0);    // 32-bits offset (4 bytes)
 }
 void Assembler::addr_nop_8() {
+assert(UseAddressNop, "no CPU support");
 // 8 bytes: NOP DWORD PTR [EAX+EAX*0+0] 32-bits offset
 emit_byte(0x0F);
 emit_byte(0x1F);
 emit_byte(0x84); // emit_rm(cbuf, 0x2, EAX_enc, 0x4);
 emit_byte(0x00); // emit_rm(cbuf, 0x0, EAX_enc, EAX_enc);
 emit_long(0);    // 32-bits offset (4 bytes)
 }
 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-emit_byte(0xF2);
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2);
-int encode = prefix_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0x58);
 emit_byte(0xC0 | encode);
 }
 void Assembler::addsd(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 InstructionMark im(this);
-emit_byte(0xF2);
+simd_prefix(dst, dst, src, VEX_SIMD_F2);
-prefix(src, dst);
-emit_byte(0x0F);
 emit_byte(0x58);
 emit_operand(dst, src);
 }
 void Assembler::addss(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-emit_byte(0xF3);
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3);
-int encode = prefix_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0x58);
 emit_byte(0xC0 | encode);
 }
 void Assembler::addss(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
 InstructionMark im(this);
-emit_byte(0xF3);
+simd_prefix(dst, dst, src, VEX_SIMD_F3);
-prefix(src, dst);
-emit_byte(0x0F);
 emit_byte(0x58);
 emit_operand(dst, src);
+}
+void Assembler::andl(Address dst, int32_t imm32) {
+InstructionMark im(this);
+prefix(dst);
+emit_byte(0x81);
+emit_operand(rsp, dst, 4);
+emit_long(imm32);
 }
 void Assembler::andl(Register dst, int32_t imm32) {
 prefix(dst);
 emit_arith(0x81, 0xE0, dst, imm32);
 }
 void Assembler::andpd(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 InstructionMark im(this);
-emit_byte(0x66);
+simd_prefix(dst, dst, src, VEX_SIMD_66);
-prefix(src, dst);
-emit_byte(0x0F);
 emit_byte(0x54);
 emit_operand(dst, src);
+}
+void Assembler::andpd(XMMRegister dst, XMMRegister src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66);
+emit_byte(0x54);
+emit_byte(0xC0 | encode);
+}
+void Assembler::andps(XMMRegister dst, Address src) {
+NOT_LP64(assert(VM_Version::supports_sse(), ""));
+InstructionMark im(this);
+simd_prefix(dst, dst, src, VEX_SIMD_NONE);
+emit_byte(0x54);
+emit_operand(dst, src);
+}
+void Assembler::andps(XMMRegister dst, XMMRegister src) {
+NOT_LP64(assert(VM_Version::supports_sse(), ""));
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE);
+emit_byte(0x54);
+emit_byte(0xC0 | encode);
 }
 void Assembler::bsfl(Register dst, Register src) {
 int encode = prefix_and_encode(dst->encoding(), src->encoding());
 emit_byte(0x0F);
 emit_data(int(0), rtype, operand);
 }
 }
 void Assembler::call(Register dst) {
-// This was originally using a 32bit register encoding
+int encode = prefix_and_encode(dst->encoding());
-// and surely we want 64bit!
-// this is a 32bit encoding but in 64bit mode the default
-// operand size is 64bit so there is no need for the
-// wide prefix. So prefix only happens if we use the
-// new registers. Much like push/pop.
-int x = offset();
-// this may be true but dbx disassembles it as if it
-// were 32bits...
-// int encode = prefix_and_encode(dst->encoding());
-// if (offset() != x) assert(dst->encoding() >= 8, "what?");
-int encode = prefixq_and_encode(dst->encoding());
 emit_byte(0xFF);
 emit_byte(0xD0 | encode);
 }
 void Assembler::comisd(XMMRegister dst, Address src) {
 // NOTE: dbx seems to decode this as comiss even though the
 // 0x66 is there. Strangly ucomisd comes out correct
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-emit_byte(0x66);
+InstructionMark im(this);
-comiss(dst, src);
+simd_prefix(dst, src, VEX_SIMD_66);
+emit_byte(0x2F);
+emit_operand(dst, src);
+}
+void Assembler::comisd(XMMRegister dst, XMMRegister src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66);
+emit_byte(0x2F);
+emit_byte(0xC0 | encode);
 }
 void Assembler::comiss(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
+InstructionMark im(this);
-InstructionMark im(this);
+simd_prefix(dst, src, VEX_SIMD_NONE);
-prefix(src, dst);
-emit_byte(0x0F);
 emit_byte(0x2F);
 emit_operand(dst, src);
 }
+void Assembler::comiss(XMMRegister dst, XMMRegister src) {
+NOT_LP64(assert(VM_Version::supports_sse(), ""));
+int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_NONE);
+emit_byte(0x2F);
+emit_byte(0xC0 | encode);
+}
 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-emit_byte(0xF3);
+int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_F3);
-int encode = prefix_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0xE6);
 emit_byte(0xC0 | encode);
 }
 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-int encode = prefix_and_encode(dst->encoding(), src->encoding());
+int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_NONE);
-emit_byte(0x0F);
 emit_byte(0x5B);
 emit_byte(0xC0 | encode);
 }
 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-emit_byte(0xF2);
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2);
-int encode = prefix_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0x5A);
 emit_byte(0xC0 | encode);
 }
+void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+InstructionMark im(this);
+simd_prefix(dst, dst, src, VEX_SIMD_F2);
+emit_byte(0x5A);
+emit_operand(dst, src);
+}
 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-emit_byte(0xF2);
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2);
-int encode = prefix_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0x2A);
 emit_byte(0xC0 | encode);
 }
+void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+InstructionMark im(this);
+simd_prefix(dst, dst, src, VEX_SIMD_F2);
+emit_byte(0x2A);
+emit_operand(dst, src);
+}
 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-emit_byte(0xF3);
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3);
-int encode = prefix_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0x2A);
 emit_byte(0xC0 | encode);
 }
+void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
+NOT_LP64(assert(VM_Version::supports_sse(), ""));
+InstructionMark im(this);
+simd_prefix(dst, dst, src, VEX_SIMD_F3);
+emit_byte(0x2A);
+emit_operand(dst, src);
+}
 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-emit_byte(0xF3);
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3);
-int encode = prefix_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0x5A);
 emit_byte(0xC0 | encode);
 }
+void Assembler::cvtss2sd(XMMRegister dst, Address src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+InstructionMark im(this);
+simd_prefix(dst, dst, src, VEX_SIMD_F3);
+emit_byte(0x5A);
+emit_operand(dst, src);
+}
 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-emit_byte(0xF2);
+int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_F2);
-int encode = prefix_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0x2C);
 emit_byte(0xC0 | encode);
 }
 void Assembler::cvttss2sil(Register dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-emit_byte(0xF3);
+int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_F3);
-int encode = prefix_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0x2C);
 emit_byte(0xC0 | encode);
 }
 void Assembler::decl(Address dst) {
 }
 void Assembler::divsd(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 InstructionMark im(this);
-emit_byte(0xF2);
+simd_prefix(dst, dst, src, VEX_SIMD_F2);
-prefix(src, dst);
-emit_byte(0x0F);
 emit_byte(0x5E);
 emit_operand(dst, src);
 }
 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-emit_byte(0xF2);
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2);
-int encode = prefix_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0x5E);
 emit_byte(0xC0 | encode);
 }
 void Assembler::divss(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
 InstructionMark im(this);
-emit_byte(0xF3);
+simd_prefix(dst, dst, src, VEX_SIMD_F3);
-prefix(src, dst);
-emit_byte(0x0F);
 emit_byte(0x5E);
 emit_operand(dst, src);
 }
 void Assembler::divss(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-emit_byte(0xF3);
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3);
-int encode = prefix_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0x5E);
 emit_byte(0xC0 | encode);
 }
 void Assembler::emms() {
 void Assembler::jccb(Condition cc, Label& L) {
 if (L.is_bound()) {
 const int short_size = 2;
 address entry = target(L);
-assert(is8bit((intptr_t)entry - ((intptr_t)_code_pos + short_size)),
+#ifdef ASSERT
-"Dispacement too large for a short jmp");
+intptr_t dist = (intptr_t)entry - ((intptr_t)_code_pos + short_size);
+intptr_t delta = short_branch_delta();
+if (delta != 0) {
+dist += (dist < 0 ? (-delta) :delta);
+}
+assert(is8bit(dist), "Dispacement too large for a short jmp");
+#endif
 intptr_t offs = (intptr_t)entry - (intptr_t)_code_pos;
 // 0111 tttn #8-bit disp
 emit_byte(0x70 | cc);
 emit_byte((offs - short_size) & 0xFF);
 } else {
 void Assembler::jmpb(Label& L) {
 if (L.is_bound()) {
 const int short_size = 2;
 address entry = target(L);
-assert(is8bit((entry - _code_pos) + short_size),
-"Dispacement too large for a short jmp");
 assert(entry != NULL, "jmp most probably wrong");
+#ifdef ASSERT
+intptr_t dist = (intptr_t)entry - ((intptr_t)_code_pos + short_size);
+intptr_t delta = short_branch_delta();
+if (delta != 0) {
+dist += (dist < 0 ? (-delta) :delta);
+}
+assert(is8bit(dist), "Dispacement too large for a short jmp");
+#endif
 intptr_t offs = entry - _code_pos;
 emit_byte(0xEB);
 emit_byte((offs - short_size) & 0xFF);
 } else {
 InstructionMark im(this);
 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 }
 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-int dstenc = dst->encoding();
+int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66);
-int srcenc = src->encoding();
-emit_byte(0x66);
-if (dstenc < 8) {
-if (srcenc >= 8) {
-prefix(REX_B);
-srcenc -= 8;
-}
-} else {
-if (srcenc < 8) {
-prefix(REX_R);
-} else {
-prefix(REX_RB);
-srcenc -= 8;
-}
-dstenc -= 8;
-}
-emit_byte(0x0F);
 emit_byte(0x28);
-emit_byte(0xC0 | dstenc << 3 | srcenc);
+emit_byte(0xC0 | encode);
 }
 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-int dstenc = dst->encoding();
+int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_NONE);
-int srcenc = src->encoding();
-if (dstenc < 8) {
-if (srcenc >= 8) {
-prefix(REX_B);
-srcenc -= 8;
-}
-} else {
-if (srcenc < 8) {
-prefix(REX_R);
-} else {
-prefix(REX_RB);
-srcenc -= 8;
-}
-dstenc -= 8;
-}
-emit_byte(0x0F);
 emit_byte(0x28);
-emit_byte(0xC0 | dstenc << 3 | srcenc);
+emit_byte(0xC0 | encode);
 }
 void Assembler::movb(Register dst, Address src) {
 NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
 InstructionMark im(this);
 emit_operand(src, dst);
 }
 void Assembler::movdl(XMMRegister dst, Register src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-emit_byte(0x66);
+int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66);
-int encode = prefix_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0x6E);
 emit_byte(0xC0 | encode);
 }
 void Assembler::movdl(Register dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-emit_byte(0x66);
 // swap src/dst to get correct prefix
-int encode = prefix_and_encode(src->encoding(), dst->encoding());
+int encode = simd_prefix_and_encode(src, dst, VEX_SIMD_66);
-emit_byte(0x0F);
 emit_byte(0x7E);
 emit_byte(0xC0 | encode);
 }
 void Assembler::movdl(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 InstructionMark im(this);
-emit_byte(0x66);
+simd_prefix(dst, src, VEX_SIMD_66);
-prefix(src, dst);
-emit_byte(0x0F);
 emit_byte(0x6E);
 emit_operand(dst, src);
 }
+void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
-void Assembler::movdqa(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-InstructionMark im(this);
+int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66);
-emit_byte(0x66);
+emit_byte(0x6F);
-prefix(src, dst);
+emit_byte(0xC0 | encode);
-emit_byte(0x0F);
+}
+void Assembler::movdqu(XMMRegister dst, Address src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+InstructionMark im(this);
+simd_prefix(dst, src, VEX_SIMD_F3);
 emit_byte(0x6F);
 emit_operand(dst, src);
 }
-void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
+void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-emit_byte(0x66);
+int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_F3);
-int encode = prefixq_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0x6F);
 emit_byte(0xC0 | encode);
 }
-void Assembler::movdqa(Address dst, XMMRegister src) {
-NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-InstructionMark im(this);
-emit_byte(0x66);
-prefix(dst, src);
-emit_byte(0x0F);
-emit_byte(0x7F);
-emit_operand(src, dst);
-}
-void Assembler::movdqu(XMMRegister dst, Address src) {
-NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-InstructionMark im(this);
-emit_byte(0xF3);
-prefix(src, dst);
-emit_byte(0x0F);
-emit_byte(0x6F);
-emit_operand(dst, src);
-}
-void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
-NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-emit_byte(0xF3);
-int encode = prefixq_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
-emit_byte(0x6F);
-emit_byte(0xC0 | encode);
-}
 void Assembler::movdqu(Address dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 InstructionMark im(this);
-emit_byte(0xF3);
+simd_prefix(dst, src, VEX_SIMD_F3);
-prefix(dst, src);
-emit_byte(0x0F);
 emit_byte(0x7F);
 emit_operand(src, dst);
 }
 // Uses zero extension on 64bit
 // when loading from memory. But for old Opteron use movlpd instead of movsd.
 // The selection is done in MacroAssembler::movdbl() and movflt().
 void Assembler::movlpd(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 InstructionMark im(this);
-emit_byte(0x66);
+simd_prefix(dst, dst, src, VEX_SIMD_66);
-prefix(src, dst);
-emit_byte(0x0F);
 emit_byte(0x12);
 emit_operand(dst, src);
 }
 void Assembler::movq( MMXRegister dst, Address src ) {
 }
 void Assembler::movq(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 InstructionMark im(this);
-emit_byte(0xF3);
+simd_prefix(dst, src, VEX_SIMD_F3);
-prefix(src, dst);
-emit_byte(0x0F);
 emit_byte(0x7E);
 emit_operand(dst, src);
 }
 void Assembler::movq(Address dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 InstructionMark im(this);
-emit_byte(0x66);
+simd_prefix(dst, src, VEX_SIMD_66);
-prefix(dst, src);
-emit_byte(0x0F);
 emit_byte(0xD6);
 emit_operand(src, dst);
 }
 void Assembler::movsbl(Register dst, Address src) { // movsxb
 emit_byte(0xC0 | encode);
 }
 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-emit_byte(0xF2);
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2);
-int encode = prefix_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0x10);
 emit_byte(0xC0 | encode);
 }
 void Assembler::movsd(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 InstructionMark im(this);
-emit_byte(0xF2);
+simd_prefix(dst, src, VEX_SIMD_F2);
-prefix(src, dst);
-emit_byte(0x0F);
 emit_byte(0x10);
 emit_operand(dst, src);
 }
 void Assembler::movsd(Address dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 InstructionMark im(this);
-emit_byte(0xF2);
+simd_prefix(dst, src, VEX_SIMD_F2);
-prefix(dst, src);
-emit_byte(0x0F);
 emit_byte(0x11);
 emit_operand(src, dst);
 }
 void Assembler::movss(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-emit_byte(0xF3);
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3);
-int encode = prefix_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0x10);
 emit_byte(0xC0 | encode);
 }
 void Assembler::movss(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
 InstructionMark im(this);
-emit_byte(0xF3);
+simd_prefix(dst, src, VEX_SIMD_F3);
-prefix(src, dst);
-emit_byte(0x0F);
 emit_byte(0x10);
 emit_operand(dst, src);
 }
 void Assembler::movss(Address dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
 InstructionMark im(this);
-emit_byte(0xF3);
+simd_prefix(dst, src, VEX_SIMD_F3);
-prefix(dst, src);
-emit_byte(0x0F);
 emit_byte(0x11);
 emit_operand(src, dst);
 }
 void Assembler::movswl(Register dst, Address src) { // movsxw
 }
 void Assembler::mulsd(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 InstructionMark im(this);
-emit_byte(0xF2);
+simd_prefix(dst, dst, src, VEX_SIMD_F2);
-prefix(src, dst);
-emit_byte(0x0F);
 emit_byte(0x59);
 emit_operand(dst, src);
 }
 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-emit_byte(0xF2);
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2);
-int encode = prefix_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0x59);
 emit_byte(0xC0 | encode);
 }
 void Assembler::mulss(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
 InstructionMark im(this);
-emit_byte(0xF3);
+simd_prefix(dst, dst, src, VEX_SIMD_F3);
-prefix(src, dst);
-emit_byte(0x0F);
 emit_byte(0x59);
 emit_operand(dst, src);
 }
 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-emit_byte(0xF3);
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3);
-int encode = prefix_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0x59);
 emit_byte(0xC0 | encode);
 }
 void Assembler::negl(Register dst) {
 void Assembler::orl(Register dst, Register src) {
 (void) prefix_and_encode(dst->encoding(), src->encoding());
 emit_arith(0x0B, 0xC0, dst, src);
 }
+void Assembler::packuswb(XMMRegister dst, Address src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
+InstructionMark im(this);
+simd_prefix(dst, dst, src, VEX_SIMD_66);
+emit_byte(0x67);
+emit_operand(dst, src);
+}
+void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66);
+emit_byte(0x67);
+emit_byte(0xC0 | encode);
+}
 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
 assert(VM_Version::supports_sse4_2(), "");
+InstructionMark im(this);
-InstructionMark im(this);
+simd_prefix(dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A);
-emit_byte(0x66);
-prefix(src, dst);
-emit_byte(0x0F);
-emit_byte(0x3A);
 emit_byte(0x61);
 emit_operand(dst, src);
 emit_byte(imm8);
 }
 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
 assert(VM_Version::supports_sse4_2(), "");
+int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A);
-emit_byte(0x66);
-int encode = prefixq_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
-emit_byte(0x3A);
 emit_byte(0x61);
 emit_byte(0xC0 | encode);
 emit_byte(imm8);
+}
+void Assembler::pmovzxbw(XMMRegister dst, Address src) {
+assert(VM_Version::supports_sse4_1(), "");
+InstructionMark im(this);
+simd_prefix(dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
+emit_byte(0x30);
+emit_operand(dst, src);
+}
+void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
+assert(VM_Version::supports_sse4_1(), "");
+int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
+emit_byte(0x30);
+emit_byte(0xC0 | encode);
 }
 // generic
 void Assembler::pop(Register dst) {
 int encode = prefix_and_encode(dst->encoding());
 a_byte(p);
 }
 void Assembler::por(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66);
-emit_byte(0x66);
-int  encode = prefix_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0xEB);
 emit_byte(0xC0 | encode);
+}
+void Assembler::por(XMMRegister dst, Address src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
+InstructionMark im(this);
+simd_prefix(dst, dst, src, VEX_SIMD_66);
+emit_byte(0xEB);
+emit_operand(dst, src);
 }
 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
 assert(isByte(mode), "invalid value");
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66);
-emit_byte(0x66);
-int encode = prefix_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0x70);
 emit_byte(0xC0 | encode);
 emit_byte(mode & 0xFF);
 }
 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
 assert(isByte(mode), "invalid value");
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 InstructionMark im(this);
-emit_byte(0x66);
+simd_prefix(dst, src, VEX_SIMD_66);
-prefix(src, dst);
-emit_byte(0x0F);
 emit_byte(0x70);
 emit_operand(dst, src);
 emit_byte(mode & 0xFF);
 }
 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
 assert(isByte(mode), "invalid value");
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_F2);
-emit_byte(0xF2);
-int encode = prefix_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0x70);
 emit_byte(0xC0 | encode);
 emit_byte(mode & 0xFF);
 }
 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
 assert(isByte(mode), "invalid value");
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 InstructionMark im(this);
-emit_byte(0xF2);
+simd_prefix(dst, src, VEX_SIMD_F2);
-prefix(src, dst); // QQ new
-emit_byte(0x0F);
 emit_byte(0x70);
 emit_operand(dst, src);
 emit_byte(mode & 0xFF);
 }
 void Assembler::psrlq(XMMRegister dst, int shift) {
 // Shift 64 bit value logically right by specified number of bits.
 // HMM Table D-1 says sse2 or mmx.
 // Do not confuse it with psrldq SSE2 instruction which
 // shifts 128 bit value in xmm register by number of bytes.
-NOT_LP64(assert(VM_Version::supports_sse(), ""));
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66);
-int encode = prefixq_and_encode(xmm2->encoding(), dst->encoding());
-emit_byte(0x66);
-emit_byte(0x0F);
 emit_byte(0x73);
 emit_byte(0xC0 | encode);
 emit_byte(shift);
 }
 void Assembler::psrldq(XMMRegister dst, int shift) {
 // Shift 128 bit value in xmm register by number of bytes.
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+int encode = simd_prefix_and_encode(xmm3, dst, dst, VEX_SIMD_66);
-int encode = prefixq_and_encode(xmm3->encoding(), dst->encoding());
-emit_byte(0x66);
-emit_byte(0x0F);
 emit_byte(0x73);
 emit_byte(0xC0 | encode);
 emit_byte(shift);
 }
 void Assembler::ptest(XMMRegister dst, Address src) {
 assert(VM_Version::supports_sse4_1(), "");
+assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
 InstructionMark im(this);
-emit_byte(0x66);
+simd_prefix(dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
-prefix(src, dst);
-emit_byte(0x0F);
-emit_byte(0x38);
 emit_byte(0x17);
 emit_operand(dst, src);
 }
 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
 assert(VM_Version::supports_sse4_1(), "");
+int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
-emit_byte(0x66);
-int encode = prefixq_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
-emit_byte(0x38);
 emit_byte(0x17);
 emit_byte(0xC0 | encode);
 }
+void Assembler::punpcklbw(XMMRegister dst, Address src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
+InstructionMark im(this);
+simd_prefix(dst, dst, src, VEX_SIMD_66);
+emit_byte(0x60);
+emit_operand(dst, src);
+}
 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-emit_byte(0x66);
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66);
-int encode = prefix_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0x60);
+emit_byte(0xC0 | encode);
+}
+void Assembler::punpckldq(XMMRegister dst, Address src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
+InstructionMark im(this);
+simd_prefix(dst, dst, src, VEX_SIMD_66);
+emit_byte(0x62);
+emit_operand(dst, src);
+}
+void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66);
+emit_byte(0x62);
 emit_byte(0xC0 | encode);
 }
 void Assembler::push(int32_t imm32) {
 // in 64bits we push 64bits onto the stack but only
 }
 #endif
 void Assembler::pxor(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-InstructionMark im(this);
+assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
-emit_byte(0x66);
+InstructionMark im(this);
-prefix(src, dst);
+simd_prefix(dst, dst, src, VEX_SIMD_66);
-emit_byte(0x0F);
 emit_byte(0xEF);
 emit_operand(dst, src);
 }
 void Assembler::pxor(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-InstructionMark im(this);
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66);
-emit_byte(0x66);
-int encode = prefix_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0xEF);
 emit_byte(0xC0 | encode);
 }
 void Assembler::rcll(Register dst, int imm8) {
 void Assembler::smovl() {
 emit_byte(0xA5);
 }
 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
-// HMM Table D-1 says sse2
-// NOT_LP64(assert(VM_Version::supports_sse(), ""));
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-emit_byte(0xF2);
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2);
-int encode = prefix_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0x51);
 emit_byte(0xC0 | encode);
 }
 void Assembler::sqrtsd(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 InstructionMark im(this);
-emit_byte(0xF2);
+simd_prefix(dst, dst, src, VEX_SIMD_F2);
-prefix(src, dst);
-emit_byte(0x0F);
 emit_byte(0x51);
 emit_operand(dst, src);
 }
 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
-// HMM Table D-1 says sse2
+NOT_LP64(assert(VM_Version::supports_sse(), ""));
-// NOT_LP64(assert(VM_Version::supports_sse(), ""));
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3);
-NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-emit_byte(0xF3);
-int encode = prefix_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0x51);
 emit_byte(0xC0 | encode);
 }
 void Assembler::sqrtss(XMMRegister dst, Address src) {
-NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+NOT_LP64(assert(VM_Version::supports_sse(), ""));
 InstructionMark im(this);
-emit_byte(0xF3);
+simd_prefix(dst, dst, src, VEX_SIMD_F3);
-prefix(src, dst);
-emit_byte(0x0F);
 emit_byte(0x51);
 emit_operand(dst, src);
 }
 void Assembler::stmxcsr( Address dst) {
 void Assembler::subl(Register dst, int32_t imm32) {
 prefix(dst);
 emit_arith(0x81, 0xE8, dst, imm32);
 }
+// Force generation of a 4 byte immediate value even if it fits into 8bit
+void Assembler::subl_imm32(Register dst, int32_t imm32) {
+prefix(dst);
+emit_arith_imm32(0x81, 0xE8, dst, imm32);
+}
 void Assembler::subl(Register dst, Address src) {
 InstructionMark im(this);
 prefix(src, dst);
 emit_byte(0x2B);
 emit_operand(dst, src);
 emit_arith(0x2B, 0xC0, dst, src);
 }
 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-emit_byte(0xF2);
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2);
-int encode = prefix_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0x5C);
 emit_byte(0xC0 | encode);
 }
 void Assembler::subsd(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 InstructionMark im(this);
-emit_byte(0xF2);
+simd_prefix(dst, dst, src, VEX_SIMD_F2);
-prefix(src, dst);
-emit_byte(0x0F);
 emit_byte(0x5C);
 emit_operand(dst, src);
 }
 void Assembler::subss(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-emit_byte(0xF3);
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3);
-int encode = prefix_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0x5C);
 emit_byte(0xC0 | encode);
 }
 void Assembler::subss(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
 InstructionMark im(this);
-emit_byte(0xF3);
+simd_prefix(dst, dst, src, VEX_SIMD_F3);
-prefix(src, dst);
-emit_byte(0x0F);
 emit_byte(0x5C);
 emit_operand(dst, src);
 }
 void Assembler::testb(Register dst, int imm8) {
 emit_operand(dst, src);
 }
 void Assembler::ucomisd(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-emit_byte(0x66);
+InstructionMark im(this);
-ucomiss(dst, src);
+simd_prefix(dst, src, VEX_SIMD_66);
+emit_byte(0x2E);
+emit_operand(dst, src);
 }
 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-emit_byte(0x66);
+int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66);
-ucomiss(dst, src);
+emit_byte(0x2E);
+emit_byte(0xC0 | encode);
 }
 void Assembler::ucomiss(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
+InstructionMark im(this);
-InstructionMark im(this);
+simd_prefix(dst, src, VEX_SIMD_NONE);
-prefix(src, dst);
-emit_byte(0x0F);
 emit_byte(0x2E);
 emit_operand(dst, src);
 }
 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-int encode = prefix_and_encode(dst->encoding(), src->encoding());
+int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_NONE);
-emit_byte(0x0F);
 emit_byte(0x2E);
 emit_byte(0xC0 | encode);
 }
 emit_arith(0x33, 0xC0, dst, src);
 }
 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-emit_byte(0x66);
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66);
-xorps(dst, src);
+emit_byte(0x57);
+emit_byte(0xC0 | encode);
 }
 void Assembler::xorpd(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 InstructionMark im(this);
-emit_byte(0x66);
+simd_prefix(dst, dst, src, VEX_SIMD_66);
-prefix(src, dst);
-emit_byte(0x0F);
 emit_byte(0x57);
 emit_operand(dst, src);
 }
 void Assembler::xorps(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-int encode = prefix_and_encode(dst->encoding(), src->encoding());
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE);
-emit_byte(0x0F);
 emit_byte(0x57);
 emit_byte(0xC0 | encode);
 }
 void Assembler::xorps(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
 InstructionMark im(this);
-prefix(src, dst);
+simd_prefix(dst, dst, src, VEX_SIMD_NONE);
-emit_byte(0x0F);
 emit_byte(0x57);
 emit_operand(dst, src);
 }
+// AVX 3-operands non destructive source instructions (encoded with VEX prefix)
+void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
+assert(VM_Version::supports_avx(), "");
+InstructionMark im(this);
+vex_prefix(dst, nds, src, VEX_SIMD_F2);
+emit_byte(0x58);
+emit_operand(dst, src);
+}
+void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
+assert(VM_Version::supports_avx(), "");
+int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_F2);
+emit_byte(0x58);
+emit_byte(0xC0 | encode);
+}
+void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
+assert(VM_Version::supports_avx(), "");
+InstructionMark im(this);
+vex_prefix(dst, nds, src, VEX_SIMD_F3);
+emit_byte(0x58);
+emit_operand(dst, src);
+}
+void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
+assert(VM_Version::supports_avx(), "");
+int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_F3);
+emit_byte(0x58);
+emit_byte(0xC0 | encode);
+}
+void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src) {
+assert(VM_Version::supports_avx(), "");
+InstructionMark im(this);
+vex_prefix(dst, nds, src, VEX_SIMD_66); // 128-bit vector
+emit_byte(0x54);
+emit_operand(dst, src);
+}
+void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src) {
+assert(VM_Version::supports_avx(), "");
+InstructionMark im(this);
+vex_prefix(dst, nds, src, VEX_SIMD_NONE); // 128-bit vector
+emit_byte(0x54);
+emit_operand(dst, src);
+}
+void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
+assert(VM_Version::supports_avx(), "");
+InstructionMark im(this);
+vex_prefix(dst, nds, src, VEX_SIMD_F2);
+emit_byte(0x5E);
+emit_operand(dst, src);
+}
+void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
+assert(VM_Version::supports_avx(), "");
+int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_F2);
+emit_byte(0x5E);
+emit_byte(0xC0 | encode);
+}
+void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
+assert(VM_Version::supports_avx(), "");
+InstructionMark im(this);
+vex_prefix(dst, nds, src, VEX_SIMD_F3);
+emit_byte(0x5E);
+emit_operand(dst, src);
+}
+void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
+assert(VM_Version::supports_avx(), "");
+int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_F3);
+emit_byte(0x5E);
+emit_byte(0xC0 | encode);
+}
+void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
+assert(VM_Version::supports_avx(), "");
+InstructionMark im(this);
+vex_prefix(dst, nds, src, VEX_SIMD_F2);
+emit_byte(0x59);
+emit_operand(dst, src);
+}
+void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
+assert(VM_Version::supports_avx(), "");
+int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_F2);
+emit_byte(0x59);
+emit_byte(0xC0 | encode);
+}
+void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
+InstructionMark im(this);
+vex_prefix(dst, nds, src, VEX_SIMD_F3);
+emit_byte(0x59);
+emit_operand(dst, src);
+}
+void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
+assert(VM_Version::supports_avx(), "");
+int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_F3);
+emit_byte(0x59);
+emit_byte(0xC0 | encode);
+}
+void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
+assert(VM_Version::supports_avx(), "");
+InstructionMark im(this);
+vex_prefix(dst, nds, src, VEX_SIMD_F2);
+emit_byte(0x5C);
+emit_operand(dst, src);
+}
+void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
+assert(VM_Version::supports_avx(), "");
+int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_F2);
+emit_byte(0x5C);
+emit_byte(0xC0 | encode);
+}
+void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
+assert(VM_Version::supports_avx(), "");
+InstructionMark im(this);
+vex_prefix(dst, nds, src, VEX_SIMD_F3);
+emit_byte(0x5C);
+emit_operand(dst, src);
+}
+void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
+assert(VM_Version::supports_avx(), "");
+int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_F3);
+emit_byte(0x5C);
+emit_byte(0xC0 | encode);
+}
+void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src) {
+assert(VM_Version::supports_avx(), "");
+InstructionMark im(this);
+vex_prefix(dst, nds, src, VEX_SIMD_66); // 128-bit vector
+emit_byte(0x57);
+emit_operand(dst, src);
+}
+void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src) {
+assert(VM_Version::supports_avx(), "");
+InstructionMark im(this);
+vex_prefix(dst, nds, src, VEX_SIMD_NONE); // 128-bit vector
+emit_byte(0x57);
+emit_operand(dst, src);
+}
 #ifndef _LP64
 // 32bit only pieces of the assembler
 void Assembler::cmp_literal32(Register src1, int32_t imm32, RelocationHolder const& rspec) {
 void Assembler::fyl2x() {
 emit_byte(0xD9);
 emit_byte(0xF1);
 }
+// SSE SIMD prefix byte values corresponding to VexSimdPrefix encoding.
+static int simd_pre[4] = { 0, 0x66, 0xF3, 0xF2 };
+// SSE opcode second byte values (first is 0x0F) corresponding to VexOpcode encoding.
+static int simd_opc[4] = { 0,    0, 0x38, 0x3A };
+// Generate SSE legacy REX prefix and SIMD opcode based on VEX encoding.
+void Assembler::rex_prefix(Address adr, XMMRegister xreg, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
+if (pre > 0) {
+emit_byte(simd_pre[pre]);
+}
+if (rex_w) {
+prefixq(adr, xreg);
+} else {
+prefix(adr, xreg);
+}
+if (opc > 0) {
+emit_byte(0x0F);
+int opc2 = simd_opc[opc];
+if (opc2 > 0) {
+emit_byte(opc2);
+}
+}
+}
+int Assembler::rex_prefix_and_encode(int dst_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool rex_w) {
+if (pre > 0) {
+emit_byte(simd_pre[pre]);
+}
+int encode = (rex_w) ? prefixq_and_encode(dst_enc, src_enc) :
+prefix_and_encode(dst_enc, src_enc);
+if (opc > 0) {
+emit_byte(0x0F);
+int opc2 = simd_opc[opc];
+if (opc2 > 0) {
+emit_byte(opc2);
+}
+}
+return encode;
+}
+void Assembler::vex_prefix(bool vex_r, bool vex_b, bool vex_x, bool vex_w, int nds_enc, VexSimdPrefix pre, VexOpcode opc, bool vector256) {
+if (vex_b || vex_x || vex_w || (opc == VEX_OPCODE_0F_38) || (opc == VEX_OPCODE_0F_3A)) {
+prefix(VEX_3bytes);
+int byte1 = (vex_r ? VEX_R : 0) | (vex_x ? VEX_X : 0) | (vex_b ? VEX_B : 0);
+byte1 = (~byte1) & 0xE0;
+byte1 |= opc;
+a_byte(byte1);
+int byte2 = ((~nds_enc) & 0xf) << 3;
+byte2 |= (vex_w ? VEX_W : 0) | (vector256 ? 4 : 0) | pre;
+emit_byte(byte2);
+} else {
+prefix(VEX_2bytes);
+int byte1 = vex_r ? VEX_R : 0;
+byte1 = (~byte1) & 0x80;
+byte1 |= ((~nds_enc) & 0xf) << 3;
+byte1 |= (vector256 ? 4 : 0) | pre;
+emit_byte(byte1);
+}
+}
+void Assembler::vex_prefix(Address adr, int nds_enc, int xreg_enc, VexSimdPrefix pre, VexOpcode opc, bool vex_w, bool vector256){
+bool vex_r = (xreg_enc >= 8);
+bool vex_b = adr.base_needs_rex();
+bool vex_x = adr.index_needs_rex();
+vex_prefix(vex_r, vex_b, vex_x, vex_w, nds_enc, pre, opc, vector256);
+}
+int Assembler::vex_prefix_and_encode(int dst_enc, int nds_enc, int src_enc, VexSimdPrefix pre, VexOpcode opc, bool vex_w, bool vector256) {
+bool vex_r = (dst_enc >= 8);
+bool vex_b = (src_enc >= 8);
+bool vex_x = false;
+vex_prefix(vex_r, vex_b, vex_x, vex_w, nds_enc, pre, opc, vector256);
+return (((dst_enc & 7) << 3) | (src_enc & 7));
+}
+void Assembler::simd_prefix(XMMRegister xreg, XMMRegister nds, Address adr, VexSimdPrefix pre, VexOpcode opc, bool rex_w, bool vector256) {
+if (UseAVX > 0) {
+int xreg_enc = xreg->encoding();
+int  nds_enc = nds->is_valid() ? nds->encoding() : 0;
+vex_prefix(adr, nds_enc, xreg_enc, pre, opc, rex_w, vector256);
+} else {
+assert((nds == xreg) || (nds == xnoreg), "wrong sse encoding");
+rex_prefix(adr, xreg, pre, opc, rex_w);
+}
+}
+int Assembler::simd_prefix_and_encode(XMMRegister dst, XMMRegister nds, XMMRegister src, VexSimdPrefix pre, VexOpcode opc, bool rex_w, bool vector256) {
+int dst_enc = dst->encoding();
+int src_enc = src->encoding();
+if (UseAVX > 0) {
+int nds_enc = nds->is_valid() ? nds->encoding() : 0;
+return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, rex_w, vector256);
+} else {
+assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
+return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, rex_w);
+}
+}
 #ifndef _LP64
 void Assembler::incl(Register dst) {
 // Don't use it directly. Use MacroAssembler::incrementl() instead.
 emit_byte(0x40 | dst->encoding());
 }
 void Assembler::lea(Register dst, Address src) {
 leal(dst, src);
 }
 }
 }
 }
 }
+void Assembler::prefixq(Address adr, XMMRegister src) {
+if (src->encoding() < 8) {
+if (adr.base_needs_rex()) {
+if (adr.index_needs_rex()) {
+prefix(REX_WXB);
+} else {
+prefix(REX_WB);
+}
+} else {
+if (adr.index_needs_rex()) {
+prefix(REX_WX);
+} else {
+prefix(REX_W);
+}
+}
+} else {
+if (adr.base_needs_rex()) {
+if (adr.index_needs_rex()) {
+prefix(REX_WRXB);
+} else {
+prefix(REX_WRB);
+}
+} else {
+if (adr.index_needs_rex()) {
+prefix(REX_WRX);
+} else {
+prefix(REX_WR);
+}
+}
+}
+}
 void Assembler::adcq(Register dst, int32_t imm32) {
 (void) prefixq_and_encode(dst->encoding());
 emit_arith(0x81, 0xD0, dst, imm32);
 }
 emit_operand(reg, adr);
 }
 void Assembler::cvtsi2sdq(XMMRegister dst, Register src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-emit_byte(0xF2);
+int encode = simd_prefix_and_encode_q(dst, dst, src, VEX_SIMD_F2);
-int encode = prefixq_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0x2A);
 emit_byte(0xC0 | encode);
 }
+void Assembler::cvtsi2sdq(XMMRegister dst, Address src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+InstructionMark im(this);
+simd_prefix_q(dst, dst, src, VEX_SIMD_F2);
+emit_byte(0x2A);
+emit_operand(dst, src);
+}
 void Assembler::cvtsi2ssq(XMMRegister dst, Register src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-emit_byte(0xF3);
+int encode = simd_prefix_and_encode_q(dst, dst, src, VEX_SIMD_F3);
-int encode = prefixq_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0x2A);
 emit_byte(0xC0 | encode);
 }
+void Assembler::cvtsi2ssq(XMMRegister dst, Address src) {
+NOT_LP64(assert(VM_Version::supports_sse(), ""));
+InstructionMark im(this);
+simd_prefix_q(dst, dst, src, VEX_SIMD_F3);
+emit_byte(0x2A);
+emit_operand(dst, src);
+}
 void Assembler::cvttsd2siq(Register dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-emit_byte(0xF2);
+int encode = simd_prefix_and_encode_q(dst, src, VEX_SIMD_F2);
-int encode = prefixq_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0x2C);
 emit_byte(0xC0 | encode);
 }
 void Assembler::cvttss2siq(Register dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-emit_byte(0xF3);
+int encode = simd_prefix_and_encode_q(dst, src, VEX_SIMD_F3);
-int encode = prefixq_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0x2C);
 emit_byte(0xC0 | encode);
 }
 void Assembler::decl(Register dst) {
 emit_byte(0xC0 | encode);
 }
 void Assembler::movdq(XMMRegister dst, Register src) {
 // table D-1 says MMX/SSE2
-NOT_LP64(assert(VM_Version::supports_sse2() || VM_Version::supports_mmx(), ""));
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-emit_byte(0x66);
+int encode = simd_prefix_and_encode_q(dst, src, VEX_SIMD_66);
-int encode = prefixq_and_encode(dst->encoding(), src->encoding());
-emit_byte(0x0F);
 emit_byte(0x6E);
 emit_byte(0xC0 | encode);
 }
 void Assembler::movdq(Register dst, XMMRegister src) {
 // table D-1 says MMX/SSE2
-NOT_LP64(assert(VM_Version::supports_sse2() || VM_Version::supports_mmx(), ""));
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-emit_byte(0x66);
 // swap src/dst to get correct prefix
-int encode = prefixq_and_encode(src->encoding(), dst->encoding());
+int encode = simd_prefix_and_encode_q(src, dst, VEX_SIMD_66);
-emit_byte(0x0F);
 emit_byte(0x7E);
 emit_byte(0xC0 | encode);
 }
 void Assembler::movq(Register dst, Register src) {
 void Assembler::subq(Register dst, int32_t imm32) {
 (void) prefixq_and_encode(dst->encoding());
 emit_arith(0x81, 0xE8, dst, imm32);
 }
+// Force generation of a 4 byte immediate value even if it fits into 8bit
+void Assembler::subq_imm32(Register dst, int32_t imm32) {
+(void) prefixq_and_encode(dst->encoding());
+emit_arith_imm32(0x81, 0xE8, dst, imm32);
+}
 void Assembler::subq(Register dst, Address src) {
 InstructionMark im(this);
 prefixq(src, dst);
 emit_byte(0x2B);
 emit_operand(dst, src);
 xorl(swap_reg, tmp_reg);
 if (swap_reg_contains_mark) {
 null_check_offset = offset();
 }
 movl(tmp_reg, klass_addr);
-xorl(swap_reg, Address(tmp_reg, Klass::prototype_header_offset_in_bytes() + klassOopDesc::klass_part_offset_in_bytes()));
+xorl(swap_reg, Address(tmp_reg, Klass::prototype_header_offset()));
 andl(swap_reg, ~((int) markOopDesc::age_mask_in_place));
 if (need_tmp_reg) {
 pop(tmp_reg);
 }
 if (counters != NULL) {
 if (need_tmp_reg) {
 push(tmp_reg);
 }
 get_thread(tmp_reg);
 movl(swap_reg, klass_addr);
-orl(tmp_reg, Address(swap_reg, Klass::prototype_header_offset_in_bytes() + klassOopDesc::klass_part_offset_in_bytes()));
+orl(tmp_reg, Address(swap_reg, Klass::prototype_header_offset()));
 movl(swap_reg, saved_mark_addr);
 if (os::is_MP()) {
 lock();
 }
 cmpxchgptr(tmp_reg, Address(obj_reg, 0));
 movl(swap_reg, saved_mark_addr);
 if (need_tmp_reg) {
 push(tmp_reg);
 }
 movl(tmp_reg, klass_addr);
-movl(tmp_reg, Address(tmp_reg, Klass::prototype_header_offset_in_bytes() + klassOopDesc::klass_part_offset_in_bytes()));
+movl(tmp_reg, Address(tmp_reg, Klass::prototype_header_offset()));
 if (os::is_MP()) {
 lock();
 }
 cmpxchgptr(tmp_reg, Address(obj_reg, 0));
 if (need_tmp_reg) {
 cdql();
 } else {
 movl(hi, lo);
 sarl(hi, 31);
 }
-}
-void MacroAssembler::fat_nop() {
-// A 5 byte nop that is safe for patching (see patch_verified_entry)
-emit_byte(0x26); // es:
-emit_byte(0x2e); // cs:
-emit_byte(0x64); // fs:
-emit_byte(0x65); // gs:
-emit_byte(0x90);
 }
 void MacroAssembler::jC2(Register tmp, Label& L) {
 // set parity bit if FPU flag C2 is set (via rax)
 save_rax(tmp);
 if (value == min_jint) { subq(dst, value); return; }
 if (value <  0) { incrementq(dst, -value); return; }
 if (value == 0) {                        ; return; }
 if (value == 1 && UseIncDec) { decq(dst) ; return; }
 /* else */      { subq(dst, value)       ; return; }
-}
-void MacroAssembler::fat_nop() {
-// A 5 byte nop that is safe for patching (see patch_verified_entry)
-// Recommened sequence from 'Software Optimization Guide for the AMD
-// Hammer Processor'
-emit_byte(0x66);
-emit_byte(0x66);
-emit_byte(0x90);
-emit_byte(0x66);
-emit_byte(0x90);
 }
 void MacroAssembler::incrementq(Register reg, int value) {
 if (value == min_jint) { addq(reg, value); return; }
 if (value <  0) { decrementq(reg, -value); return; }
 void MacroAssembler::addptr(Address dst, Register src) {
 LP64_ONLY(addq(dst, src)) NOT_LP64(addl(dst, src));
 }
+void MacroAssembler::addsd(XMMRegister dst, AddressLiteral src) {
+if (reachable(src)) {
+Assembler::addsd(dst, as_Address(src));
+} else {
+lea(rscratch1, src);
+Assembler::addsd(dst, Address(rscratch1, 0));
+}
+}
+void MacroAssembler::addss(XMMRegister dst, AddressLiteral src) {
+if (reachable(src)) {
+addss(dst, as_Address(src));
+} else {
+lea(rscratch1, src);
+addss(dst, Address(rscratch1, 0));
+}
+}
 void MacroAssembler::align(int modulus) {
 if (offset() % modulus != 0) {
 nop(modulus - (offset() % modulus));
 }
 }
 void MacroAssembler::andpd(XMMRegister dst, AddressLiteral src) {
+// Used in sign-masking with aligned address.
+assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
 if (reachable(src)) {
-andpd(dst, as_Address(src));
+Assembler::andpd(dst, as_Address(src));
 } else {
 lea(rscratch1, src);
-andpd(dst, Address(rscratch1, 0));
+Assembler::andpd(dst, Address(rscratch1, 0));
+}
+}
+void MacroAssembler::andps(XMMRegister dst, AddressLiteral src) {
+// Used in sign-masking with aligned address.
+assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
+if (reachable(src)) {
+Assembler::andps(dst, as_Address(src));
+} else {
+lea(rscratch1, src);
+Assembler::andps(dst, Address(rscratch1, 0));
 }
 }
 void MacroAssembler::andptr(Register dst, int32_t imm32) {
 LP64_ONLY(andq(dst, imm32)) NOT_LP64(andl(dst, imm32));
 }
 // debugging support
 assert(number_of_arguments >= 0   , "cannot have negative number of arguments");
 LP64_ONLY(assert(java_thread == r15_thread, "unexpected register"));
 #ifdef ASSERT
-LP64_ONLY(if (UseCompressedOops) verify_heapbase("call_VM_base");)
+// TraceBytecodes does not use r12 but saves it over the call, so don't verify
+// r12 is the heapbase.
+LP64_ONLY(if (UseCompressedOops && !TraceBytecodes) verify_heapbase("call_VM_base");)
 #endif // ASSERT
 assert(java_thread != oop_result  , "cannot use the same register for java_thread & oop_result");
 assert(java_thread != last_java_sp, "cannot use the same register for java_thread & last_java_sp");
 LP64_ONLY(cmpxchgq(reg, adr)) NOT_LP64(cmpxchgl(reg, adr));
 }
 void MacroAssembler::comisd(XMMRegister dst, AddressLiteral src) {
 if (reachable(src)) {
-comisd(dst, as_Address(src));
+Assembler::comisd(dst, as_Address(src));
 } else {
 lea(rscratch1, src);
-comisd(dst, Address(rscratch1, 0));
+Assembler::comisd(dst, Address(rscratch1, 0));
 }
 }
 void MacroAssembler::comiss(XMMRegister dst, AddressLiteral src) {
 if (reachable(src)) {
-comiss(dst, as_Address(src));
+Assembler::comiss(dst, as_Address(src));
 } else {
 lea(rscratch1, src);
-comiss(dst, Address(rscratch1, 0));
+Assembler::comiss(dst, Address(rscratch1, 0));
 }
 }
 void MacroAssembler::cond_inc32(Condition cond, AddressLiteral counter_addr) {
 addl(reg, offset);
 }
 bind (_is_positive);
 sarl(reg, shift_value);
+}
+void MacroAssembler::divsd(XMMRegister dst, AddressLiteral src) {
+if (reachable(src)) {
+Assembler::divsd(dst, as_Address(src));
+} else {
+lea(rscratch1, src);
+Assembler::divsd(dst, Address(rscratch1, 0));
+}
+}
+void MacroAssembler::divss(XMMRegister dst, AddressLiteral src) {
+if (reachable(src)) {
+Assembler::divss(dst, as_Address(src));
+} else {
+lea(rscratch1, src);
+Assembler::divss(dst, Address(rscratch1, 0));
+}
 }
 // !defined(COMPILER2) is because of stupid core builds
 #if !defined(_LP64) || defined(COMPILER1) || !defined(COMPILER2)
 void MacroAssembler::empty_FPU_stack() {
 void MacroAssembler::enter() {
 push(rbp);
 mov(rbp, rsp);
 }
+// A 5 byte nop that is safe for patching (see patch_verified_entry)
+void MacroAssembler::fat_nop() {
+if (UseAddressNop) {
+addr_nop_5();
+} else {
+emit_byte(0x26); // es:
+emit_byte(0x2e); // cs:
+emit_byte(0x64); // fs:
+emit_byte(0x65); // gs:
+emit_byte(0x90);
+}
+}
 void MacroAssembler::fcmp(Register tmp) {
 fcmp(tmp, 1, true, true);
 }
 void MacroAssembler::fcmp(Register tmp, int index, bool pop_left, bool pop_right) {
 void MacroAssembler::movptr(Address dst, Register src) {
 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 }
+void MacroAssembler::movsd(XMMRegister dst, AddressLiteral src) {
+if (reachable(src)) {
+Assembler::movsd(dst, as_Address(src));
+} else {
+lea(rscratch1, src);
+Assembler::movsd(dst, Address(rscratch1, 0));
+}
+}
 void MacroAssembler::movss(XMMRegister dst, AddressLiteral src) {
 if (reachable(src)) {
-movss(dst, as_Address(src));
+Assembler::movss(dst, as_Address(src));
 } else {
 lea(rscratch1, src);
-movss(dst, Address(rscratch1, 0));
+Assembler::movss(dst, Address(rscratch1, 0));
+}
+}
+void MacroAssembler::mulsd(XMMRegister dst, AddressLiteral src) {
+if (reachable(src)) {
+Assembler::mulsd(dst, as_Address(src));
+} else {
+lea(rscratch1, src);
+Assembler::mulsd(dst, Address(rscratch1, 0));
+}
+}
+void MacroAssembler::mulss(XMMRegister dst, AddressLiteral src) {
+if (reachable(src)) {
+Assembler::mulss(dst, as_Address(src));
+} else {
+lea(rscratch1, src);
+Assembler::mulss(dst, Address(rscratch1, 0));
 }
 }
 void MacroAssembler::null_check(Register reg, int offset) {
 if (needs_explicit_null_check(offset)) {
 void MacroAssembler::testl(Register dst, AddressLiteral src) {
 assert(reachable(src), "Address should be reachable");
 testl(dst, as_Address(src));
 }
+void MacroAssembler::sqrtsd(XMMRegister dst, AddressLiteral src) {
+if (reachable(src)) {
+Assembler::sqrtsd(dst, as_Address(src));
+} else {
+lea(rscratch1, src);
+Assembler::sqrtsd(dst, Address(rscratch1, 0));
+}
+}
+void MacroAssembler::sqrtss(XMMRegister dst, AddressLiteral src) {
+if (reachable(src)) {
+Assembler::sqrtss(dst, as_Address(src));
+} else {
+lea(rscratch1, src);
+Assembler::sqrtss(dst, Address(rscratch1, 0));
+}
+}
+void MacroAssembler::subsd(XMMRegister dst, AddressLiteral src) {
+if (reachable(src)) {
+Assembler::subsd(dst, as_Address(src));
+} else {
+lea(rscratch1, src);
+Assembler::subsd(dst, Address(rscratch1, 0));
+}
+}
+void MacroAssembler::subss(XMMRegister dst, AddressLiteral src) {
+if (reachable(src)) {
+Assembler::subss(dst, as_Address(src));
+} else {
+lea(rscratch1, src);
+Assembler::subss(dst, Address(rscratch1, 0));
+}
+}
+void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src) {
+if (reachable(src)) {
+Assembler::ucomisd(dst, as_Address(src));
+} else {
+lea(rscratch1, src);
+Assembler::ucomisd(dst, Address(rscratch1, 0));
+}
+}
+void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src) {
+if (reachable(src)) {
+Assembler::ucomiss(dst, as_Address(src));
+} else {
+lea(rscratch1, src);
+Assembler::ucomiss(dst, Address(rscratch1, 0));
+}
+}
+void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src) {
+// Used in sign-bit flipping with aligned address.
+assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
+if (reachable(src)) {
+Assembler::xorpd(dst, as_Address(src));
+} else {
+lea(rscratch1, src);
+Assembler::xorpd(dst, Address(rscratch1, 0));
+}
+}
+void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src) {
+// Used in sign-bit flipping with aligned address.
+assert((UseAVX > 0) || (((intptr_t)src.target() & 15) == 0), "SSE mode requires address alignment 16 bytes");
+if (reachable(src)) {
+Assembler::xorps(dst, as_Address(src));
+} else {
+lea(rscratch1, src);
+Assembler::xorps(dst, Address(rscratch1, 0));
+}
+}
+// AVX 3-operands instructions
+void MacroAssembler::vaddsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
+if (reachable(src)) {
+vaddsd(dst, nds, as_Address(src));
+} else {
+lea(rscratch1, src);
+vaddsd(dst, nds, Address(rscratch1, 0));
+}
+}
+void MacroAssembler::vaddss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
+if (reachable(src)) {
+vaddss(dst, nds, as_Address(src));
+} else {
+lea(rscratch1, src);
+vaddss(dst, nds, Address(rscratch1, 0));
+}
+}
+void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
+if (reachable(src)) {
+vandpd(dst, nds, as_Address(src));
+} else {
+lea(rscratch1, src);
+vandpd(dst, nds, Address(rscratch1, 0));
+}
+}
+void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
+if (reachable(src)) {
+vandps(dst, nds, as_Address(src));
+} else {
+lea(rscratch1, src);
+vandps(dst, nds, Address(rscratch1, 0));
+}
+}
+void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
+if (reachable(src)) {
+vdivsd(dst, nds, as_Address(src));
+} else {
+lea(rscratch1, src);
+vdivsd(dst, nds, Address(rscratch1, 0));
+}
+}
+void MacroAssembler::vdivss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
+if (reachable(src)) {
+vdivss(dst, nds, as_Address(src));
+} else {
+lea(rscratch1, src);
+vdivss(dst, nds, Address(rscratch1, 0));
+}
+}
+void MacroAssembler::vmulsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
+if (reachable(src)) {
+vmulsd(dst, nds, as_Address(src));
+} else {
+lea(rscratch1, src);
+vmulsd(dst, nds, Address(rscratch1, 0));
+}
+}
+void MacroAssembler::vmulss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
+if (reachable(src)) {
+vmulss(dst, nds, as_Address(src));
+} else {
+lea(rscratch1, src);
+vmulss(dst, nds, Address(rscratch1, 0));
+}
+}
+void MacroAssembler::vsubsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
+if (reachable(src)) {
+vsubsd(dst, nds, as_Address(src));
+} else {
+lea(rscratch1, src);
+vsubsd(dst, nds, Address(rscratch1, 0));
+}
+}
+void MacroAssembler::vsubss(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
+if (reachable(src)) {
+vsubss(dst, nds, as_Address(src));
+} else {
+lea(rscratch1, src);
+vsubss(dst, nds, Address(rscratch1, 0));
+}
+}
+void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
+if (reachable(src)) {
+vxorpd(dst, nds, as_Address(src));
+} else {
+lea(rscratch1, src);
+vxorpd(dst, nds, Address(rscratch1, 0));
+}
+}
+void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
+if (reachable(src)) {
+vxorps(dst, nds, as_Address(src));
+} else {
+lea(rscratch1, src);
+vxorps(dst, nds, Address(rscratch1, 0));
+}
+}
 //////////////////////////////////////////////////////////////////////////////////
 #ifndef SERIALGC
 void MacroAssembler::g1_write_barrier_pre(Register obj,
 Register pre_val,
 void MacroAssembler::subptr(Register dst, int32_t imm32) {
 LP64_ONLY(subq(dst, imm32)) NOT_LP64(subl(dst, imm32));
 }
+// Force generation of a 4 byte immediate value even if it fits into 8bit
+void MacroAssembler::subptr_imm32(Register dst, int32_t imm32) {
+LP64_ONLY(subq_imm32(dst, imm32)) NOT_LP64(subl_imm32(dst, imm32));
+}
 void MacroAssembler::subptr(Register dst, Register src) {
 LP64_ONLY(subq(dst, src)) NOT_LP64(subl(dst, src));
 }
 // C++ bool manipulation
 void MacroAssembler::incr_allocated_bytes(Register thread,
 Register var_size_in_bytes,
 int con_size_in_bytes,
 Register t1) {
+if (!thread->is_valid()) {
+#ifdef _LP64
+thread = r15_thread;
+#else
+assert(t1->is_valid(), "need temp reg");
+thread = t1;
+get_thread(thread);
+#endif
+}
 #ifdef _LP64
 if (var_size_in_bytes->is_valid()) {
 addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes);
 } else {
 addq(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes);
 }
 #else
-if (!thread->is_valid()) {
-assert(t1->is_valid(), "need temp reg");
-thread = t1;
-get_thread(thread);
-}
 if (var_size_in_bytes->is_valid()) {
 addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), var_size_in_bytes);
 } else {
 addl(Address(thread, in_bytes(JavaThread::allocated_bytes_offset())), con_size_in_bytes);
 }
 if (L_success == NULL)   { L_success   = &L_fallthrough; label_nulls++; }
 if (L_failure == NULL)   { L_failure   = &L_fallthrough; label_nulls++; }
 if (L_slow_path == NULL) { L_slow_path = &L_fallthrough; label_nulls++; }
 assert(label_nulls <= 1, "at most one NULL in the batch");
-int sc_offset = (klassOopDesc::header_size() * HeapWordSize +
+int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
-Klass::secondary_super_cache_offset_in_bytes());
+int sco_offset = in_bytes(Klass::super_check_offset_offset());
-int sco_offset = (klassOopDesc::header_size() * HeapWordSize +
-Klass::super_check_offset_offset_in_bytes());
 Address super_check_offset_addr(super_klass, sco_offset);
 // Hacked jcc, which "knows" that L_fallthrough, at least, is in
 // range of a jccb.  If this routine grows larger, reconsider at
 // least some of these.
 if (L_success == NULL)   { L_success   = &L_fallthrough; label_nulls++; }
 if (L_failure == NULL)   { L_failure   = &L_fallthrough; label_nulls++; }
 assert(label_nulls <= 1, "at most one NULL in the batch");
 // a couple of useful fields in sub_klass:
-int ss_offset = (klassOopDesc::header_size() * HeapWordSize +
+int ss_offset = in_bytes(Klass::secondary_supers_offset());
-Klass::secondary_supers_offset_in_bytes());
+int sc_offset = in_bytes(Klass::secondary_super_cache_offset());
-int sc_offset = (klassOopDesc::header_size() * HeapWordSize +
-Klass::secondary_super_cache_offset_in_bytes());
 Address secondary_supers_addr(sub_klass, ss_offset);
 Address super_cache_addr(     sub_klass, sc_offset);
 // Do a linear scan of the secondary super-klass chain.
 // This code is rarely used, so simplicity is a virtue here.
 #undef IS_A_TEMP
 bind(L_fallthrough);
 }
-void MacroAssembler::ucomisd(XMMRegister dst, AddressLiteral src) {
-ucomisd(dst, as_Address(src));
-}
-void MacroAssembler::ucomiss(XMMRegister dst, AddressLiteral src) {
-ucomiss(dst, as_Address(src));
-}
-void MacroAssembler::xorpd(XMMRegister dst, AddressLiteral src) {
-if (reachable(src)) {
-xorpd(dst, as_Address(src));
-} else {
-lea(rscratch1, src);
-xorpd(dst, Address(rscratch1, 0));
-}
-}
-void MacroAssembler::xorps(XMMRegister dst, AddressLiteral src) {
-if (reachable(src)) {
-xorps(dst, as_Address(src));
-} else {
-lea(rscratch1, src);
-xorps(dst, Address(rscratch1, 0));
-}
-}
 void MacroAssembler::cmov32(Condition cc, Register dst, Address src) {
 if (VM_Version::supports_cmov()) {
 cmovl(cc, dst, src);
 } else {
 assert (Universe::heap() != NULL, "java heap should be initialized");
 movl(dst, Address(src, oopDesc::klass_offset_in_bytes()));
 if (Universe::narrow_oop_shift() != 0) {
 assert(LogMinObjAlignmentInBytes == Universe::narrow_oop_shift(), "decode alg wrong");
 if (LogMinObjAlignmentInBytes == Address::times_8) {
-movq(dst, Address(r12_heapbase, dst, Address::times_8, Klass::prototype_header_offset_in_bytes() + klassOopDesc::klass_part_offset_in_bytes()));
+movq(dst, Address(r12_heapbase, dst, Address::times_8, Klass::prototype_header_offset()));
 } else {
 // OK to use shift since we don't need to preserve flags.
 shlq(dst, LogMinObjAlignmentInBytes);
-movq(dst, Address(r12_heapbase, dst, Address::times_1, Klass::prototype_header_offset_in_bytes() + klassOopDesc::klass_part_offset_in_bytes()));
+movq(dst, Address(r12_heapbase, dst, Address::times_1, Klass::prototype_header_offset()));
 }
 } else {
-movq(dst, Address(dst, Klass::prototype_header_offset_in_bytes() + klassOopDesc::klass_part_offset_in_bytes()));
+movq(dst, Address(dst, Klass::prototype_header_offset()));
 }
 } else
 #endif
 {
 movptr(dst, Address(src, oopDesc::klass_offset_in_bytes()));
-movptr(dst, Address(dst, Klass::prototype_header_offset_in_bytes() + klassOopDesc::klass_part_offset_in_bytes()));
+movptr(dst, Address(dst, Klass::prototype_header_offset()));
 }
 }
 void MacroAssembler::store_klass(Register dst, Register src) {
 #ifdef _LP64
 movptr(r12_heapbase, ExternalAddress((address)Universe::narrow_oop_base_addr()));
 }
 }
 #endif // _LP64
+// C2 compiled method's prolog code.
+void MacroAssembler::verified_entry(int framesize, bool stack_bang, bool fp_mode_24b) {
+// WARNING: Initial instruction MUST be 5 bytes or longer so that
+// NativeJump::patch_verified_entry will be able to patch out the entry
+// code safely. The push to verify stack depth is ok at 5 bytes,
+// the frame allocation can be either 3 or 6 bytes. So if we don't do
+// stack bang then we must use the 6 byte frame allocation even if
+// we have no frame. :-(
+assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
+// Remove word for return addr
+framesize -= wordSize;
+// Calls to C2R adapters often do not accept exceptional returns.
+// We require that their callers must bang for them.  But be careful, because
+// some VM calls (such as call site linkage) can use several kilobytes of
+// stack.  But the stack safety zone should account for that.
+// See bugs 4446381, 4468289, 4497237.
+if (stack_bang) {
+generate_stack_overflow_check(framesize);
+// We always push rbp, so that on return to interpreter rbp, will be
+// restored correctly and we can correct the stack.
+push(rbp);
+// Remove word for ebp
+framesize -= wordSize;
+// Create frame
+if (framesize) {
+subptr(rsp, framesize);
+}
+} else {
+// Create frame (force generation of a 4 byte immediate value)
+subptr_imm32(rsp, framesize);
+// Save RBP register now.
+framesize -= wordSize;
+movptr(Address(rsp, framesize), rbp);
+}
+if (VerifyStackAtCalls) { // Majik cookie to verify stack depth
+framesize -= wordSize;
+movptr(Address(rsp, framesize), (int32_t)0xbadb100d);
+}
+#ifndef _LP64
+// If method sets FPU control word do it now
+if (fp_mode_24b) {
+fldcw(ExternalAddress(StubRoutines::addr_fpu_cntrl_wrd_24()));
+}
+if (UseSSE >= 2 && VerifyFPU) {
+verify_FPU(0, "FPU stack must be clean on entry");
+}
+#endif
+#ifdef ASSERT
+if (VerifyStackAtCalls) {
+Label L;
+push(rax);
+mov(rax, rsp);
+andptr(rax, StackAlignmentInBytes-1);
+cmpptr(rax, StackAlignmentInBytes-wordSize);
+pop(rax);
+jcc(Assembler::equal, L);
+stop("Stack is not properly aligned!");
+bind(L);
+}
+#endif
+}
 // IndexOf for constant substrings with size >= 8 chars
 // which don't need to be loaded through stack.
 void MacroAssembler::string_indexofC8(Register str1, Register str2,
 Register cnt1, Register cnt2,
 int int_cnt2,  Register result,
 XMMRegister vec, Register tmp) {
+ShortBranchVerifier sbv(this);
 assert(UseSSE42Intrinsics, "SSE4.2 is required");
 // This method uses pcmpestri inxtruction with bound registers
 //   inputs:
 //     xmm - substring
 addptr(tmp, cnt2);
 movdqu(vec, Address(str2, tmp, Address::times_2, 0));
 pcmpestri(vec, Address(result, tmp, Address::times_2, 0), 0x0d);
 }
 // Need to reload strings pointers if not matched whole vector
-jccb(Assembler::noOverflow, RELOAD_SUBSTR); // OF == 0
+jcc(Assembler::noOverflow, RELOAD_SUBSTR); // OF == 0
 addptr(cnt2, 8);
-jccb(Assembler::negative, SCAN_SUBSTR);
+jcc(Assembler::negative, SCAN_SUBSTR);
 // Fall through if found full substring
 } // (int_cnt2 > 8)
 bind(RET_FOUND);
 // Small strings are loaded through stack if they cross page boundary.
 void MacroAssembler::string_indexof(Register str1, Register str2,
 Register cnt1, Register cnt2,
 int int_cnt2,  Register result,
 XMMRegister vec, Register tmp) {
+ShortBranchVerifier sbv(this);
 assert(UseSSE42Intrinsics, "SSE4.2 is required");
 //
 // int_cnt2 is length of small (< 8 chars) constant substring
 // or (-1) for non constant substring in which case its length
 // is in cnt2 register.
 // Compare strings.
 void MacroAssembler::string_compare(Register str1, Register str2,
 Register cnt1, Register cnt2, Register result,
 XMMRegister vec1) {
+ShortBranchVerifier sbv(this);
 Label LENGTH_DIFF_LABEL, POP_LABEL, DONE_LABEL, WHILE_HEAD_LABEL;
 // Compute the minimum of the string lengths and the
 // difference of the string lengths (stack).
 // Do the conditional move stuff
 // Compare char[] arrays aligned to 4 bytes or substrings.
 void MacroAssembler::char_arrays_equals(bool is_array_equ, Register ary1, Register ary2,
 Register limit, Register result, Register chr,
 XMMRegister vec1, XMMRegister vec2) {
+ShortBranchVerifier sbv(this);
 Label TRUE_LABEL, FALSE_LABEL, DONE, COMPARE_VECTORS, COMPARE_CHAR;
 int length_offset  = arrayOopDesc::length_offset_in_bytes();
 int base_offset    = arrayOopDesc::base_offset_in_bytes(T_CHAR);
 #define BIND(label) bind(label); BLOCK_COMMENT(#label ":")
 void MacroAssembler::generate_fill(BasicType t, bool aligned,
 Register to, Register value, Register count,
 Register rtmp, XMMRegister xtmp) {
+ShortBranchVerifier sbv(this);
 assert_different_registers(to, value, count, rtmp);
 Label L_exit, L_skip_align1, L_skip_align2, L_fill_byte;
 Label L_fill_2_bytes, L_fill_4_bytes;
 int shift = -1;

Mercurial > hg > graal-jvmci-8

comparison src/cpu/x86/vm/assembler_x86.cpp @ 4970:33df1aeaebbf