truffle: src/cpu/x86/vm/assembler

comparison src/cpu/x86/vm/assembler_x86.cpp @ 6614:006050192a5a

6340864: Implement vectorization optimizations in hotspot-server Summary: Added asm encoding and mach nodes for vector arithmetic instructions on x86. Reviewed-by: roland

author	kvn
date	Mon, 20 Aug 2012 09:07:21 -0700
parents	1d7922586cf6
children	c38f13903fdf da91efe96a93

comparison

equal deleted inserted replaced

-:d5ec46c7da5c
+:006050192a5a
 emit_long(0);    // 32-bits offset (4 bytes)
 }
 void Assembler::addsd(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2);
+emit_simd_arith(0x58, dst, src, VEX_SIMD_F2);
-emit_byte(0x58);
-emit_byte(0xC0 | encode);
 }
 void Assembler::addsd(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-InstructionMark im(this);
+emit_simd_arith(0x58, dst, src, VEX_SIMD_F2);
-simd_prefix(dst, dst, src, VEX_SIMD_F2);
-emit_byte(0x58);
-emit_operand(dst, src);
 }
 void Assembler::addss(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3);
+emit_simd_arith(0x58, dst, src, VEX_SIMD_F3);
-emit_byte(0x58);
-emit_byte(0xC0 | encode);
 }
 void Assembler::addss(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-InstructionMark im(this);
+emit_simd_arith(0x58, dst, src, VEX_SIMD_F3);
-simd_prefix(dst, dst, src, VEX_SIMD_F3);
-emit_byte(0x58);
-emit_operand(dst, src);
 }
 void Assembler::andl(Address dst, int32_t imm32) {
 InstructionMark im(this);
 prefix(dst);
 }
 void Assembler::andl(Register dst, Register src) {
 (void) prefix_and_encode(dst->encoding(), src->encoding());
 emit_arith(0x23, 0xC0, dst, src);
-}
-void Assembler::andpd(XMMRegister dst, Address src) {
-NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-InstructionMark im(this);
-simd_prefix(dst, dst, src, VEX_SIMD_66);
-emit_byte(0x54);
-emit_operand(dst, src);
-}
-void Assembler::andpd(XMMRegister dst, XMMRegister src) {
-NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66);
-emit_byte(0x54);
-emit_byte(0xC0 | encode);
-}
-void Assembler::andps(XMMRegister dst, Address src) {
-NOT_LP64(assert(VM_Version::supports_sse(), ""));
-InstructionMark im(this);
-simd_prefix(dst, dst, src, VEX_SIMD_NONE);
-emit_byte(0x54);
-emit_operand(dst, src);
-}
-void Assembler::andps(XMMRegister dst, XMMRegister src) {
-NOT_LP64(assert(VM_Version::supports_sse(), ""));
-int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE);
-emit_byte(0x54);
-emit_byte(0xC0 | encode);
 }
 void Assembler::bsfl(Register dst, Register src) {
 int encode = prefix_and_encode(dst->encoding(), src->encoding());
 emit_byte(0x0F);
 void Assembler::comisd(XMMRegister dst, Address src) {
 // NOTE: dbx seems to decode this as comiss even though the
 // 0x66 is there. Strangly ucomisd comes out correct
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-InstructionMark im(this);
+emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_66);
-simd_prefix(dst, src, VEX_SIMD_66);
-emit_byte(0x2F);
-emit_operand(dst, src);
 }
 void Assembler::comisd(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66);
+emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_66);
-emit_byte(0x2F);
-emit_byte(0xC0 | encode);
 }
 void Assembler::comiss(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-InstructionMark im(this);
+emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_NONE);
-simd_prefix(dst, src, VEX_SIMD_NONE);
-emit_byte(0x2F);
-emit_operand(dst, src);
 }
 void Assembler::comiss(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_NONE);
+emit_simd_arith_nonds(0x2F, dst, src, VEX_SIMD_NONE);
-emit_byte(0x2F);
-emit_byte(0xC0 | encode);
 }
 void Assembler::cvtdq2pd(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_F3);
+emit_simd_arith_nonds(0xE6, dst, src, VEX_SIMD_F3);
-emit_byte(0xE6);
-emit_byte(0xC0 | encode);
 }
 void Assembler::cvtdq2ps(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_NONE);
+emit_simd_arith_nonds(0x5B, dst, src, VEX_SIMD_NONE);
-emit_byte(0x5B);
-emit_byte(0xC0 | encode);
 }
 void Assembler::cvtsd2ss(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2);
+emit_simd_arith(0x5A, dst, src, VEX_SIMD_F2);
-emit_byte(0x5A);
-emit_byte(0xC0 | encode);
 }
 void Assembler::cvtsd2ss(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-InstructionMark im(this);
+emit_simd_arith(0x5A, dst, src, VEX_SIMD_F2);
-simd_prefix(dst, dst, src, VEX_SIMD_F2);
-emit_byte(0x5A);
-emit_operand(dst, src);
 }
 void Assembler::cvtsi2sdl(XMMRegister dst, Register src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2);
 emit_byte(0xC0 | encode);
 }
 void Assembler::cvtsi2sdl(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-InstructionMark im(this);
+emit_simd_arith(0x2A, dst, src, VEX_SIMD_F2);
-simd_prefix(dst, dst, src, VEX_SIMD_F2);
-emit_byte(0x2A);
-emit_operand(dst, src);
 }
 void Assembler::cvtsi2ssl(XMMRegister dst, Register src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
 int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3);
 emit_byte(0xC0 | encode);
 }
 void Assembler::cvtsi2ssl(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-InstructionMark im(this);
+emit_simd_arith(0x2A, dst, src, VEX_SIMD_F3);
-simd_prefix(dst, dst, src, VEX_SIMD_F3);
-emit_byte(0x2A);
-emit_operand(dst, src);
 }
 void Assembler::cvtss2sd(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3);
+emit_simd_arith(0x5A, dst, src, VEX_SIMD_F3);
-emit_byte(0x5A);
-emit_byte(0xC0 | encode);
 }
 void Assembler::cvtss2sd(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-InstructionMark im(this);
+emit_simd_arith(0x5A, dst, src, VEX_SIMD_F3);
-simd_prefix(dst, dst, src, VEX_SIMD_F3);
-emit_byte(0x5A);
-emit_operand(dst, src);
 }
 void Assembler::cvttsd2sil(Register dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 emit_operand(rcx, dst);
 }
 void Assembler::divsd(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-InstructionMark im(this);
+emit_simd_arith(0x5E, dst, src, VEX_SIMD_F2);
-simd_prefix(dst, dst, src, VEX_SIMD_F2);
-emit_byte(0x5E);
-emit_operand(dst, src);
 }
 void Assembler::divsd(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2);
+emit_simd_arith(0x5E, dst, src, VEX_SIMD_F2);
-emit_byte(0x5E);
-emit_byte(0xC0 | encode);
 }
 void Assembler::divss(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-InstructionMark im(this);
+emit_simd_arith(0x5E, dst, src, VEX_SIMD_F3);
-simd_prefix(dst, dst, src, VEX_SIMD_F3);
-emit_byte(0x5E);
-emit_operand(dst, src);
 }
 void Assembler::divss(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3);
+emit_simd_arith(0x5E, dst, src, VEX_SIMD_F3);
-emit_byte(0x5E);
-emit_byte(0xC0 | encode);
 }
 void Assembler::emms() {
 NOT_LP64(assert(VM_Version::supports_mmx(), ""));
 emit_byte(0x0F);
 LP64_ONLY(movq(dst, src)) NOT_LP64(movl(dst, src));
 }
 void Assembler::movapd(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66);
+emit_simd_arith_nonds(0x28, dst, src, VEX_SIMD_66);
-emit_byte(0x28);
-emit_byte(0xC0 | encode);
 }
 void Assembler::movaps(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_NONE);
+emit_simd_arith_nonds(0x28, dst, src, VEX_SIMD_NONE);
-emit_byte(0x28);
-emit_byte(0xC0 | encode);
 }
 void Assembler::movlhps(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
 int encode = simd_prefix_and_encode(dst, src, src, VEX_SIMD_NONE);
 emit_operand(src, dst);
 }
 void Assembler::movdqa(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66);
+emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_66);
-emit_byte(0x6F);
-emit_byte(0xC0 | encode);
 }
 void Assembler::movdqu(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-InstructionMark im(this);
+emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_F3);
-simd_prefix(dst, src, VEX_SIMD_F3);
-emit_byte(0x6F);
-emit_operand(dst, src);
 }
 void Assembler::movdqu(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_F3);
+emit_simd_arith_nonds(0x6F, dst, src, VEX_SIMD_F3);
-emit_byte(0x6F);
-emit_byte(0xC0 | encode);
 }
 void Assembler::movdqu(Address dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 InstructionMark im(this);
 // New cpus require to use movsd and movss to avoid partial register stall
 // when loading from memory. But for old Opteron use movlpd instead of movsd.
 // The selection is done in MacroAssembler::movdbl() and movflt().
 void Assembler::movlpd(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-InstructionMark im(this);
+emit_simd_arith(0x12, dst, src, VEX_SIMD_66);
-simd_prefix(dst, dst, src, VEX_SIMD_66);
-emit_byte(0x12);
-emit_operand(dst, src);
 }
 void Assembler::movq( MMXRegister dst, Address src ) {
 assert( VM_Version::supports_mmx(), "" );
 emit_byte(0x0F);
 emit_byte(0xC0 | encode);
 }
 void Assembler::movsd(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2);
+emit_simd_arith(0x10, dst, src, VEX_SIMD_F2);
-emit_byte(0x10);
-emit_byte(0xC0 | encode);
 }
 void Assembler::movsd(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-InstructionMark im(this);
+emit_simd_arith_nonds(0x10, dst, src, VEX_SIMD_F2);
-simd_prefix(dst, src, VEX_SIMD_F2);
-emit_byte(0x10);
-emit_operand(dst, src);
 }
 void Assembler::movsd(Address dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 InstructionMark im(this);
 emit_operand(src, dst);
 }
 void Assembler::movss(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3);
+emit_simd_arith(0x10, dst, src, VEX_SIMD_F3);
-emit_byte(0x10);
-emit_byte(0xC0 | encode);
 }
 void Assembler::movss(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-InstructionMark im(this);
+emit_simd_arith_nonds(0x10, dst, src, VEX_SIMD_F3);
-simd_prefix(dst, src, VEX_SIMD_F3);
-emit_byte(0x10);
-emit_operand(dst, src);
 }
 void Assembler::movss(Address dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
 InstructionMark im(this);
 emit_byte(0xE0 | encode);
 }
 void Assembler::mulsd(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-InstructionMark im(this);
+emit_simd_arith(0x59, dst, src, VEX_SIMD_F2);
-simd_prefix(dst, dst, src, VEX_SIMD_F2);
-emit_byte(0x59);
-emit_operand(dst, src);
 }
 void Assembler::mulsd(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2);
+emit_simd_arith(0x59, dst, src, VEX_SIMD_F2);
-emit_byte(0x59);
-emit_byte(0xC0 | encode);
 }
 void Assembler::mulss(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-InstructionMark im(this);
+emit_simd_arith(0x59, dst, src, VEX_SIMD_F3);
-simd_prefix(dst, dst, src, VEX_SIMD_F3);
-emit_byte(0x59);
-emit_operand(dst, src);
 }
 void Assembler::mulss(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3);
+emit_simd_arith(0x59, dst, src, VEX_SIMD_F3);
-emit_byte(0x59);
-emit_byte(0xC0 | encode);
 }
 void Assembler::negl(Register dst) {
 int encode = prefix_and_encode(dst->encoding());
 emit_byte(0xF7);
 }
 void Assembler::packuswb(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
-InstructionMark im(this);
+emit_simd_arith(0x67, dst, src, VEX_SIMD_66);
-simd_prefix(dst, dst, src, VEX_SIMD_66);
-emit_byte(0x67);
-emit_operand(dst, src);
 }
 void Assembler::packuswb(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66);
+emit_simd_arith(0x67, dst, src, VEX_SIMD_66);
-emit_byte(0x67);
-emit_byte(0xC0 | encode);
 }
 void Assembler::pcmpestri(XMMRegister dst, Address src, int imm8) {
 assert(VM_Version::supports_sse4_2(), "");
 InstructionMark im(this);
 emit_byte(imm8);
 }
 void Assembler::pcmpestri(XMMRegister dst, XMMRegister src, int imm8) {
 assert(VM_Version::supports_sse4_2(), "");
-int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66, VEX_OPCODE_0F_3A);
+int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_3A);
 emit_byte(0x61);
 emit_byte(0xC0 | encode);
 emit_byte(imm8);
 }
 emit_operand(dst, src);
 }
 void Assembler::pmovzxbw(XMMRegister dst, XMMRegister src) {
 assert(VM_Version::supports_sse4_1(), "");
-int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
+int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
 emit_byte(0x30);
 emit_byte(0xC0 | encode);
 }
 // generic
 void Assembler::prefix(Prefix p) {
 a_byte(p);
 }
-void Assembler::por(XMMRegister dst, XMMRegister src) {
-NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66);
-emit_byte(0xEB);
-emit_byte(0xC0 | encode);
-}
-void Assembler::por(XMMRegister dst, Address src) {
-NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
-InstructionMark im(this);
-simd_prefix(dst, dst, src, VEX_SIMD_66);
-emit_byte(0xEB);
-emit_operand(dst, src);
-}
 void Assembler::pshufd(XMMRegister dst, XMMRegister src, int mode) {
 assert(isByte(mode), "invalid value");
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66);
+emit_simd_arith_nonds(0x70, dst, src, VEX_SIMD_66);
-emit_byte(0x70);
-emit_byte(0xC0 | encode);
 emit_byte(mode & 0xFF);
 }
 void Assembler::pshufd(XMMRegister dst, Address src, int mode) {
 }
 void Assembler::pshuflw(XMMRegister dst, XMMRegister src, int mode) {
 assert(isByte(mode), "invalid value");
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_F2);
+emit_simd_arith_nonds(0x70, dst, src, VEX_SIMD_F2);
-emit_byte(0x70);
-emit_byte(0xC0 | encode);
 emit_byte(mode & 0xFF);
 }
 void Assembler::pshuflw(XMMRegister dst, Address src, int mode) {
 assert(isByte(mode), "invalid value");
 InstructionMark im(this);
 simd_prefix(dst, src, VEX_SIMD_F2);
 emit_byte(0x70);
 emit_operand(dst, src);
 emit_byte(mode & 0xFF);
-}
-void Assembler::psrlq(XMMRegister dst, int shift) {
-// Shift 64 bit value logically right by specified number of bits.
-// HMM Table D-1 says sse2 or mmx.
-// Do not confuse it with psrldq SSE2 instruction which
-// shifts 128 bit value in xmm register by number of bytes.
-NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66);
-emit_byte(0x73);
-emit_byte(0xC0 | encode);
-emit_byte(shift);
 }
 void Assembler::psrldq(XMMRegister dst, int shift) {
 // Shift 128 bit value in xmm register by number of bytes.
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 emit_operand(dst, src);
 }
 void Assembler::ptest(XMMRegister dst, XMMRegister src) {
 assert(VM_Version::supports_sse4_1(), "");
-int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
+int encode = simd_prefix_and_encode(dst, xnoreg, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
 emit_byte(0x17);
 emit_byte(0xC0 | encode);
 }
 void Assembler::punpcklbw(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
-InstructionMark im(this);
+emit_simd_arith(0x60, dst, src, VEX_SIMD_66);
-simd_prefix(dst, dst, src, VEX_SIMD_66);
-emit_byte(0x60);
-emit_operand(dst, src);
 }
 void Assembler::punpcklbw(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66);
+emit_simd_arith(0x60, dst, src, VEX_SIMD_66);
-emit_byte(0x60);
-emit_byte(0xC0 | encode);
 }
 void Assembler::punpckldq(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
 assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
-InstructionMark im(this);
+emit_simd_arith(0x62, dst, src, VEX_SIMD_66);
-simd_prefix(dst, dst, src, VEX_SIMD_66);
-emit_byte(0x62);
-emit_operand(dst, src);
 }
 void Assembler::punpckldq(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66);
+emit_simd_arith(0x62, dst, src, VEX_SIMD_66);
-emit_byte(0x62);
-emit_byte(0xC0 | encode);
 }
 void Assembler::punpcklqdq(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66);
+emit_simd_arith(0x6C, dst, src, VEX_SIMD_66);
-emit_byte(0x6C);
-emit_byte(0xC0 | encode);
 }
 void Assembler::push(int32_t imm32) {
 // in 64bits we push 64bits onto the stack but only
 // take a 32bit immediate
 prefix(src);
 emit_byte(0xFF);
 emit_operand(rsi, src);
 }
 #endif
-void Assembler::pxor(XMMRegister dst, Address src) {
-NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-assert((UseAVX > 0), "SSE mode requires address alignment 16 bytes");
-InstructionMark im(this);
-simd_prefix(dst, dst, src, VEX_SIMD_66);
-emit_byte(0xEF);
-emit_operand(dst, src);
-}
-void Assembler::pxor(XMMRegister dst, XMMRegister src) {
-NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66);
-emit_byte(0xEF);
-emit_byte(0xC0 | encode);
-}
 void Assembler::rcll(Register dst, int imm8) {
 assert(isShiftCount(imm8), "illegal shift count");
 int encode = prefix_and_encode(dst->encoding());
 if (imm8 == 1) {
 emit_byte(0xA5);
 }
 void Assembler::sqrtsd(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2);
+emit_simd_arith(0x51, dst, src, VEX_SIMD_F2);
-emit_byte(0x51);
-emit_byte(0xC0 | encode);
 }
 void Assembler::sqrtsd(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-InstructionMark im(this);
+emit_simd_arith(0x51, dst, src, VEX_SIMD_F2);
-simd_prefix(dst, dst, src, VEX_SIMD_F2);
-emit_byte(0x51);
-emit_operand(dst, src);
 }
 void Assembler::sqrtss(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3);
+emit_simd_arith(0x51, dst, src, VEX_SIMD_F3);
-emit_byte(0x51);
-emit_byte(0xC0 | encode);
 }
 void Assembler::sqrtss(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-InstructionMark im(this);
+emit_simd_arith(0x51, dst, src, VEX_SIMD_F3);
-simd_prefix(dst, dst, src, VEX_SIMD_F3);
-emit_byte(0x51);
-emit_operand(dst, src);
 }
 void Assembler::stmxcsr( Address dst) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
 InstructionMark im(this);
 emit_arith(0x2B, 0xC0, dst, src);
 }
 void Assembler::subsd(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F2);
+emit_simd_arith(0x5C, dst, src, VEX_SIMD_F2);
-emit_byte(0x5C);
-emit_byte(0xC0 | encode);
 }
 void Assembler::subsd(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-InstructionMark im(this);
+emit_simd_arith(0x5C, dst, src, VEX_SIMD_F2);
-simd_prefix(dst, dst, src, VEX_SIMD_F2);
-emit_byte(0x5C);
-emit_operand(dst, src);
 }
 void Assembler::subss(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_F3);
+emit_simd_arith(0x5C, dst, src, VEX_SIMD_F3);
-emit_byte(0x5C);
-emit_byte(0xC0 | encode);
 }
 void Assembler::subss(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-InstructionMark im(this);
+emit_simd_arith(0x5C, dst, src, VEX_SIMD_F3);
-simd_prefix(dst, dst, src, VEX_SIMD_F3);
-emit_byte(0x5C);
-emit_operand(dst, src);
 }
 void Assembler::testb(Register dst, int imm8) {
 NOT_LP64(assert(dst->has_byte_register(), "must have byte register"));
 (void) prefix_and_encode(dst->encoding(), true);
 emit_operand(dst, src);
 }
 void Assembler::ucomisd(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-InstructionMark im(this);
+emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_66);
-simd_prefix(dst, src, VEX_SIMD_66);
-emit_byte(0x2E);
-emit_operand(dst, src);
 }
 void Assembler::ucomisd(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_66);
+emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_66);
-emit_byte(0x2E);
-emit_byte(0xC0 | encode);
 }
 void Assembler::ucomiss(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-InstructionMark im(this);
+emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_NONE);
-simd_prefix(dst, src, VEX_SIMD_NONE);
-emit_byte(0x2E);
-emit_operand(dst, src);
 }
 void Assembler::ucomiss(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-int encode = simd_prefix_and_encode(dst, src, VEX_SIMD_NONE);
+emit_simd_arith_nonds(0x2E, dst, src, VEX_SIMD_NONE);
-emit_byte(0x2E);
-emit_byte(0xC0 | encode);
 }
 void Assembler::xaddl(Address dst, Register src) {
 InstructionMark im(this);
 void Assembler::xorl(Register dst, Register src) {
 (void) prefix_and_encode(dst->encoding(), src->encoding());
 emit_arith(0x33, 0xC0, dst, src);
 }
+// AVX 3-operands scalar float-point arithmetic instructions
+void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
+}
+void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
+}
+void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
+}
+void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
+}
+void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
+}
+void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
+}
+void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
+}
+void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
+}
+void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
+}
+void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
+}
+void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
+}
+void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
+}
+void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
+}
+void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F2, /* vector256 */ false);
+}
+void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
+}
+void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_F3, /* vector256 */ false);
+}
+//====================VECTOR ARITHMETIC=====================================
+// Float-point vector arithmetic
+void Assembler::addpd(XMMRegister dst, XMMRegister src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0x58, dst, src, VEX_SIMD_66);
+}
+void Assembler::addps(XMMRegister dst, XMMRegister src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0x58, dst, src, VEX_SIMD_NONE);
+}
+void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::vaddps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_NONE, vector256);
+}
+void Assembler::vaddpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::vaddps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x58, dst, nds, src, VEX_SIMD_NONE, vector256);
+}
+void Assembler::subpd(XMMRegister dst, XMMRegister src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0x5C, dst, src, VEX_SIMD_66);
+}
+void Assembler::subps(XMMRegister dst, XMMRegister src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0x5C, dst, src, VEX_SIMD_NONE);
+}
+void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::vsubps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_NONE, vector256);
+}
+void Assembler::vsubpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::vsubps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x5C, dst, nds, src, VEX_SIMD_NONE, vector256);
+}
+void Assembler::mulpd(XMMRegister dst, XMMRegister src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0x59, dst, src, VEX_SIMD_66);
+}
+void Assembler::mulps(XMMRegister dst, XMMRegister src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0x59, dst, src, VEX_SIMD_NONE);
+}
+void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::vmulps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_NONE, vector256);
+}
+void Assembler::vmulpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::vmulps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x59, dst, nds, src, VEX_SIMD_NONE, vector256);
+}
+void Assembler::divpd(XMMRegister dst, XMMRegister src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0x5E, dst, src, VEX_SIMD_66);
+}
+void Assembler::divps(XMMRegister dst, XMMRegister src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0x5E, dst, src, VEX_SIMD_NONE);
+}
+void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::vdivps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_NONE, vector256);
+}
+void Assembler::vdivpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::vdivps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x5E, dst, nds, src, VEX_SIMD_NONE, vector256);
+}
+void Assembler::andpd(XMMRegister dst, XMMRegister src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0x54, dst, src, VEX_SIMD_66);
+}
+void Assembler::andps(XMMRegister dst, XMMRegister src) {
+NOT_LP64(assert(VM_Version::supports_sse(), ""));
+emit_simd_arith(0x54, dst, src, VEX_SIMD_NONE);
+}
+void Assembler::andps(XMMRegister dst, Address src) {
+NOT_LP64(assert(VM_Version::supports_sse(), ""));
+emit_simd_arith(0x54, dst, src, VEX_SIMD_NONE);
+}
+void Assembler::andpd(XMMRegister dst, Address src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0x54, dst, src, VEX_SIMD_66);
+}
+void Assembler::vandpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::vandps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_NONE, vector256);
+}
+void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x54, dst, nds, src, VEX_SIMD_NONE, vector256);
+}
 void Assembler::xorpd(XMMRegister dst, XMMRegister src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66);
+emit_simd_arith(0x57, dst, src, VEX_SIMD_66);
-emit_byte(0x57);
+}
-emit_byte(0xC0 | encode);
+void Assembler::xorps(XMMRegister dst, XMMRegister src) {
+NOT_LP64(assert(VM_Version::supports_sse(), ""));
+emit_simd_arith(0x57, dst, src, VEX_SIMD_NONE);
 }
 void Assembler::xorpd(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse2(), ""));
-InstructionMark im(this);
+emit_simd_arith(0x57, dst, src, VEX_SIMD_66);
-simd_prefix(dst, dst, src, VEX_SIMD_66);
-emit_byte(0x57);
-emit_operand(dst, src);
-}
-void Assembler::xorps(XMMRegister dst, XMMRegister src) {
-NOT_LP64(assert(VM_Version::supports_sse(), ""));
-int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_NONE);
-emit_byte(0x57);
-emit_byte(0xC0 | encode);
 }
 void Assembler::xorps(XMMRegister dst, Address src) {
 NOT_LP64(assert(VM_Version::supports_sse(), ""));
-InstructionMark im(this);
+emit_simd_arith(0x57, dst, src, VEX_SIMD_NONE);
-simd_prefix(dst, dst, src, VEX_SIMD_NONE);
-emit_byte(0x57);
-emit_operand(dst, src);
-}
-// AVX 3-operands non destructive source instructions (encoded with VEX prefix)
-void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, Address src) {
-assert(VM_Version::supports_avx(), "");
-InstructionMark im(this);
-vex_prefix(dst, nds, src, VEX_SIMD_F2);
-emit_byte(0x58);
-emit_operand(dst, src);
-}
-void Assembler::vaddsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
-assert(VM_Version::supports_avx(), "");
-int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_F2);
-emit_byte(0x58);
-emit_byte(0xC0 | encode);
-}
-void Assembler::vaddss(XMMRegister dst, XMMRegister nds, Address src) {
-assert(VM_Version::supports_avx(), "");
-InstructionMark im(this);
-vex_prefix(dst, nds, src, VEX_SIMD_F3);
-emit_byte(0x58);
-emit_operand(dst, src);
-}
-void Assembler::vaddss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
-assert(VM_Version::supports_avx(), "");
-int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_F3);
-emit_byte(0x58);
-emit_byte(0xC0 | encode);
-}
-void Assembler::vandpd(XMMRegister dst, XMMRegister nds, Address src) {
-assert(VM_Version::supports_avx(), "");
-InstructionMark im(this);
-vex_prefix(dst, nds, src, VEX_SIMD_66); // 128-bit vector
-emit_byte(0x54);
-emit_operand(dst, src);
-}
-void Assembler::vandps(XMMRegister dst, XMMRegister nds, Address src) {
-assert(VM_Version::supports_avx(), "");
-InstructionMark im(this);
-vex_prefix(dst, nds, src, VEX_SIMD_NONE); // 128-bit vector
-emit_byte(0x54);
-emit_operand(dst, src);
-}
-void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, Address src) {
-assert(VM_Version::supports_avx(), "");
-InstructionMark im(this);
-vex_prefix(dst, nds, src, VEX_SIMD_F2);
-emit_byte(0x5E);
-emit_operand(dst, src);
-}
-void Assembler::vdivsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
-assert(VM_Version::supports_avx(), "");
-int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_F2);
-emit_byte(0x5E);
-emit_byte(0xC0 | encode);
-}
-void Assembler::vdivss(XMMRegister dst, XMMRegister nds, Address src) {
-assert(VM_Version::supports_avx(), "");
-InstructionMark im(this);
-vex_prefix(dst, nds, src, VEX_SIMD_F3);
-emit_byte(0x5E);
-emit_operand(dst, src);
-}
-void Assembler::vdivss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
-assert(VM_Version::supports_avx(), "");
-int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_F3);
-emit_byte(0x5E);
-emit_byte(0xC0 | encode);
-}
-void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, Address src) {
-assert(VM_Version::supports_avx(), "");
-InstructionMark im(this);
-vex_prefix(dst, nds, src, VEX_SIMD_F2);
-emit_byte(0x59);
-emit_operand(dst, src);
-}
-void Assembler::vmulsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
-assert(VM_Version::supports_avx(), "");
-int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_F2);
-emit_byte(0x59);
-emit_byte(0xC0 | encode);
-}
-void Assembler::vmulss(XMMRegister dst, XMMRegister nds, Address src) {
-InstructionMark im(this);
-vex_prefix(dst, nds, src, VEX_SIMD_F3);
-emit_byte(0x59);
-emit_operand(dst, src);
-}
-void Assembler::vmulss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
-assert(VM_Version::supports_avx(), "");
-int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_F3);
-emit_byte(0x59);
-emit_byte(0xC0 | encode);
-}
-void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, Address src) {
-assert(VM_Version::supports_avx(), "");
-InstructionMark im(this);
-vex_prefix(dst, nds, src, VEX_SIMD_F2);
-emit_byte(0x5C);
-emit_operand(dst, src);
-}
-void Assembler::vsubsd(XMMRegister dst, XMMRegister nds, XMMRegister src) {
-assert(VM_Version::supports_avx(), "");
-int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_F2);
-emit_byte(0x5C);
-emit_byte(0xC0 | encode);
-}
-void Assembler::vsubss(XMMRegister dst, XMMRegister nds, Address src) {
-assert(VM_Version::supports_avx(), "");
-InstructionMark im(this);
-vex_prefix(dst, nds, src, VEX_SIMD_F3);
-emit_byte(0x5C);
-emit_operand(dst, src);
-}
-void Assembler::vsubss(XMMRegister dst, XMMRegister nds, XMMRegister src) {
-assert(VM_Version::supports_avx(), "");
-int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_F3);
-emit_byte(0x5C);
-emit_byte(0xC0 | encode);
-}
-void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src) {
-assert(VM_Version::supports_avx(), "");
-InstructionMark im(this);
-vex_prefix(dst, nds, src, VEX_SIMD_66); // 128-bit vector
-emit_byte(0x57);
-emit_operand(dst, src);
 }
 void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
 assert(VM_Version::supports_avx(), "");
-int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector256);
+emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_66, vector256);
-emit_byte(0x57);
-emit_byte(0xC0 | encode);
-}
-void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src) {
-assert(VM_Version::supports_avx(), "");
-InstructionMark im(this);
-vex_prefix(dst, nds, src, VEX_SIMD_NONE); // 128-bit vector
-emit_byte(0x57);
-emit_operand(dst, src);
 }
 void Assembler::vxorps(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
 assert(VM_Version::supports_avx(), "");
-int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_NONE, vector256);
+emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_NONE, vector256);
-emit_byte(0x57);
+}
+void Assembler::vxorpd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::vxorps(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
+assert(VM_Version::supports_avx(), "");
+emit_vex_arith(0x57, dst, nds, src, VEX_SIMD_NONE, vector256);
+}
+// Integer vector arithmetic
+void Assembler::paddb(XMMRegister dst, XMMRegister src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0xFC, dst, src, VEX_SIMD_66);
+}
+void Assembler::paddw(XMMRegister dst, XMMRegister src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0xFD, dst, src, VEX_SIMD_66);
+}
+void Assembler::paddd(XMMRegister dst, XMMRegister src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0xFE, dst, src, VEX_SIMD_66);
+}
+void Assembler::paddq(XMMRegister dst, XMMRegister src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0xD4, dst, src, VEX_SIMD_66);
+}
+void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xFC, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xFD, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xFE, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xD4, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::vpaddb(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xFC, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::vpaddw(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xFD, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::vpaddd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xFE, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::vpaddq(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xD4, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::psubb(XMMRegister dst, XMMRegister src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0xF8, dst, src, VEX_SIMD_66);
+}
+void Assembler::psubw(XMMRegister dst, XMMRegister src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0xF9, dst, src, VEX_SIMD_66);
+}
+void Assembler::psubd(XMMRegister dst, XMMRegister src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0xFA, dst, src, VEX_SIMD_66);
+}
+void Assembler::psubq(XMMRegister dst, XMMRegister src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0xFB, dst, src, VEX_SIMD_66);
+}
+void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xF8, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xF9, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xFA, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xFB, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::vpsubb(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xF8, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::vpsubw(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xF9, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::vpsubd(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xFA, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::vpsubq(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xFB, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::pmullw(XMMRegister dst, XMMRegister src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0xD5, dst, src, VEX_SIMD_66);
+}
+void Assembler::pmulld(XMMRegister dst, XMMRegister src) {
+assert(VM_Version::supports_sse4_1(), "");
+int encode = simd_prefix_and_encode(dst, dst, src, VEX_SIMD_66, VEX_OPCODE_0F_38);
+emit_byte(0x40);
 emit_byte(0xC0 | encode);
 }
+void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xD5, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector256, VEX_OPCODE_0F_38);
+emit_byte(0x40);
+emit_byte(0xC0 | encode);
+}
+void Assembler::vpmullw(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xD5, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::vpmulld(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+InstructionMark im(this);
+int dst_enc = dst->encoding();
+int nds_enc = nds->is_valid() ? nds->encoding() : 0;
+vex_prefix(src, nds_enc, dst_enc, VEX_SIMD_66, VEX_OPCODE_0F_38, false, vector256);
+emit_byte(0x40);
+emit_operand(dst, src);
+}
+// Shift packed integers left by specified number of bits.
+void Assembler::psllw(XMMRegister dst, int shift) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+// XMM6 is for /6 encoding: 66 0F 71 /6 ib
+int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66);
+emit_byte(0x71);
+emit_byte(0xC0 | encode);
+emit_byte(shift & 0xFF);
+}
+void Assembler::pslld(XMMRegister dst, int shift) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+// XMM6 is for /6 encoding: 66 0F 72 /6 ib
+int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66);
+emit_byte(0x72);
+emit_byte(0xC0 | encode);
+emit_byte(shift & 0xFF);
+}
+void Assembler::psllq(XMMRegister dst, int shift) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+// XMM6 is for /6 encoding: 66 0F 73 /6 ib
+int encode = simd_prefix_and_encode(xmm6, dst, dst, VEX_SIMD_66);
+emit_byte(0x73);
+emit_byte(0xC0 | encode);
+emit_byte(shift & 0xFF);
+}
+void Assembler::psllw(XMMRegister dst, XMMRegister shift) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0xF1, dst, shift, VEX_SIMD_66);
+}
+void Assembler::pslld(XMMRegister dst, XMMRegister shift) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0xF2, dst, shift, VEX_SIMD_66);
+}
+void Assembler::psllq(XMMRegister dst, XMMRegister shift) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0xF3, dst, shift, VEX_SIMD_66);
+}
+void Assembler::vpsllw(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+// XMM6 is for /6 encoding: 66 0F 71 /6 ib
+emit_vex_arith(0x71, xmm6, dst, src, VEX_SIMD_66, vector256);
+emit_byte(shift & 0xFF);
+}
+void Assembler::vpslld(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+// XMM6 is for /6 encoding: 66 0F 72 /6 ib
+emit_vex_arith(0x72, xmm6, dst, src, VEX_SIMD_66, vector256);
+emit_byte(shift & 0xFF);
+}
+void Assembler::vpsllq(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+// XMM6 is for /6 encoding: 66 0F 73 /6 ib
+emit_vex_arith(0x73, xmm6, dst, src, VEX_SIMD_66, vector256);
+emit_byte(shift & 0xFF);
+}
+void Assembler::vpsllw(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xF1, dst, src, shift, VEX_SIMD_66, vector256);
+}
+void Assembler::vpslld(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xF2, dst, src, shift, VEX_SIMD_66, vector256);
+}
+void Assembler::vpsllq(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xF3, dst, src, shift, VEX_SIMD_66, vector256);
+}
+// Shift packed integers logically right by specified number of bits.
+void Assembler::psrlw(XMMRegister dst, int shift) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+// XMM2 is for /2 encoding: 66 0F 71 /2 ib
+int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66);
+emit_byte(0x71);
+emit_byte(0xC0 | encode);
+emit_byte(shift & 0xFF);
+}
+void Assembler::psrld(XMMRegister dst, int shift) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+// XMM2 is for /2 encoding: 66 0F 72 /2 ib
+int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66);
+emit_byte(0x72);
+emit_byte(0xC0 | encode);
+emit_byte(shift & 0xFF);
+}
+void Assembler::psrlq(XMMRegister dst, int shift) {
+// Do not confuse it with psrldq SSE2 instruction which
+// shifts 128 bit value in xmm register by number of bytes.
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+// XMM2 is for /2 encoding: 66 0F 73 /2 ib
+int encode = simd_prefix_and_encode(xmm2, dst, dst, VEX_SIMD_66);
+emit_byte(0x73);
+emit_byte(0xC0 | encode);
+emit_byte(shift & 0xFF);
+}
+void Assembler::psrlw(XMMRegister dst, XMMRegister shift) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0xD1, dst, shift, VEX_SIMD_66);
+}
+void Assembler::psrld(XMMRegister dst, XMMRegister shift) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0xD2, dst, shift, VEX_SIMD_66);
+}
+void Assembler::psrlq(XMMRegister dst, XMMRegister shift) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0xD3, dst, shift, VEX_SIMD_66);
+}
+void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+// XMM2 is for /2 encoding: 66 0F 73 /2 ib
+emit_vex_arith(0x71, xmm2, dst, src, VEX_SIMD_66, vector256);
+emit_byte(shift & 0xFF);
+}
+void Assembler::vpsrld(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+// XMM2 is for /2 encoding: 66 0F 73 /2 ib
+emit_vex_arith(0x72, xmm2, dst, src, VEX_SIMD_66, vector256);
+emit_byte(shift & 0xFF);
+}
+void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+// XMM2 is for /2 encoding: 66 0F 73 /2 ib
+emit_vex_arith(0x73, xmm2, dst, src, VEX_SIMD_66, vector256);
+emit_byte(shift & 0xFF);
+}
+void Assembler::vpsrlw(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xD1, dst, src, shift, VEX_SIMD_66, vector256);
+}
+void Assembler::vpsrld(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xD2, dst, src, shift, VEX_SIMD_66, vector256);
+}
+void Assembler::vpsrlq(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xD3, dst, src, shift, VEX_SIMD_66, vector256);
+}
+// Shift packed integers arithmetically right by specified number of bits.
+void Assembler::psraw(XMMRegister dst, int shift) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+// XMM4 is for /4 encoding: 66 0F 71 /4 ib
+int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66);
+emit_byte(0x71);
+emit_byte(0xC0 | encode);
+emit_byte(shift & 0xFF);
+}
+void Assembler::psrad(XMMRegister dst, int shift) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+// XMM4 is for /4 encoding: 66 0F 72 /4 ib
+int encode = simd_prefix_and_encode(xmm4, dst, dst, VEX_SIMD_66);
+emit_byte(0x72);
+emit_byte(0xC0 | encode);
+emit_byte(shift & 0xFF);
+}
+void Assembler::psraw(XMMRegister dst, XMMRegister shift) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0xE1, dst, shift, VEX_SIMD_66);
+}
+void Assembler::psrad(XMMRegister dst, XMMRegister shift) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0xE2, dst, shift, VEX_SIMD_66);
+}
+void Assembler::vpsraw(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+// XMM4 is for /4 encoding: 66 0F 71 /4 ib
+emit_vex_arith(0x71, xmm4, dst, src, VEX_SIMD_66, vector256);
+emit_byte(shift & 0xFF);
+}
+void Assembler::vpsrad(XMMRegister dst, XMMRegister src, int shift, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+// XMM4 is for /4 encoding: 66 0F 71 /4 ib
+emit_vex_arith(0x72, xmm4, dst, src, VEX_SIMD_66, vector256);
+emit_byte(shift & 0xFF);
+}
+void Assembler::vpsraw(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xE1, dst, src, shift, VEX_SIMD_66, vector256);
+}
+void Assembler::vpsrad(XMMRegister dst, XMMRegister src, XMMRegister shift, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xE2, dst, src, shift, VEX_SIMD_66, vector256);
+}
+// AND packed integers
+void Assembler::pand(XMMRegister dst, XMMRegister src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0xDB, dst, src, VEX_SIMD_66);
+}
+void Assembler::vpand(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xDB, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::vpand(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xDB, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::por(XMMRegister dst, XMMRegister src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0xEB, dst, src, VEX_SIMD_66);
+}
+void Assembler::vpor(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xEB, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::vpor(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xEB, dst, nds, src, VEX_SIMD_66, vector256);
+}
+void Assembler::pxor(XMMRegister dst, XMMRegister src) {
+NOT_LP64(assert(VM_Version::supports_sse2(), ""));
+emit_simd_arith(0xEF, dst, src, VEX_SIMD_66);
+}
 void Assembler::vpxor(XMMRegister dst, XMMRegister nds, XMMRegister src, bool vector256) {
-assert(VM_Version::supports_avx2() || (!vector256) && VM_Version::supports_avx(), "");
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
-int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector256);
+emit_vex_arith(0xEF, dst, nds, src, VEX_SIMD_66, vector256);
-emit_byte(0xEF);
+}
-emit_byte(0xC0 | encode);
-}
+void Assembler::vpxor(XMMRegister dst, XMMRegister nds, Address src, bool vector256) {
+assert(VM_Version::supports_avx() && !vector256 || VM_Version::supports_avx2(), "256 bit integer vectors requires AVX2");
+emit_vex_arith(0xEF, dst, nds, src, VEX_SIMD_66, vector256);
+}
 void Assembler::vinsertf128h(XMMRegister dst, XMMRegister nds, XMMRegister src) {
 assert(VM_Version::supports_avx(), "");
 bool vector256 = true;
 int encode = vex_prefix_and_encode(dst, nds, src, VEX_SIMD_66, vector256, VEX_OPCODE_0F_3A);
 return vex_prefix_and_encode(dst_enc, nds_enc, src_enc, pre, opc, rex_w, vector256);
 } else {
 assert((nds == dst) || (nds == src) || (nds == xnoreg), "wrong sse encoding");
 return rex_prefix_and_encode(dst_enc, src_enc, pre, opc, rex_w);
 }
+}
+void Assembler::emit_simd_arith(int opcode, XMMRegister dst, Address src, VexSimdPrefix pre) {
+InstructionMark im(this);
+simd_prefix(dst, dst, src, pre);
+emit_byte(opcode);
+emit_operand(dst, src);
+}
+void Assembler::emit_simd_arith(int opcode, XMMRegister dst, XMMRegister src, VexSimdPrefix pre) {
+int encode = simd_prefix_and_encode(dst, dst, src, pre);
+emit_byte(opcode);
+emit_byte(0xC0 | encode);
+}
+// Versions with no second source register (non-destructive source).
+void Assembler::emit_simd_arith_nonds(int opcode, XMMRegister dst, Address src, VexSimdPrefix pre) {
+InstructionMark im(this);
+simd_prefix(dst, xnoreg, src, pre);
+emit_byte(opcode);
+emit_operand(dst, src);
+}
+void Assembler::emit_simd_arith_nonds(int opcode, XMMRegister dst, XMMRegister src, VexSimdPrefix pre) {
+int encode = simd_prefix_and_encode(dst, xnoreg, src, pre);
+emit_byte(opcode);
+emit_byte(0xC0 | encode);
+}
+// 3-operands AVX instructions
+void Assembler::emit_vex_arith(int opcode, XMMRegister dst, XMMRegister nds,
+Address src, VexSimdPrefix pre, bool vector256) {
+InstructionMark im(this);
+vex_prefix(dst, nds, src, pre, vector256);
+emit_byte(opcode);
+emit_operand(dst, src);
+}
+void Assembler::emit_vex_arith(int opcode, XMMRegister dst, XMMRegister nds,
+XMMRegister src, VexSimdPrefix pre, bool vector256) {
+int encode = vex_prefix_and_encode(dst, nds, src, pre, vector256);
+emit_byte(opcode);
+emit_byte(0xC0 | encode);
 }
 #ifndef _LP64
 void Assembler::incl(Register dst) {
 lea(rscratch1, src);
 vaddss(dst, nds, Address(rscratch1, 0));
 }
 }
-void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
+void MacroAssembler::vandpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, bool vector256) {
 if (reachable(src)) {
-vandpd(dst, nds, as_Address(src));
+vandpd(dst, nds, as_Address(src), vector256);
 } else {
 lea(rscratch1, src);
-vandpd(dst, nds, Address(rscratch1, 0));
+vandpd(dst, nds, Address(rscratch1, 0), vector256);
 }
 }
-void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
+void MacroAssembler::vandps(XMMRegister dst, XMMRegister nds, AddressLiteral src, bool vector256) {
 if (reachable(src)) {
-vandps(dst, nds, as_Address(src));
+vandps(dst, nds, as_Address(src), vector256);
 } else {
 lea(rscratch1, src);
-vandps(dst, nds, Address(rscratch1, 0));
+vandps(dst, nds, Address(rscratch1, 0), vector256);
 }
 }
 void MacroAssembler::vdivsd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
 if (reachable(src)) {
 lea(rscratch1, src);
 vsubss(dst, nds, Address(rscratch1, 0));
 }
 }
-void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
+void MacroAssembler::vxorpd(XMMRegister dst, XMMRegister nds, AddressLiteral src, bool vector256) {
 if (reachable(src)) {
-vxorpd(dst, nds, as_Address(src));
+vxorpd(dst, nds, as_Address(src), vector256);
 } else {
 lea(rscratch1, src);
-vxorpd(dst, nds, Address(rscratch1, 0));
+vxorpd(dst, nds, Address(rscratch1, 0), vector256);
 }
 }
-void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src) {
+void MacroAssembler::vxorps(XMMRegister dst, XMMRegister nds, AddressLiteral src, bool vector256) {
 if (reachable(src)) {
-vxorps(dst, nds, as_Address(src));
+vxorps(dst, nds, as_Address(src), vector256);
 } else {
 lea(rscratch1, src);
-vxorps(dst, nds, Address(rscratch1, 0));
+vxorps(dst, nds, Address(rscratch1, 0), vector256);
 }
 }
 //////////////////////////////////////////////////////////////////////////////////

Mercurial > hg > truffle

comparison src/cpu/x86/vm/assembler_x86.cpp @ 6614:006050192a5a