truffle: src/cpu/x86/vm/x86

comparison src/cpu/x86/vm/x86_64.ad @ 4970:33df1aeaebbf

Merge with http://hg.openjdk.java.net/hsx/hsx24/hotspot/

author	Thomas Wuerthinger <thomas.wuerthinger@oracle.com>
date	Mon, 27 Feb 2012 13:10:13 +0100
parents	9b8ce46870df
children	61b82be3b1ff

comparison

equal deleted inserted replaced

-:2cfb7fb2dce7
+:33df1aeaebbf
 //
-// Copyright (c) 2003, 2011, Oracle and/or its affiliates. All rights reserved.
+// Copyright (c) 2003, 2012, Oracle and/or its affiliates. All rights reserved.
 // DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 //
 // This code is free software; you can redistribute it and/or modify it
 // under the terms of the GNU General Public License version 2 only, as
 // published by the Free Software Foundation.
 #define   RELOC_DISP32   Assembler::disp32_operand
 #define __ _masm.
 static int preserve_SP_size() {
-return LP64_ONLY(1 +) 2;  // [rex,] op, rm(reg/reg)
+return 3;  // rex.w, op, rm(reg/reg)
 }
 // !!!!! Special hack to get all types of calls to specify the byte offset
 //       from the start of the call to the point where the return address
 //       will point.
 int CallDynamicJavaDirectNode::compute_padding(int current_offset) const
 {
 current_offset += 11; // skip movq instruction + call opcode byte
 return round_to(current_offset, alignment_required()) - current_offset;
 }
-#ifndef PRODUCT
-void MachBreakpointNode::format(PhaseRegAlloc*, outputStream* st) const
-{
-st->print("INT3");
-}
-#endif
 // EMIT_RM()
 void emit_rm(CodeBuffer &cbuf, int f1, int f2, int f3) {
 unsigned char c = (unsigned char) ((f1 << 6) | (f2 << 3) | f3);
 cbuf.insts()->emit_int8(c);
 }
 }
 }
 }
-void encode_copy(CodeBuffer &cbuf, int dstenc, int srcenc)
-{
-if (dstenc != srcenc) {
-if (dstenc < 8) {
-if (srcenc >= 8) {
-emit_opcode(cbuf, Assembler::REX_B);
-srcenc -= 8;
-}
-} else {
-if (srcenc < 8) {
-emit_opcode(cbuf, Assembler::REX_R);
-} else {
-emit_opcode(cbuf, Assembler::REX_RB);
-srcenc -= 8;
-}
-dstenc -= 8;
-}
-emit_opcode(cbuf, 0x8B);
-emit_rm(cbuf, 0x3, dstenc, srcenc);
-}
-}
-void encode_CopyXD( CodeBuffer &cbuf, int dst_encoding, int src_encoding ) {
-if( dst_encoding == src_encoding ) {
-// reg-reg copy, use an empty encoding
-} else {
-MacroAssembler _masm(&cbuf);
-__ movdqa(as_XMMRegister(dst_encoding), as_XMMRegister(src_encoding));
-}
-}
 // This could be in MacroAssembler but it's fairly C2 specific
 void emit_cmpfp_fixup(MacroAssembler& _masm) {
 Label exit;
 __ jccb(Assembler::noParity, exit);
 __ pushf();
+//
+// comiss/ucomiss instructions set ZF,PF,CF flags and
+// zero OF,AF,SF for NaN values.
+// Fixup flags by zeroing ZF,PF so that compare of NaN
+// values returns 'less than' result (CF is set).
+// Leave the rest of flags unchanged.
+//
+//    7 6 5 4 3 2 1 0
+//   |S|Z|r|A|r|P|r|C|  (r - reserved bit)
+//    0 0 1 0 1 0 1 1   (0x2B)
+//
 __ andq(Address(rsp, 0), 0xffffff2b);
 __ popf();
 __ bind(exit);
-__ nop(); // (target for branch to avoid branch to branch)
+}
+void emit_cmpfp3(MacroAssembler& _masm, Register dst) {
+Label done;
+__ movl(dst, -1);
+__ jcc(Assembler::parity, done);
+__ jcc(Assembler::below, done);
+__ setb(Assembler::notEqual, dst);
+__ movzbl(dst, dst);
+__ bind(done);
 }
 //=============================================================================
 const RegMask& MachConstantBaseNode::_out_RegMask = RegMask::Empty;
 #endif
 //=============================================================================
 #ifndef PRODUCT
-void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const
+void MachPrologNode::format(PhaseRegAlloc* ra_, outputStream* st) const {
-{
 Compile* C = ra_->C;
 int framesize = C->frame_slots() << LogBytesPerInt;
 assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
-// Remove wordSize for return adr already pushed
+// Remove wordSize for return addr which is already pushed.
-// and another for the RBP we are going to save
+framesize -= wordSize;
-framesize -= 2*wordSize;
-bool need_nop = true;
-// Calls to C2R adapters often do not accept exceptional returns.
-// We require that their callers must bang for them.  But be
-// careful, because some VM calls (such as call site linkage) can
-// use several kilobytes of stack.  But the stack safety zone should
-// account for that.  See bugs 4446381, 4468289, 4497237.
 if (C->need_stack_bang(framesize)) {
-st->print_cr("# stack bang"); st->print("\t");
+framesize -= wordSize;
-need_nop = false;
+st->print("# stack bang");
+st->print("\n\t");
+st->print("pushq   rbp\t# Save rbp");
+if (framesize) {
+st->print("\n\t");
+st->print("subq    rsp, #%d\t# Create frame",framesize);
+}
+} else {
+st->print("subq    rsp, #%d\t# Create frame",framesize);
+st->print("\n\t");
+framesize -= wordSize;
+st->print("movq    [rsp + #%d], rbp\t# Save rbp",framesize);
 }
-st->print_cr("pushq   rbp"); st->print("\t");
 if (VerifyStackAtCalls) {
-// Majik cookie to verify stack depth
+st->print("\n\t");
-st->print_cr("pushq   0xffffffffbadb100d"
+framesize -= wordSize;
-"\t# Majik cookie for stack depth check");
+st->print("movq    [rsp + #%d], 0xbadb100d\t# Majik cookie for stack depth check",framesize);
-st->print("\t");
+#ifdef ASSERT
-framesize -= wordSize; // Remove 2 for cookie
+st->print("\n\t");
-need_nop = false;
+st->print("# stack alignment check");
+#endif
 }
+st->cr();
-if (framesize) {
-st->print("subq    rsp, #%d\t# Create frame", framesize);
-if (framesize < 0x80 && need_nop) {
-st->print("\n\tnop\t# nop for patch_verified_entry");
-}
-}
 }
 #endif
-void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const
+void MachPrologNode::emit(CodeBuffer &cbuf, PhaseRegAlloc *ra_) const {
-{
 Compile* C = ra_->C;
+MacroAssembler _masm(&cbuf);
-// WARNING: Initial instruction MUST be 5 bytes or longer so that
-// NativeJump::patch_verified_entry will be able to patch out the entry
-// code safely. The fldcw is ok at 6 bytes, the push to verify stack
-// depth is ok at 5 bytes, the frame allocation can be either 3 or
-// 6 bytes. So if we don't do the fldcw or the push then we must
-// use the 6 byte frame allocation even if we have no frame. :-(
-// If method sets FPU control word do it now
 int framesize = C->frame_slots() << LogBytesPerInt;
-assert((framesize & (StackAlignmentInBytes-1)) == 0, "frame size not aligned");
-// Remove wordSize for return adr already pushed
+__ verified_entry(framesize, C->need_stack_bang(framesize), false);
-// and another for the RBP we are going to save
-framesize -= 2*wordSize;
-bool need_nop = true;
-// Calls to C2R adapters often do not accept exceptional returns.
-// We require that their callers must bang for them.  But be
-// careful, because some VM calls (such as call site linkage) can
-// use several kilobytes of stack.  But the stack safety zone should
-// account for that.  See bugs 4446381, 4468289, 4497237.
-if (C->need_stack_bang(framesize)) {
-MacroAssembler masm(&cbuf);
-masm.generate_stack_overflow_check(framesize);
-need_nop = false;
-}
-// We always push rbp so that on return to interpreter rbp will be
-// restored correctly and we can correct the stack.
-emit_opcode(cbuf, 0x50 | RBP_enc);
-if (VerifyStackAtCalls) {
-// Majik cookie to verify stack depth
-emit_opcode(cbuf, 0x68); // pushq (sign-extended) 0xbadb100d
-emit_d32(cbuf, 0xbadb100d);
-framesize -= wordSize; // Remove 2 for cookie
-need_nop = false;
-}
-if (framesize) {
-emit_opcode(cbuf, Assembler::REX_W);
-if (framesize < 0x80) {
-emit_opcode(cbuf, 0x83);   // sub  SP,#framesize
-emit_rm(cbuf, 0x3, 0x05, RSP_enc);
-emit_d8(cbuf, framesize);
-if (need_nop) {
-emit_opcode(cbuf, 0x90); // nop
-}
-} else {
-emit_opcode(cbuf, 0x81);   // sub  SP,#framesize
-emit_rm(cbuf, 0x3, 0x05, RSP_enc);
-emit_d32(cbuf, framesize);
-}
-}
 C->set_frame_complete(cbuf.insts_size());
-#ifdef ASSERT
-if (VerifyStackAtCalls) {
-Label L;
-MacroAssembler masm(&cbuf);
-masm.push(rax);
-masm.mov(rax, rsp);
-masm.andptr(rax, StackAlignmentInBytes-1);
-masm.cmpptr(rax, StackAlignmentInBytes-wordSize);
-masm.pop(rax);
-masm.jcc(Assembler::equal, L);
-masm.stop("Stack is not properly aligned!");
-masm.bind(L);
-}
-#endif
 if (C->has_mach_constant_base_node()) {
 // NOTE: We set the table base offset here because users might be
 // emitted before MachConstantBaseNode.
 Compile::ConstantTable& constant_table = C->constant_table();
 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 // 64-bit
 int offset = ra_->reg2offset(src_first);
 if (cbuf) {
-emit_opcode(*cbuf, UseXmmLoadAndClearUpper ? 0xF2 : 0x66);
+MacroAssembler _masm(cbuf);
-if (Matcher::_regEncode[dst_first] >= 8) {
+__ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
-emit_opcode(*cbuf, Assembler::REX_R);
-}
-emit_opcode(*cbuf, 0x0F);
-emit_opcode(*cbuf, UseXmmLoadAndClearUpper ? 0x10 : 0x12);
-encode_RegMem(*cbuf,
-Matcher::_regEncode[dst_first],
-RSP_enc, 0x4, 0, offset,
-false);
 #ifndef PRODUCT
 } else if (!do_size) {
 st->print("%s  %s, [rsp + #%d]\t# spill",
 UseXmmLoadAndClearUpper ? "movsd " : "movlpd",
 Matcher::regName[dst_first],
 offset);
 #endif
 }
 return
 ((offset == 0) ? 0 : (offset < 0x80 ? 1 : 4)) +
-((Matcher::_regEncode[dst_first] < 8)
+((Matcher::_regEncode[dst_first] >= 8)
-? 5
+? 6
-: 6); // REX
+: (5 + ((UseAVX>0)?1:0))); // REX
 } else {
 // 32-bit
 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 int offset = ra_->reg2offset(src_first);
 if (cbuf) {
-emit_opcode(*cbuf, 0xF3);
+MacroAssembler _masm(cbuf);
-if (Matcher::_regEncode[dst_first] >= 8) {
+__ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), Address(rsp, offset));
-emit_opcode(*cbuf, Assembler::REX_R);
-}
-emit_opcode(*cbuf, 0x0F);
-emit_opcode(*cbuf, 0x10);
-encode_RegMem(*cbuf,
-Matcher::_regEncode[dst_first],
-RSP_enc, 0x4, 0, offset,
-false);
 #ifndef PRODUCT
 } else if (!do_size) {
 st->print("movss   %s, [rsp + #%d]\t# spill",
 Matcher::regName[dst_first],
 offset);
 #endif
 }
 return
 ((offset == 0) ? 0 : (offset < 0x80 ? 1 : 4)) +
-((Matcher::_regEncode[dst_first] < 8)
+((Matcher::_regEncode[dst_first] >= 8)
-? 5
+? 6
-: 6); // REX
+: (5 + ((UseAVX>0)?1:0))); // REX
 }
 }
 } else if (src_first_rc == rc_int) {
 // gpr ->
 if (dst_first_rc == rc_stack) {
 // gpr -> xmm
 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 // 64-bit
 if (cbuf) {
-emit_opcode(*cbuf, 0x66);
+MacroAssembler _masm(cbuf);
-if (Matcher::_regEncode[dst_first] < 8) {
+__ movdq( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
-if (Matcher::_regEncode[src_first] < 8) {
-emit_opcode(*cbuf, Assembler::REX_W);
-} else {
-emit_opcode(*cbuf, Assembler::REX_WB);
-}
-} else {
-if (Matcher::_regEncode[src_first] < 8) {
-emit_opcode(*cbuf, Assembler::REX_WR);
-} else {
-emit_opcode(*cbuf, Assembler::REX_WRB);
-}
-}
-emit_opcode(*cbuf, 0x0F);
-emit_opcode(*cbuf, 0x6E);
-emit_rm(*cbuf, 0x3,
-Matcher::_regEncode[dst_first] & 7,
-Matcher::_regEncode[src_first] & 7);
 #ifndef PRODUCT
 } else if (!do_size) {
 st->print("movdq   %s, %s\t# spill",
 Matcher::regName[dst_first],
 Matcher::regName[src_first]);
 } else {
 // 32-bit
 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 if (cbuf) {
-emit_opcode(*cbuf, 0x66);
+MacroAssembler _masm(cbuf);
-if (Matcher::_regEncode[dst_first] < 8) {
+__ movdl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_Register(Matcher::_regEncode[src_first]));
-if (Matcher::_regEncode[src_first] >= 8) {
-emit_opcode(*cbuf, Assembler::REX_B);
-}
-} else {
-if (Matcher::_regEncode[src_first] < 8) {
-emit_opcode(*cbuf, Assembler::REX_R);
-} else {
-emit_opcode(*cbuf, Assembler::REX_RB);
-}
-}
-emit_opcode(*cbuf, 0x0F);
-emit_opcode(*cbuf, 0x6E);
-emit_rm(*cbuf, 0x3,
-Matcher::_regEncode[dst_first] & 7,
-Matcher::_regEncode[src_first] & 7);
 #ifndef PRODUCT
 } else if (!do_size) {
 st->print("movdl   %s, %s\t# spill",
 Matcher::regName[dst_first],
 Matcher::regName[src_first]);
 #endif
 }
 return
-(Matcher::_regEncode[src_first] < 8 && Matcher::_regEncode[dst_first] < 8)
+(Matcher::_regEncode[src_first] >= 8 || Matcher::_regEncode[dst_first] >= 8)
-? 4
+? 5
-: 5; // REX
+: (4 + ((UseAVX>0)?1:0)); // REX
 }
 }
 } else if (src_first_rc == rc_float) {
 // xmm ->
 if (dst_first_rc == rc_stack) {
 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 // 64-bit
 int offset = ra_->reg2offset(dst_first);
 if (cbuf) {
-emit_opcode(*cbuf, 0xF2);
+MacroAssembler _masm(cbuf);
-if (Matcher::_regEncode[src_first] >= 8) {
+__ movdbl( Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
-emit_opcode(*cbuf, Assembler::REX_R);
-}
-emit_opcode(*cbuf, 0x0F);
-emit_opcode(*cbuf, 0x11);
-encode_RegMem(*cbuf,
-Matcher::_regEncode[src_first],
-RSP_enc, 0x4, 0, offset,
-false);
 #ifndef PRODUCT
 } else if (!do_size) {
 st->print("movsd   [rsp + #%d], %s\t# spill",
 offset,
 Matcher::regName[src_first]);
 #endif
 }
 return
 ((offset == 0) ? 0 : (offset < 0x80 ? 1 : 4)) +
-((Matcher::_regEncode[src_first] < 8)
+((Matcher::_regEncode[src_first] >= 8)
-? 5
+? 6
-: 6); // REX
+: (5 + ((UseAVX>0)?1:0))); // REX
 } else {
 // 32-bit
 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 int offset = ra_->reg2offset(dst_first);
 if (cbuf) {
-emit_opcode(*cbuf, 0xF3);
+MacroAssembler _masm(cbuf);
-if (Matcher::_regEncode[src_first] >= 8) {
+__ movflt(Address(rsp, offset), as_XMMRegister(Matcher::_regEncode[src_first]));
-emit_opcode(*cbuf, Assembler::REX_R);
-}
-emit_opcode(*cbuf, 0x0F);
-emit_opcode(*cbuf, 0x11);
-encode_RegMem(*cbuf,
-Matcher::_regEncode[src_first],
-RSP_enc, 0x4, 0, offset,
-false);
 #ifndef PRODUCT
 } else if (!do_size) {
 st->print("movss   [rsp + #%d], %s\t# spill",
 offset,
 Matcher::regName[src_first]);
 #endif
 }
 return
 ((offset == 0) ? 0 : (offset < 0x80 ? 1 : 4)) +
-((Matcher::_regEncode[src_first] < 8)
+((Matcher::_regEncode[src_first] >=8)
-? 5
+? 6
-: 6); // REX
+: (5 + ((UseAVX>0)?1:0))); // REX
 }
 } else if (dst_first_rc == rc_int) {
 // xmm -> gpr
 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 // 64-bit
 if (cbuf) {
-emit_opcode(*cbuf, 0x66);
+MacroAssembler _masm(cbuf);
-if (Matcher::_regEncode[dst_first] < 8) {
+__ movdq( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
-if (Matcher::_regEncode[src_first] < 8) {
-emit_opcode(*cbuf, Assembler::REX_W);
-} else {
-emit_opcode(*cbuf, Assembler::REX_WR); // attention!
-}
-} else {
-if (Matcher::_regEncode[src_first] < 8) {
-emit_opcode(*cbuf, Assembler::REX_WB); // attention!
-} else {
-emit_opcode(*cbuf, Assembler::REX_WRB);
-}
-}
-emit_opcode(*cbuf, 0x0F);
-emit_opcode(*cbuf, 0x7E);
-emit_rm(*cbuf, 0x3,
-Matcher::_regEncode[src_first] & 7,
-Matcher::_regEncode[dst_first] & 7);
 #ifndef PRODUCT
 } else if (!do_size) {
 st->print("movdq   %s, %s\t# spill",
 Matcher::regName[dst_first],
 Matcher::regName[src_first]);
 } else {
 // 32-bit
 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 if (cbuf) {
-emit_opcode(*cbuf, 0x66);
+MacroAssembler _masm(cbuf);
-if (Matcher::_regEncode[dst_first] < 8) {
+__ movdl( as_Register(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
-if (Matcher::_regEncode[src_first] >= 8) {
-emit_opcode(*cbuf, Assembler::REX_R); // attention!
-}
-} else {
-if (Matcher::_regEncode[src_first] < 8) {
-emit_opcode(*cbuf, Assembler::REX_B); // attention!
-} else {
-emit_opcode(*cbuf, Assembler::REX_RB);
-}
-}
-emit_opcode(*cbuf, 0x0F);
-emit_opcode(*cbuf, 0x7E);
-emit_rm(*cbuf, 0x3,
-Matcher::_regEncode[src_first] & 7,
-Matcher::_regEncode[dst_first] & 7);
 #ifndef PRODUCT
 } else if (!do_size) {
 st->print("movdl   %s, %s\t# spill",
 Matcher::regName[dst_first],
 Matcher::regName[src_first]);
 #endif
 }
 return
-(Matcher::_regEncode[src_first] < 8 && Matcher::_regEncode[dst_first] < 8)
+(Matcher::_regEncode[src_first] >= 8 || Matcher::_regEncode[dst_first] >= 8)
-? 4
+? 5
-: 5; // REX
+: (4 + ((UseAVX>0)?1:0)); // REX
 }
 } else if (dst_first_rc == rc_float) {
 // xmm -> xmm
 if ((src_first & 1) == 0 && src_first + 1 == src_second &&
 (dst_first & 1) == 0 && dst_first + 1 == dst_second) {
 // 64-bit
 if (cbuf) {
-emit_opcode(*cbuf, UseXmmRegToRegMoveAll ? 0x66 : 0xF2);
+MacroAssembler _masm(cbuf);
-if (Matcher::_regEncode[dst_first] < 8) {
+__ movdbl( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
-if (Matcher::_regEncode[src_first] >= 8) {
-emit_opcode(*cbuf, Assembler::REX_B);
-}
-} else {
-if (Matcher::_regEncode[src_first] < 8) {
-emit_opcode(*cbuf, Assembler::REX_R);
-} else {
-emit_opcode(*cbuf, Assembler::REX_RB);
-}
-}
-emit_opcode(*cbuf, 0x0F);
-emit_opcode(*cbuf, UseXmmRegToRegMoveAll ? 0x28 : 0x10);
-emit_rm(*cbuf, 0x3,
-Matcher::_regEncode[dst_first] & 7,
-Matcher::_regEncode[src_first] & 7);
 #ifndef PRODUCT
 } else if (!do_size) {
 st->print("%s  %s, %s\t# spill",
 UseXmmRegToRegMoveAll ? "movapd" : "movsd ",
 Matcher::regName[dst_first],
 Matcher::regName[src_first]);
 #endif
 }
 return
-(Matcher::_regEncode[src_first] < 8 && Matcher::_regEncode[dst_first] < 8)
+(Matcher::_regEncode[src_first] >= 8 || Matcher::_regEncode[dst_first] >= 8)
-? 4
+? 5
-: 5; // REX
+: (4 + ((UseAVX>0)?1:0)); // REX
 } else {
 // 32-bit
 assert(!((src_first & 1) == 0 && src_first + 1 == src_second), "no transform");
 assert(!((dst_first & 1) == 0 && dst_first + 1 == dst_second), "no transform");
 if (cbuf) {
-if (!UseXmmRegToRegMoveAll)
+MacroAssembler _masm(cbuf);
-emit_opcode(*cbuf, 0xF3);
+__ movflt( as_XMMRegister(Matcher::_regEncode[dst_first]), as_XMMRegister(Matcher::_regEncode[src_first]));
-if (Matcher::_regEncode[dst_first] < 8) {
-if (Matcher::_regEncode[src_first] >= 8) {
-emit_opcode(*cbuf, Assembler::REX_B);
-}
-} else {
-if (Matcher::_regEncode[src_first] < 8) {
-emit_opcode(*cbuf, Assembler::REX_R);
-} else {
-emit_opcode(*cbuf, Assembler::REX_RB);
-}
-}
-emit_opcode(*cbuf, 0x0F);
-emit_opcode(*cbuf, UseXmmRegToRegMoveAll ? 0x28 : 0x10);
-emit_rm(*cbuf, 0x3,
-Matcher::_regEncode[dst_first] & 7,
-Matcher::_regEncode[src_first] & 7);
 #ifndef PRODUCT
 } else if (!do_size) {
 st->print("%s  %s, %s\t# spill",
 UseXmmRegToRegMoveAll ? "movaps" : "movss ",
 Matcher::regName[dst_first],
 Matcher::regName[src_first]);
 #endif
 }
-return
+return ((UseAVX>0) ? 5:
-(Matcher::_regEncode[src_first] < 8 && Matcher::_regEncode[dst_first] < 8)
+((Matcher::_regEncode[src_first] >= 8 || Matcher::_regEncode[dst_first] >= 8)
-? (UseXmmRegToRegMoveAll ? 3 : 4)
+? (UseXmmRegToRegMoveAll ? 4 : 5)
-: (UseXmmRegToRegMoveAll ? 4 : 5); // REX
+: (UseXmmRegToRegMoveAll ? 3 : 4))); // REX
 }
 }
 }
 assert(0," foo ");
 uint MachSpillCopyNode::size(PhaseRegAlloc *ra_) const
 {
 return implementation(NULL, ra_, true, NULL);
 }
-//=============================================================================
-#ifndef PRODUCT
-void MachNopNode::format(PhaseRegAlloc*, outputStream* st) const
-{
-st->print("nop \t# %d bytes pad for loops and calls", _count);
-}
-#endif
-void MachNopNode::emit(CodeBuffer &cbuf, PhaseRegAlloc*) const
-{
-MacroAssembler _masm(&cbuf);
-__ nop(_count);
-}
-uint MachNopNode::size(PhaseRegAlloc*) const
-{
-return _count;
-}
 //=============================================================================
 #ifndef PRODUCT
 void BoxLockNode::format(PhaseRegAlloc* ra_, outputStream* st) const
 {
 %{
 emit_opcode(cbuf, $opcode$$constant);
 emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
 %}
-enc_class cmpfp_fixup() %{
-MacroAssembler _masm(&cbuf);
-emit_cmpfp_fixup(_masm);
-%}
-enc_class cmpfp3(rRegI dst)
-%{
-int dstenc = $dst$$reg;
-// movl $dst, -1
-if (dstenc >= 8) {
-emit_opcode(cbuf, Assembler::REX_B);
-}
-emit_opcode(cbuf, 0xB8 | (dstenc & 7));
-emit_d32(cbuf, -1);
-// jp,s done
-emit_opcode(cbuf, 0x7A);
-emit_d8(cbuf, dstenc < 4 ? 0x08 : 0x0A);
-// jb,s done
-emit_opcode(cbuf, 0x72);
-emit_d8(cbuf, dstenc < 4 ? 0x06 : 0x08);
-// setne $dst
-if (dstenc >= 4) {
-emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_B);
-}
-emit_opcode(cbuf, 0x0F);
-emit_opcode(cbuf, 0x95);
-emit_opcode(cbuf, 0xC0 | (dstenc & 7));
-// movzbl $dst, $dst
-if (dstenc >= 4) {
-emit_opcode(cbuf, dstenc < 8 ? Assembler::REX : Assembler::REX_RB);
-}
-emit_opcode(cbuf, 0x0F);
-emit_opcode(cbuf, 0xB6);
-emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
-%}
 enc_class cdql_enc(no_rax_rdx_RegI div)
 %{
 // Full implementation of Java idiv and irem; checks for
 // special case as described in JVM spec., p.243 & p.271.
 //
 // CMOV
 $$$emit8$primary;
 emit_cc(cbuf, $secondary, $cop$$cmpcode);
 %}
-enc_class enc_cmovf_branch(cmpOp cop, regF dst, regF src)
-%{
-// Invert sense of branch from sense of cmov
-emit_cc(cbuf, 0x70, $cop$$cmpcode ^ 1);
-emit_d8(cbuf, ($dst$$reg < 8 && $src$$reg < 8)
-? (UseXmmRegToRegMoveAll ? 3 : 4)
-: (UseXmmRegToRegMoveAll ? 4 : 5) ); // REX
-// UseXmmRegToRegMoveAll ? movaps(dst, src) : movss(dst, src)
-if (!UseXmmRegToRegMoveAll) emit_opcode(cbuf, 0xF3);
-if ($dst$$reg < 8) {
-if ($src$$reg >= 8) {
-emit_opcode(cbuf, Assembler::REX_B);
-}
-} else {
-if ($src$$reg < 8) {
-emit_opcode(cbuf, Assembler::REX_R);
-} else {
-emit_opcode(cbuf, Assembler::REX_RB);
-}
-}
-emit_opcode(cbuf, 0x0F);
-emit_opcode(cbuf, UseXmmRegToRegMoveAll ? 0x28 : 0x10);
-emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
-%}
-enc_class enc_cmovd_branch(cmpOp cop, regD dst, regD src)
-%{
-// Invert sense of branch from sense of cmov
-emit_cc(cbuf, 0x70, $cop$$cmpcode ^ 1);
-emit_d8(cbuf, $dst$$reg < 8 && $src$$reg < 8 ? 4 : 5); // REX
-//  UseXmmRegToRegMoveAll ? movapd(dst, src) : movsd(dst, src)
-emit_opcode(cbuf, UseXmmRegToRegMoveAll ? 0x66 : 0xF2);
-if ($dst$$reg < 8) {
-if ($src$$reg >= 8) {
-emit_opcode(cbuf, Assembler::REX_B);
-}
-} else {
-if ($src$$reg < 8) {
-emit_opcode(cbuf, Assembler::REX_R);
-} else {
-emit_opcode(cbuf, Assembler::REX_RB);
-}
-}
-emit_opcode(cbuf, 0x0F);
-emit_opcode(cbuf, UseXmmRegToRegMoveAll ? 0x28 : 0x10);
-emit_rm(cbuf, 0x3, $dst$$reg & 7, $src$$reg & 7);
-%}
 enc_class enc_PartialSubtypeCheck()
 %{
 Register Rrdi = as_Register(RDI_enc); // result register
 Register Rrax = as_Register(RAX_enc); // super class
 Register Rrcx = as_Register(RCX_enc); // killed
 // CALL directly to the runtime
 emit_d32_reloc(cbuf,
 (int) ($meth$$method - ((intptr_t) cbuf.insts_end()) - 4),
 runtime_call_Relocation::spec(),
 RELOC_DISP32);
-%}
-enc_class preserve_SP %{
-debug_only(int off0 = cbuf.insts_size());
-MacroAssembler _masm(&cbuf);
-// RBP is preserved across all calls, even compiled calls.
-// Use it to preserve RSP in places where the callee might change the SP.
-__ movptr(rbp_mh_SP_save, rsp);
-debug_only(int off1 = cbuf.insts_size());
-assert(off1 - off0 == preserve_SP_size(), "correct size prediction");
-%}
-enc_class restore_SP %{
-MacroAssembler _masm(&cbuf);
-__ movptr(rsp, rbp_mh_SP_save);
 %}
 enc_class Java_Static_Call(method meth)
 %{
 // JAVA STATIC CALL
 // This next line should be generated from ADLC
 if ($src->constant_is_oop()) {
 emit_d64_reloc(cbuf, $src$$constant, relocInfo::oop_type, RELOC_IMM64);
 } else {
 emit_d64(cbuf, $src$$constant);
-}
-%}
-// Encode a reg-reg copy.  If it is useless, then empty encoding.
-enc_class enc_copy(rRegI dst, rRegI src)
-%{
-encode_copy(cbuf, $dst$$reg, $src$$reg);
-%}
-// Encode xmm reg-reg copy.  If it is useless, then empty encoding.
-enc_class enc_CopyXD( RegD dst, RegD src ) %{
-encode_CopyXD( cbuf, $dst$$reg, $src$$reg );
-%}
-enc_class enc_copy_always(rRegI dst, rRegI src)
-%{
-int srcenc = $src$$reg;
-int dstenc = $dst$$reg;
-if (dstenc < 8) {
-if (srcenc >= 8) {
-emit_opcode(cbuf, Assembler::REX_B);
-srcenc -= 8;
-}
-} else {
-if (srcenc < 8) {
-emit_opcode(cbuf, Assembler::REX_R);
-} else {
-emit_opcode(cbuf, Assembler::REX_RB);
-srcenc -= 8;
-}
-dstenc -= 8;
-}
-emit_opcode(cbuf, 0x8B);
-emit_rm(cbuf, 0x3, dstenc, srcenc);
-%}
-enc_class enc_copy_wide(rRegL dst, rRegL src)
-%{
-int srcenc = $src$$reg;
-int dstenc = $dst$$reg;
-if (dstenc != srcenc) {
-if (dstenc < 8) {
-if (srcenc < 8) {
-emit_opcode(cbuf, Assembler::REX_W);
-} else {
-emit_opcode(cbuf, Assembler::REX_WB);
-srcenc -= 8;
-}
-} else {
-if (srcenc < 8) {
-emit_opcode(cbuf, Assembler::REX_WR);
-} else {
-emit_opcode(cbuf, Assembler::REX_WRB);
-srcenc -= 8;
-}
-dstenc -= 8;
-}
-emit_opcode(cbuf, 0x8B);
-emit_rm(cbuf, 0x3, dstenc, srcenc);
 }
 %}
 enc_class Con32(immI src)
 %{
 emit_opcode(cbuf, 0xB6);
 emit_rm(cbuf, 0x3, dstenc & 7, dstenc & 7);
 %}
 enc_class Push_ResultXD(regD dst) %{
-int dstenc = $dst$$reg;
+MacroAssembler _masm(&cbuf);
+__ fstp_d(Address(rsp, 0));
-store_to_stackslot( cbuf, 0xDD, 0x03, 0 ); //FSTP [RSP]
+__ movdbl($dst$$XMMRegister, Address(rsp, 0));
+__ addptr(rsp, 8);
-// UseXmmLoadAndClearUpper ? movsd dst,[rsp] : movlpd dst,[rsp]
-emit_opcode  (cbuf, UseXmmLoadAndClearUpper ? 0xF2 : 0x66);
-if (dstenc >= 8) {
-emit_opcode(cbuf, Assembler::REX_R);
-}
-emit_opcode  (cbuf, 0x0F );
-emit_opcode  (cbuf, UseXmmLoadAndClearUpper ? 0x10 : 0x12 );
-encode_RegMem(cbuf, dstenc, RSP_enc, 0x4, 0, 0, false);
-// add rsp,8
-emit_opcode(cbuf, Assembler::REX_W);
-emit_opcode(cbuf,0x83);
-emit_rm(cbuf,0x3, 0x0, RSP_enc);
-emit_d8(cbuf,0x08);
 %}
 enc_class Push_SrcXD(regD src) %{
-int srcenc = $src$$reg;
-// subq rsp,#8
-emit_opcode(cbuf, Assembler::REX_W);
-emit_opcode(cbuf, 0x83);
-emit_rm(cbuf, 0x3, 0x5, RSP_enc);
-emit_d8(cbuf, 0x8);
-// movsd [rsp],src
-emit_opcode(cbuf, 0xF2);
-if (srcenc >= 8) {
-emit_opcode(cbuf, Assembler::REX_R);
-}
-emit_opcode(cbuf, 0x0F);
-emit_opcode(cbuf, 0x11);
-encode_RegMem(cbuf, srcenc, RSP_enc, 0x4, 0, 0, false);
-// fldd [rsp]
-emit_opcode(cbuf, 0x66);
-emit_opcode(cbuf, 0xDD);
-encode_RegMem(cbuf, 0x0, RSP_enc, 0x4, 0, 0, false);
-%}
-enc_class movq_ld(regD dst, memory mem) %{
 MacroAssembler _masm(&cbuf);
-__ movq($dst$$XMMRegister, $mem$$Address);
+__ subptr(rsp, 8);
-%}
+__ movdbl(Address(rsp, 0), $src$$XMMRegister);
+__ fld_d(Address(rsp, 0));
-enc_class movq_st(memory mem, regD src) %{
+%}
-MacroAssembler _masm(&cbuf);
-__ movq($mem$$Address, $src$$XMMRegister);
-%}
-enc_class pshufd_8x8(regF dst, regF src) %{
-MacroAssembler _masm(&cbuf);
-encode_CopyXD(cbuf, $dst$$reg, $src$$reg);
-__ punpcklbw(as_XMMRegister($dst$$reg), as_XMMRegister($dst$$reg));
-__ pshuflw(as_XMMRegister($dst$$reg), as_XMMRegister($dst$$reg), 0x00);
-%}
-enc_class pshufd_4x16(regF dst, regF src) %{
-MacroAssembler _masm(&cbuf);
-__ pshuflw(as_XMMRegister($dst$$reg), as_XMMRegister($src$$reg), 0x00);
-%}
-enc_class pshufd(regD dst, regD src, int mode) %{
-MacroAssembler _masm(&cbuf);
-__ pshufd(as_XMMRegister($dst$$reg), as_XMMRegister($src$$reg), $mode);
-%}
-enc_class pxor(regD dst, regD src) %{
-MacroAssembler _masm(&cbuf);
-__ pxor(as_XMMRegister($dst$$reg), as_XMMRegister($src$$reg));
-%}
-enc_class mov_i2x(regD dst, rRegI src) %{
-MacroAssembler _masm(&cbuf);
-__ movdl(as_XMMRegister($dst$$reg), as_Register($src$$reg));
-%}
 // obj: object to lock
 // box: box address (header location) -- killed
 // tmp: rax -- killed
 // scr: rbx -- killed
 (int) (OptoRuntime::rethrow_stub() - cbuf.insts_end() - 4),
 runtime_call_Relocation::spec(),
 RELOC_DISP32);
 %}
-enc_class absF_encoding(regF dst)
-%{
-int dstenc = $dst$$reg;
-address signmask_address = (address) StubRoutines::x86::float_sign_mask();
-cbuf.set_insts_mark();
-if (dstenc >= 8) {
-emit_opcode(cbuf, Assembler::REX_R);
-dstenc -= 8;
-}
-// XXX reg_mem doesn't support RIP-relative addressing yet
-emit_opcode(cbuf, 0x0F);
-emit_opcode(cbuf, 0x54);
-emit_rm(cbuf, 0x0, dstenc, 0x5);  // 00 reg 101
-emit_d32_reloc(cbuf, signmask_address);
-%}
-enc_class absD_encoding(regD dst)
-%{
-int dstenc = $dst$$reg;
-address signmask_address = (address) StubRoutines::x86::double_sign_mask();
-cbuf.set_insts_mark();
-emit_opcode(cbuf, 0x66);
-if (dstenc >= 8) {
-emit_opcode(cbuf, Assembler::REX_R);
-dstenc -= 8;
-}
-// XXX reg_mem doesn't support RIP-relative addressing yet
-emit_opcode(cbuf, 0x0F);
-emit_opcode(cbuf, 0x54);
-emit_rm(cbuf, 0x0, dstenc, 0x5);  // 00 reg 101
-emit_d32_reloc(cbuf, signmask_address);
-%}
-enc_class negF_encoding(regF dst)
-%{
-int dstenc = $dst$$reg;
-address signflip_address = (address) StubRoutines::x86::float_sign_flip();
-cbuf.set_insts_mark();
-if (dstenc >= 8) {
-emit_opcode(cbuf, Assembler::REX_R);
-dstenc -= 8;
-}
-// XXX reg_mem doesn't support RIP-relative addressing yet
-emit_opcode(cbuf, 0x0F);
-emit_opcode(cbuf, 0x57);
-emit_rm(cbuf, 0x0, dstenc, 0x5);  // 00 reg 101
-emit_d32_reloc(cbuf, signflip_address);
-%}
-enc_class negD_encoding(regD dst)
-%{
-int dstenc = $dst$$reg;
-address signflip_address = (address) StubRoutines::x86::double_sign_flip();
-cbuf.set_insts_mark();
-emit_opcode(cbuf, 0x66);
-if (dstenc >= 8) {
-emit_opcode(cbuf, Assembler::REX_R);
-dstenc -= 8;
-}
-// XXX reg_mem doesn't support RIP-relative addressing yet
-emit_opcode(cbuf, 0x0F);
-emit_opcode(cbuf, 0x57);
-emit_rm(cbuf, 0x0, dstenc, 0x5);  // 00 reg 101
-emit_d32_reloc(cbuf, signflip_address);
-%}
-enc_class f2i_fixup(rRegI dst, regF src)
-%{
-int dstenc = $dst$$reg;
-int srcenc = $src$$reg;
-// cmpl $dst, #0x80000000
-if (dstenc >= 8) {
-emit_opcode(cbuf, Assembler::REX_B);
-}
-emit_opcode(cbuf, 0x81);
-emit_rm(cbuf, 0x3, 0x7, dstenc & 7);
-emit_d32(cbuf, 0x80000000);
-// jne,s done
-emit_opcode(cbuf, 0x75);
-if (srcenc < 8 && dstenc < 8) {
-emit_d8(cbuf, 0xF);
-} else if (srcenc >= 8 && dstenc >= 8) {
-emit_d8(cbuf, 0x11);
-} else {
-emit_d8(cbuf, 0x10);
-}
-// subq rsp, #8
-emit_opcode(cbuf, Assembler::REX_W);
-emit_opcode(cbuf, 0x83);
-emit_rm(cbuf, 0x3, 0x5, RSP_enc);
-emit_d8(cbuf, 8);
-// movss [rsp], $src
-emit_opcode(cbuf, 0xF3);
-if (srcenc >= 8) {
-emit_opcode(cbuf, Assembler::REX_R);
-}
-emit_opcode(cbuf, 0x0F);
-emit_opcode(cbuf, 0x11);
-encode_RegMem(cbuf, srcenc, RSP_enc, 0x4, 0, 0, false); // 2 bytes
-// call f2i_fixup
-cbuf.set_insts_mark();
-emit_opcode(cbuf, 0xE8);
-emit_d32_reloc(cbuf,
-(int)
-(StubRoutines::x86::f2i_fixup() - cbuf.insts_end() - 4),
-runtime_call_Relocation::spec(),
-RELOC_DISP32);
-// popq $dst
-if (dstenc >= 8) {
-emit_opcode(cbuf, Assembler::REX_B);
-}
-emit_opcode(cbuf, 0x58 | (dstenc & 7));
-// done:
-%}
-enc_class f2l_fixup(rRegL dst, regF src)
-%{
-int dstenc = $dst$$reg;
-int srcenc = $src$$reg;
-address const_address = (address) StubRoutines::x86::double_sign_flip();
-// cmpq $dst, [0x8000000000000000]
-cbuf.set_insts_mark();
-emit_opcode(cbuf, dstenc < 8 ? Assembler::REX_W : Assembler::REX_WR);
-emit_opcode(cbuf, 0x39);
-// XXX reg_mem doesn't support RIP-relative addressing yet
-emit_rm(cbuf, 0x0, dstenc & 7, 0x5); // 00 reg 101
-emit_d32_reloc(cbuf, const_address);
-// jne,s done
-emit_opcode(cbuf, 0x75);
-if (srcenc < 8 && dstenc < 8) {
-emit_d8(cbuf, 0xF);
-} else if (srcenc >= 8 && dstenc >= 8) {
-emit_d8(cbuf, 0x11);
-} else {
-emit_d8(cbuf, 0x10);
-}
-// subq rsp, #8
-emit_opcode(cbuf, Assembler::REX_W);
-emit_opcode(cbuf, 0x83);
-emit_rm(cbuf, 0x3, 0x5, RSP_enc);
-emit_d8(cbuf, 8);
-// movss [rsp], $src
-emit_opcode(cbuf, 0xF3);
-if (srcenc >= 8) {
-emit_opcode(cbuf, Assembler::REX_R);
-}
-emit_opcode(cbuf, 0x0F);
-emit_opcode(cbuf, 0x11);
-encode_RegMem(cbuf, srcenc, RSP_enc, 0x4, 0, 0, false); // 2 bytes
-// call f2l_fixup
-cbuf.set_insts_mark();
-emit_opcode(cbuf, 0xE8);
-emit_d32_reloc(cbuf,
-(int)
-(StubRoutines::x86::f2l_fixup() - cbuf.insts_end() - 4),
-runtime_call_Relocation::spec(),
-RELOC_DISP32);
-// popq $dst
-if (dstenc >= 8) {
-emit_opcode(cbuf, Assembler::REX_B);
-}
-emit_opcode(cbuf, 0x58 | (dstenc & 7));
-// done:
-%}
-enc_class d2i_fixup(rRegI dst, regD src)
-%{
-int dstenc = $dst$$reg;
-int srcenc = $src$$reg;
-// cmpl $dst, #0x80000000
-if (dstenc >= 8) {
-emit_opcode(cbuf, Assembler::REX_B);
-}
-emit_opcode(cbuf, 0x81);
-emit_rm(cbuf, 0x3, 0x7, dstenc & 7);
-emit_d32(cbuf, 0x80000000);
-// jne,s done
-emit_opcode(cbuf, 0x75);
-if (srcenc < 8 && dstenc < 8) {
-emit_d8(cbuf, 0xF);
-} else if (srcenc >= 8 && dstenc >= 8) {
-emit_d8(cbuf, 0x11);
-} else {
-emit_d8(cbuf, 0x10);
-}
-// subq rsp, #8
-emit_opcode(cbuf, Assembler::REX_W);
-emit_opcode(cbuf, 0x83);
-emit_rm(cbuf, 0x3, 0x5, RSP_enc);
-emit_d8(cbuf, 8);
-// movsd [rsp], $src
-emit_opcode(cbuf, 0xF2);
-if (srcenc >= 8) {
-emit_opcode(cbuf, Assembler::REX_R);
-}
-emit_opcode(cbuf, 0x0F);
-emit_opcode(cbuf, 0x11);
-encode_RegMem(cbuf, srcenc, RSP_enc, 0x4, 0, 0, false); // 2 bytes
-// call d2i_fixup
-cbuf.set_insts_mark();
-emit_opcode(cbuf, 0xE8);
-emit_d32_reloc(cbuf,
-(int)
-(StubRoutines::x86::d2i_fixup() - cbuf.insts_end() - 4),
-runtime_call_Relocation::spec(),
-RELOC_DISP32);
-// popq $dst
-if (dstenc >= 8) {
-emit_opcode(cbuf, Assembler::REX_B);
-}
-emit_opcode(cbuf, 0x58 | (dstenc & 7));
-// done:
-%}
-enc_class d2l_fixup(rRegL dst, regD src)
-%{
-int dstenc = $dst$$reg;
-int srcenc = $src$$reg;
-address const_address = (address) StubRoutines::x86::double_sign_flip();
-// cmpq $dst, [0x8000000000000000]
-cbuf.set_insts_mark();
-emit_opcode(cbuf, dstenc < 8 ? Assembler::REX_W : Assembler::REX_WR);
-emit_opcode(cbuf, 0x39);
-// XXX reg_mem doesn't support RIP-relative addressing yet
-emit_rm(cbuf, 0x0, dstenc & 7, 0x5); // 00 reg 101
-emit_d32_reloc(cbuf, const_address);
-// jne,s done
-emit_opcode(cbuf, 0x75);
-if (srcenc < 8 && dstenc < 8) {
-emit_d8(cbuf, 0xF);
-} else if (srcenc >= 8 && dstenc >= 8) {
-emit_d8(cbuf, 0x11);
-} else {
-emit_d8(cbuf, 0x10);
-}
-// subq rsp, #8
-emit_opcode(cbuf, Assembler::REX_W);
-emit_opcode(cbuf, 0x83);
-emit_rm(cbuf, 0x3, 0x5, RSP_enc);
-emit_d8(cbuf, 8);
-// movsd [rsp], $src
-emit_opcode(cbuf, 0xF2);
-if (srcenc >= 8) {
-emit_opcode(cbuf, Assembler::REX_R);
-}
-emit_opcode(cbuf, 0x0F);
-emit_opcode(cbuf, 0x11);
-encode_RegMem(cbuf, srcenc, RSP_enc, 0x4, 0, 0, false); // 2 bytes
-// call d2l_fixup
-cbuf.set_insts_mark();
-emit_opcode(cbuf, 0xE8);
-emit_d32_reloc(cbuf,
-(int)
-(StubRoutines::x86::d2l_fixup() - cbuf.insts_end() - 4),
-runtime_call_Relocation::spec(),
-RELOC_DISP32);
-// popq $dst
-if (dstenc >= 8) {
-emit_opcode(cbuf, Assembler::REX_B);
-}
-emit_opcode(cbuf, 0x58 | (dstenc & 7));
-// done:
-%}
 %}
 //----------FRAME--------------------------------------------------------------
 // representing the register number (i.e. - use a register name) or
 // stack slot.
 // Ret Addr is on stack in slot 0 if no locks or verification or alignment.
 // Otherwise, it is above the locks and verification slot and alignment word
 return_addr(STACK - 2 +
-round_to(2 + 2 * VerifyStackAtCalls +
+round_to((Compile::current()->in_preserve_stack_slots() +
-Compile::current()->fixed_slots(),
+Compile::current()->fixed_slots()),
-WordsPerLong * 2));
+stack_alignment_in_slots()));
 // Body of function which returns an integer array locating
 // arguments either in registers or in stack slots.  Passed an array
 // of ideal registers called "sig" and a "length" count.  Stack-slot
 // offsets are based on outgoing arguments, i.e. a CALLER setting up
 %{
 match(Set dst (LoadF mem));
 ins_cost(145); // XXX
 format %{ "movss   $dst, $mem\t# float" %}
-opcode(0xF3, 0x0F, 0x10);
+ins_encode %{
-ins_encode(OpcP, REX_reg_mem(dst, mem), OpcS, OpcT, reg_mem(dst, mem));
+__ movflt($dst$$XMMRegister, $mem$$Address);
+%}
 ins_pipe(pipe_slow); // XXX
 %}
 // Load Double
 instruct loadD_partial(regD dst, memory mem)
 predicate(!UseXmmLoadAndClearUpper);
 match(Set dst (LoadD mem));
 ins_cost(145); // XXX
 format %{ "movlpd  $dst, $mem\t# double" %}
-opcode(0x66, 0x0F, 0x12);
+ins_encode %{
-ins_encode(OpcP, REX_reg_mem(dst, mem), OpcS, OpcT, reg_mem(dst, mem));
+__ movdbl($dst$$XMMRegister, $mem$$Address);
+%}
 ins_pipe(pipe_slow); // XXX
 %}
 instruct loadD(regD dst, memory mem)
 %{
 predicate(UseXmmLoadAndClearUpper);
 match(Set dst (LoadD mem));
 ins_cost(145); // XXX
 format %{ "movsd   $dst, $mem\t# double" %}
-opcode(0xF2, 0x0F, 0x10);
+ins_encode %{
-ins_encode(OpcP, REX_reg_mem(dst, mem), OpcS, OpcT, reg_mem(dst, mem));
+__ movdbl($dst$$XMMRegister, $mem$$Address);
+%}
 ins_pipe(pipe_slow); // XXX
 %}
 // Load Aligned Packed Byte to XMM register
 instruct loadA8B(regD dst, memory mem) %{
 match(Set dst (Load8B mem));
 ins_cost(125);
 format %{ "MOVQ  $dst,$mem\t! packed8B" %}
-ins_encode( movq_ld(dst, mem));
+ins_encode %{
+__ movq($dst$$XMMRegister, $mem$$Address);
+%}
 ins_pipe( pipe_slow );
 %}
 // Load Aligned Packed Short to XMM register
 instruct loadA4S(regD dst, memory mem) %{
 match(Set dst (Load4S mem));
 ins_cost(125);
 format %{ "MOVQ  $dst,$mem\t! packed4S" %}
-ins_encode( movq_ld(dst, mem));
+ins_encode %{
+__ movq($dst$$XMMRegister, $mem$$Address);
+%}
 ins_pipe( pipe_slow );
 %}
 // Load Aligned Packed Char to XMM register
 instruct loadA4C(regD dst, memory mem) %{
 match(Set dst (Load4C mem));
 ins_cost(125);
 format %{ "MOVQ  $dst,$mem\t! packed4C" %}
-ins_encode( movq_ld(dst, mem));
+ins_encode %{
+__ movq($dst$$XMMRegister, $mem$$Address);
+%}
 ins_pipe( pipe_slow );
 %}
 // Load Aligned Packed Integer to XMM register
 instruct load2IU(regD dst, memory mem) %{
 match(Set dst (Load2I mem));
 ins_cost(125);
 format %{ "MOVQ  $dst,$mem\t! packed2I" %}
-ins_encode( movq_ld(dst, mem));
+ins_encode %{
+__ movq($dst$$XMMRegister, $mem$$Address);
+%}
 ins_pipe( pipe_slow );
 %}
 // Load Aligned Packed Single to XMM
 instruct loadA2F(regD dst, memory mem) %{
 match(Set dst (Load2F mem));
-ins_cost(145);
+ins_cost(125);
 format %{ "MOVQ  $dst,$mem\t! packed2F" %}
-ins_encode( movq_ld(dst, mem));
+ins_encode %{
+__ movq($dst$$XMMRegister, $mem$$Address);
+%}
 ins_pipe( pipe_slow );
 %}
 // Load Effective Address
 instruct leaP8(rRegP dst, indOffset8 mem)
 %{
 match(Set dst src);
 ins_cost(100);
 format %{ "xorps   $dst, $dst\t# float 0.0" %}
-opcode(0x0F, 0x57);
+ins_encode %{
-ins_encode(REX_reg_reg(dst, dst), OpcP, OpcS, reg_reg(dst, dst));
+__ xorps($dst$$XMMRegister, $dst$$XMMRegister);
+%}
 ins_pipe(pipe_slow);
 %}
 // Use the same format since predicate() can not be used here.
 instruct loadConD(regD dst, immD con) %{
 %{
 match(Set dst src);
 ins_cost(100);
 format %{ "xorpd   $dst, $dst\t# double 0.0" %}
-opcode(0x66, 0x0F, 0x57);
+ins_encode %{
-ins_encode(OpcP, REX_reg_reg(dst, dst), OpcS, OpcT, reg_reg(dst, dst));
+__ xorpd ($dst$$XMMRegister, $dst$$XMMRegister);
+%}
 ins_pipe(pipe_slow);
 %}
 instruct loadSSI(rRegI dst, stackSlotI src)
 %{
 %{
 match(Set dst src);
 ins_cost(125);
 format %{ "movss   $dst, $src\t# float stk" %}
-opcode(0xF3, 0x0F, 0x10);
+ins_encode %{
-ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
+__ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
+%}
 ins_pipe(pipe_slow); // XXX
 %}
 // Use the same format since predicate() can not be used here.
 instruct loadSSD(regD dst, stackSlotD src)
 // Store Aligned Packed Byte XMM register to memory
 instruct storeA8B(memory mem, regD src) %{
 match(Set mem (Store8B mem src));
 ins_cost(145);
 format %{ "MOVQ  $mem,$src\t! packed8B" %}
-ins_encode( movq_st(mem, src));
+ins_encode %{
+__ movq($mem$$Address, $src$$XMMRegister);
+%}
 ins_pipe( pipe_slow );
 %}
 // Store Aligned Packed Char/Short XMM register to memory
 instruct storeA4C(memory mem, regD src) %{
 match(Set mem (Store4C mem src));
 ins_cost(145);
 format %{ "MOVQ  $mem,$src\t! packed4C" %}
-ins_encode( movq_st(mem, src));
+ins_encode %{
+__ movq($mem$$Address, $src$$XMMRegister);
+%}
 ins_pipe( pipe_slow );
 %}
 // Store Aligned Packed Integer XMM register to memory
 instruct storeA2I(memory mem, regD src) %{
 match(Set mem (Store2I mem src));
 ins_cost(145);
 format %{ "MOVQ  $mem,$src\t! packed2I" %}
-ins_encode( movq_st(mem, src));
+ins_encode %{
+__ movq($mem$$Address, $src$$XMMRegister);
+%}
 ins_pipe( pipe_slow );
 %}
 // Store CMS card-mark Immediate
 instruct storeImmCM0_reg(memory mem, immI0 zero)
 // Store Aligned Packed Single Float XMM register to memory
 instruct storeA2F(memory mem, regD src) %{
 match(Set mem (Store2F mem src));
 ins_cost(145);
 format %{ "MOVQ  $mem,$src\t! packed2F" %}
-ins_encode( movq_st(mem, src));
+ins_encode %{
+__ movq($mem$$Address, $src$$XMMRegister);
+%}
 ins_pipe( pipe_slow );
 %}
 // Store Float
 instruct storeF(memory mem, regF src)
 %{
 match(Set mem (StoreF mem src));
 ins_cost(95); // XXX
 format %{ "movss   $mem, $src\t# float" %}
-opcode(0xF3, 0x0F, 0x11);
+ins_encode %{
-ins_encode(OpcP, REX_reg_mem(src, mem), OpcS, OpcT, reg_mem(src, mem));
+__ movflt($mem$$Address, $src$$XMMRegister);
+%}
 ins_pipe(pipe_slow); // XXX
 %}
 // Store immediate Float value (it is faster than store from XMM register)
 instruct storeF0(memory mem, immF0 zero)
 %{
 match(Set mem (StoreD mem src));
 ins_cost(95); // XXX
 format %{ "movsd   $mem, $src\t# double" %}
-opcode(0xF2, 0x0F, 0x11);
+ins_encode %{
-ins_encode(OpcP, REX_reg_mem(src, mem), OpcS, OpcT, reg_mem(src, mem));
+__ movdbl($mem$$Address, $src$$XMMRegister);
+%}
 ins_pipe(pipe_slow); // XXX
 %}
 // Store immediate double 0.0 (it is faster than store from XMM register)
 instruct storeD0_imm(memory mem, immD0 src)
 %{
 match(Set dst src);
 ins_cost(95); // XXX
 format %{ "movss   $dst, $src\t# float stk" %}
-opcode(0xF3, 0x0F, 0x11);
+ins_encode %{
-ins_encode(OpcP, REX_reg_mem(src, dst), OpcS, OpcT, reg_mem(src, dst));
+__ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
+%}
 ins_pipe(pipe_slow); // XXX
 %}
 instruct storeSSD(stackSlotD dst, regD src)
 %{
 match(Set dst src);
 ins_cost(95); // XXX
 format %{ "movsd   $dst, $src\t# double stk" %}
-opcode(0xF2, 0x0F, 0x11);
+ins_encode %{
-ins_encode(OpcP, REX_reg_mem(src, dst), OpcS, OpcT, reg_mem(src, dst));
+__ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
+%}
 ins_pipe(pipe_slow); // XXX
 %}
 //----------BSWAP Instructions-------------------------------------------------
 instruct bytes_reverse_int(rRegI dst) %{
 format %{ "MEMBAR-volatile (unnecessary so empty encoding)" %}
 ins_encode();
 ins_pipe(empty);
 %}
+instruct membar_storestore() %{
+match(MemBarStoreStore);
+ins_cost(0);
+size(0);
+format %{ "MEMBAR-storestore (empty encoding)" %}
+ins_encode( );
+ins_pipe(empty);
+%}
 //----------Move Instructions--------------------------------------------------
 instruct castX2P(rRegP dst, rRegL src)
 %{
 match(Set dst (CastX2P src));
 format %{ "movq    $dst, $src\t# long->ptr" %}
-ins_encode(enc_copy_wide(dst, src));
+ins_encode %{
+if ($dst$$reg != $src$$reg) {
+__ movptr($dst$$Register, $src$$Register);
+}
+%}
 ins_pipe(ialu_reg_reg); // XXX
 %}
 instruct castP2X(rRegL dst, rRegP src)
 %{
 match(Set dst (CastP2X src));
 format %{ "movq    $dst, $src\t# ptr -> long" %}
-ins_encode(enc_copy_wide(dst, src));
+ins_encode %{
+if ($dst$$reg != $src$$reg) {
+__ movptr($dst$$Register, $src$$Register);
+}
+%}
 ins_pipe(ialu_reg_reg); // XXX
 %}
 // Convert oop pointer into compressed form
 ins_cost(200); // XXX
 format %{ "jn$cop    skip\t# signed cmove float\n\t"
 "movss     $dst, $src\n"
 "skip:" %}
-ins_encode(enc_cmovf_branch(cop, dst, src));
+ins_encode %{
+Label Lskip;
+// Invert sense of branch from sense of CMOV
+__ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
+__ movflt($dst$$XMMRegister, $src$$XMMRegister);
+__ bind(Lskip);
+%}
 ins_pipe(pipe_slow);
 %}
 // instruct cmovF_mem(cmpOp cop, rFlagsReg cr, regF dst, memory src)
 // %{
 ins_cost(200); // XXX
 format %{ "jn$cop    skip\t# unsigned cmove float\n\t"
 "movss     $dst, $src\n"
 "skip:" %}
-ins_encode(enc_cmovf_branch(cop, dst, src));
+ins_encode %{
+Label Lskip;
+// Invert sense of branch from sense of CMOV
+__ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
+__ movflt($dst$$XMMRegister, $src$$XMMRegister);
+__ bind(Lskip);
+%}
 ins_pipe(pipe_slow);
 %}
 instruct cmovF_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regF dst, regF src) %{
 match(Set dst (CMoveF (Binary cop cr) (Binary dst src)));
 ins_cost(200); // XXX
 format %{ "jn$cop    skip\t# signed cmove double\n\t"
 "movsd     $dst, $src\n"
 "skip:" %}
-ins_encode(enc_cmovd_branch(cop, dst, src));
+ins_encode %{
+Label Lskip;
+// Invert sense of branch from sense of CMOV
+__ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
+__ movdbl($dst$$XMMRegister, $src$$XMMRegister);
+__ bind(Lskip);
+%}
 ins_pipe(pipe_slow);
 %}
 instruct cmovD_regU(cmpOpU cop, rFlagsRegU cr, regD dst, regD src)
 %{
 ins_cost(200); // XXX
 format %{ "jn$cop    skip\t# unsigned cmove double\n\t"
 "movsd     $dst, $src\n"
 "skip:" %}
-ins_encode(enc_cmovd_branch(cop, dst, src));
+ins_encode %{
+Label Lskip;
+// Invert sense of branch from sense of CMOV
+__ jccb((Assembler::Condition)($cop$$cmpcode^1), Lskip);
+__ movdbl($dst$$XMMRegister, $src$$XMMRegister);
+__ bind(Lskip);
+%}
 ins_pipe(pipe_slow);
 %}
 instruct cmovD_regUCF(cmpOpUCF cop, rFlagsRegUCF cr, regD dst, regD src) %{
 match(Set dst (CMoveD (Binary cop cr) (Binary dst src)));
 format %{ "ucomiss $src1, $src2\n\t"
 "jnp,s   exit\n\t"
 "pushfq\t# saw NaN, set CF\n\t"
 "andq    [rsp], #0xffffff2b\n\t"
 "popfq\n"
-"exit:   nop\t# avoid branch to branch" %}
+"exit:" %}
-opcode(0x0F, 0x2E);
+ins_encode %{
-ins_encode(REX_reg_reg(src1, src2), OpcP, OpcS, reg_reg(src1, src2),
+__ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
-cmpfp_fixup);
+emit_cmpfp_fixup(_masm);
+%}
 ins_pipe(pipe_slow);
 %}
 instruct cmpF_cc_reg_CF(rFlagsRegUCF cr, regF src1, regF src2) %{
 match(Set cr (CmpF src1 src2));
-ins_cost(145);
+ins_cost(100);
 format %{ "ucomiss $src1, $src2" %}
 ins_encode %{
 __ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
 %}
 ins_pipe(pipe_slow);
 format %{ "ucomiss $src1, $src2\n\t"
 "jnp,s   exit\n\t"
 "pushfq\t# saw NaN, set CF\n\t"
 "andq    [rsp], #0xffffff2b\n\t"
 "popfq\n"
-"exit:   nop\t# avoid branch to branch" %}
+"exit:" %}
-opcode(0x0F, 0x2E);
+ins_encode %{
-ins_encode(REX_reg_mem(src1, src2), OpcP, OpcS, reg_mem(src1, src2),
+__ ucomiss($src1$$XMMRegister, $src2$$Address);
-cmpfp_fixup);
+emit_cmpfp_fixup(_masm);
+%}
 ins_pipe(pipe_slow);
 %}
 instruct cmpF_cc_memCF(rFlagsRegUCF cr, regF src1, memory src2) %{
 match(Set cr (CmpF src1 (LoadF src2)));
 ins_cost(100);
 format %{ "ucomiss $src1, $src2" %}
-opcode(0x0F, 0x2E);
+ins_encode %{
-ins_encode(REX_reg_mem(src1, src2), OpcP, OpcS, reg_mem(src1, src2));
+__ ucomiss($src1$$XMMRegister, $src2$$Address);
+%}
 ins_pipe(pipe_slow);
 %}
 instruct cmpF_cc_imm(rFlagsRegU cr, regF src, immF con) %{
 match(Set cr (CmpF src con));
 format %{ "ucomiss $src, [$constantaddress]\t# load from constant table: float=$con\n\t"
 "jnp,s   exit\n\t"
 "pushfq\t# saw NaN, set CF\n\t"
 "andq    [rsp], #0xffffff2b\n\t"
 "popfq\n"
-"exit:   nop\t# avoid branch to branch" %}
+"exit:" %}
 ins_encode %{
 __ ucomiss($src$$XMMRegister, $constantaddress($con));
 emit_cmpfp_fixup(_masm);
 %}
 ins_pipe(pipe_slow);
 format %{ "ucomisd $src1, $src2\n\t"
 "jnp,s   exit\n\t"
 "pushfq\t# saw NaN, set CF\n\t"
 "andq    [rsp], #0xffffff2b\n\t"
 "popfq\n"
-"exit:   nop\t# avoid branch to branch" %}
+"exit:" %}
-opcode(0x66, 0x0F, 0x2E);
+ins_encode %{
-ins_encode(OpcP, REX_reg_reg(src1, src2), OpcS, OpcT, reg_reg(src1, src2),
+__ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
-cmpfp_fixup);
+emit_cmpfp_fixup(_masm);
+%}
 ins_pipe(pipe_slow);
 %}
 instruct cmpD_cc_reg_CF(rFlagsRegUCF cr, regD src1, regD src2) %{
 match(Set cr (CmpD src1 src2));
 format %{ "ucomisd $src1, $src2\n\t"
 "jnp,s   exit\n\t"
 "pushfq\t# saw NaN, set CF\n\t"
 "andq    [rsp], #0xffffff2b\n\t"
 "popfq\n"
-"exit:   nop\t# avoid branch to branch" %}
+"exit:" %}
-opcode(0x66, 0x0F, 0x2E);
+ins_encode %{
-ins_encode(OpcP, REX_reg_mem(src1, src2), OpcS, OpcT, reg_mem(src1, src2),
+__ ucomisd($src1$$XMMRegister, $src2$$Address);
-cmpfp_fixup);
+emit_cmpfp_fixup(_masm);
+%}
 ins_pipe(pipe_slow);
 %}
 instruct cmpD_cc_memCF(rFlagsRegUCF cr, regD src1, memory src2) %{
 match(Set cr (CmpD src1 (LoadD src2)));
 ins_cost(100);
 format %{ "ucomisd $src1, $src2" %}
-opcode(0x66, 0x0F, 0x2E);
+ins_encode %{
-ins_encode(OpcP, REX_reg_mem(src1, src2), OpcS, OpcT, reg_mem(src1, src2));
+__ ucomisd($src1$$XMMRegister, $src2$$Address);
+%}
 ins_pipe(pipe_slow);
 %}
 instruct cmpD_cc_imm(rFlagsRegU cr, regD src, immD con) %{
 match(Set cr (CmpD src con));
 format %{ "ucomisd $src, [$constantaddress]\t# load from constant table: double=$con\n\t"
 "jnp,s   exit\n\t"
 "pushfq\t# saw NaN, set CF\n\t"
 "andq    [rsp], #0xffffff2b\n\t"
 "popfq\n"
-"exit:   nop\t# avoid branch to branch" %}
+"exit:" %}
 ins_encode %{
 __ ucomisd($src$$XMMRegister, $constantaddress($con));
 emit_cmpfp_fixup(_masm);
 %}
 ins_pipe(pipe_slow);
 "jp,s    done\n\t"
 "jb,s    done\n\t"
 "setne   $dst\n\t"
 "movzbl  $dst, $dst\n"
 "done:" %}
+ins_encode %{
-opcode(0x0F, 0x2E);
+__ ucomiss($src1$$XMMRegister, $src2$$XMMRegister);
-ins_encode(REX_reg_reg(src1, src2), OpcP, OpcS, reg_reg(src1, src2),
+emit_cmpfp3(_masm, $dst$$Register);
-cmpfp3(dst));
+%}
 ins_pipe(pipe_slow);
 %}
 // Compare into -1,0,1
 instruct cmpF_mem(rRegI dst, regF src1, memory src2, rFlagsReg cr)
 "jp,s    done\n\t"
 "jb,s    done\n\t"
 "setne   $dst\n\t"
 "movzbl  $dst, $dst\n"
 "done:" %}
+ins_encode %{
-opcode(0x0F, 0x2E);
+__ ucomiss($src1$$XMMRegister, $src2$$Address);
-ins_encode(REX_reg_mem(src1, src2), OpcP, OpcS, reg_mem(src1, src2),
+emit_cmpfp3(_masm, $dst$$Register);
-cmpfp3(dst));
+%}
 ins_pipe(pipe_slow);
 %}
 // Compare into -1,0,1
 instruct cmpF_imm(rRegI dst, regF src, immF con, rFlagsReg cr) %{
 "jb,s    done\n\t"
 "setne   $dst\n\t"
 "movzbl  $dst, $dst\n"
 "done:" %}
 ins_encode %{
-Label L_done;
-Register Rdst = $dst$$Register;
 __ ucomiss($src$$XMMRegister, $constantaddress($con));
-__ movl(Rdst, -1);
+emit_cmpfp3(_masm, $dst$$Register);
-__ jcc(Assembler::parity, L_done);
-__ jcc(Assembler::below, L_done);
-__ setb(Assembler::notEqual, Rdst);
-__ movzbl(Rdst, Rdst);
-__ bind(L_done);
 %}
 ins_pipe(pipe_slow);
 %}
 // Compare into -1,0,1
 "jp,s    done\n\t"
 "jb,s    done\n\t"
 "setne   $dst\n\t"
 "movzbl  $dst, $dst\n"
 "done:" %}
+ins_encode %{
-opcode(0x66, 0x0F, 0x2E);
+__ ucomisd($src1$$XMMRegister, $src2$$XMMRegister);
-ins_encode(OpcP, REX_reg_reg(src1, src2), OpcS, OpcT, reg_reg(src1, src2),
+emit_cmpfp3(_masm, $dst$$Register);
-cmpfp3(dst));
+%}
 ins_pipe(pipe_slow);
 %}
 // Compare into -1,0,1
 instruct cmpD_mem(rRegI dst, regD src1, memory src2, rFlagsReg cr)
 "jp,s    done\n\t"
 "jb,s    done\n\t"
 "setne   $dst\n\t"
 "movzbl  $dst, $dst\n"
 "done:" %}
+ins_encode %{
-opcode(0x66, 0x0F, 0x2E);
+__ ucomisd($src1$$XMMRegister, $src2$$Address);
-ins_encode(OpcP, REX_reg_mem(src1, src2), OpcS, OpcT, reg_mem(src1, src2),
+emit_cmpfp3(_masm, $dst$$Register);
-cmpfp3(dst));
+%}
 ins_pipe(pipe_slow);
 %}
 // Compare into -1,0,1
 instruct cmpD_imm(rRegI dst, regD src, immD con, rFlagsReg cr) %{
 "jb,s    done\n\t"
 "setne   $dst\n\t"
 "movzbl  $dst, $dst\n"
 "done:" %}
 ins_encode %{
-Register Rdst = $dst$$Register;
-Label L_done;
 __ ucomisd($src$$XMMRegister, $constantaddress($con));
-__ movl(Rdst, -1);
+emit_cmpfp3(_masm, $dst$$Register);
-__ jcc(Assembler::parity, L_done);
+%}
-__ jcc(Assembler::below, L_done);
-__ setb(Assembler::notEqual, Rdst);
-__ movzbl(Rdst, Rdst);
-__ bind(L_done);
-%}
-ins_pipe(pipe_slow);
-%}
-instruct addF_reg(regF dst, regF src)
-%{
-match(Set dst (AddF dst src));
-format %{ "addss   $dst, $src" %}
-ins_cost(150); // XXX
-opcode(0xF3, 0x0F, 0x58);
-ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
-ins_pipe(pipe_slow);
-%}
-instruct addF_mem(regF dst, memory src)
-%{
-match(Set dst (AddF dst (LoadF src)));
-format %{ "addss   $dst, $src" %}
-ins_cost(150); // XXX
-opcode(0xF3, 0x0F, 0x58);
-ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
-ins_pipe(pipe_slow);
-%}
-instruct addF_imm(regF dst, immF con) %{
-match(Set dst (AddF dst con));
-format %{ "addss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
-ins_cost(150); // XXX
-ins_encode %{
-__ addss($dst$$XMMRegister, $constantaddress($con));
-%}
-ins_pipe(pipe_slow);
-%}
-instruct addD_reg(regD dst, regD src)
-%{
-match(Set dst (AddD dst src));
-format %{ "addsd   $dst, $src" %}
-ins_cost(150); // XXX
-opcode(0xF2, 0x0F, 0x58);
-ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
-ins_pipe(pipe_slow);
-%}
-instruct addD_mem(regD dst, memory src)
-%{
-match(Set dst (AddD dst (LoadD src)));
-format %{ "addsd   $dst, $src" %}
-ins_cost(150); // XXX
-opcode(0xF2, 0x0F, 0x58);
-ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
-ins_pipe(pipe_slow);
-%}
-instruct addD_imm(regD dst, immD con) %{
-match(Set dst (AddD dst con));
-format %{ "addsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
-ins_cost(150); // XXX
-ins_encode %{
-__ addsd($dst$$XMMRegister, $constantaddress($con));
-%}
-ins_pipe(pipe_slow);
-%}
-instruct subF_reg(regF dst, regF src)
-%{
-match(Set dst (SubF dst src));
-format %{ "subss   $dst, $src" %}
-ins_cost(150); // XXX
-opcode(0xF3, 0x0F, 0x5C);
-ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
-ins_pipe(pipe_slow);
-%}
-instruct subF_mem(regF dst, memory src)
-%{
-match(Set dst (SubF dst (LoadF src)));
-format %{ "subss   $dst, $src" %}
-ins_cost(150); // XXX
-opcode(0xF3, 0x0F, 0x5C);
-ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
-ins_pipe(pipe_slow);
-%}
-instruct subF_imm(regF dst, immF con) %{
-match(Set dst (SubF dst con));
-format %{ "subss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
-ins_cost(150); // XXX
-ins_encode %{
-__ subss($dst$$XMMRegister, $constantaddress($con));
-%}
-ins_pipe(pipe_slow);
-%}
-instruct subD_reg(regD dst, regD src)
-%{
-match(Set dst (SubD dst src));
-format %{ "subsd   $dst, $src" %}
-ins_cost(150); // XXX
-opcode(0xF2, 0x0F, 0x5C);
-ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
-ins_pipe(pipe_slow);
-%}
-instruct subD_mem(regD dst, memory src)
-%{
-match(Set dst (SubD dst (LoadD src)));
-format %{ "subsd   $dst, $src" %}
-ins_cost(150); // XXX
-opcode(0xF2, 0x0F, 0x5C);
-ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
-ins_pipe(pipe_slow);
-%}
-instruct subD_imm(regD dst, immD con) %{
-match(Set dst (SubD dst con));
-format %{ "subsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
-ins_cost(150); // XXX
-ins_encode %{
-__ subsd($dst$$XMMRegister, $constantaddress($con));
-%}
-ins_pipe(pipe_slow);
-%}
-instruct mulF_reg(regF dst, regF src)
-%{
-match(Set dst (MulF dst src));
-format %{ "mulss   $dst, $src" %}
-ins_cost(150); // XXX
-opcode(0xF3, 0x0F, 0x59);
-ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
-ins_pipe(pipe_slow);
-%}
-instruct mulF_mem(regF dst, memory src)
-%{
-match(Set dst (MulF dst (LoadF src)));
-format %{ "mulss   $dst, $src" %}
-ins_cost(150); // XXX
-opcode(0xF3, 0x0F, 0x59);
-ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
-ins_pipe(pipe_slow);
-%}
-instruct mulF_imm(regF dst, immF con) %{
-match(Set dst (MulF dst con));
-format %{ "mulss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
-ins_cost(150); // XXX
-ins_encode %{
-__ mulss($dst$$XMMRegister, $constantaddress($con));
-%}
-ins_pipe(pipe_slow);
-%}
-instruct mulD_reg(regD dst, regD src)
-%{
-match(Set dst (MulD dst src));
-format %{ "mulsd   $dst, $src" %}
-ins_cost(150); // XXX
-opcode(0xF2, 0x0F, 0x59);
-ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
-ins_pipe(pipe_slow);
-%}
-instruct mulD_mem(regD dst, memory src)
-%{
-match(Set dst (MulD dst (LoadD src)));
-format %{ "mulsd   $dst, $src" %}
-ins_cost(150); // XXX
-opcode(0xF2, 0x0F, 0x59);
-ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
-ins_pipe(pipe_slow);
-%}
-instruct mulD_imm(regD dst, immD con) %{
-match(Set dst (MulD dst con));
-format %{ "mulsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
-ins_cost(150); // XXX
-ins_encode %{
-__ mulsd($dst$$XMMRegister, $constantaddress($con));
-%}
-ins_pipe(pipe_slow);
-%}
-instruct divF_reg(regF dst, regF src)
-%{
-match(Set dst (DivF dst src));
-format %{ "divss   $dst, $src" %}
-ins_cost(150); // XXX
-opcode(0xF3, 0x0F, 0x5E);
-ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
-ins_pipe(pipe_slow);
-%}
-instruct divF_mem(regF dst, memory src)
-%{
-match(Set dst (DivF dst (LoadF src)));
-format %{ "divss   $dst, $src" %}
-ins_cost(150); // XXX
-opcode(0xF3, 0x0F, 0x5E);
-ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
-ins_pipe(pipe_slow);
-%}
-instruct divF_imm(regF dst, immF con) %{
-match(Set dst (DivF dst con));
-format %{ "divss   $dst, [$constantaddress]\t# load from constant table: float=$con" %}
-ins_cost(150); // XXX
-ins_encode %{
-__ divss($dst$$XMMRegister, $constantaddress($con));
-%}
-ins_pipe(pipe_slow);
-%}
-instruct divD_reg(regD dst, regD src)
-%{
-match(Set dst (DivD dst src));
-format %{ "divsd   $dst, $src" %}
-ins_cost(150); // XXX
-opcode(0xF2, 0x0F, 0x5E);
-ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
-ins_pipe(pipe_slow);
-%}
-instruct divD_mem(regD dst, memory src)
-%{
-match(Set dst (DivD dst (LoadD src)));
-format %{ "divsd   $dst, $src" %}
-ins_cost(150); // XXX
-opcode(0xF2, 0x0F, 0x5E);
-ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
-ins_pipe(pipe_slow);
-%}
-instruct divD_imm(regD dst, immD con) %{
-match(Set dst (DivD dst con));
-format %{ "divsd   $dst, [$constantaddress]\t# load from constant table: double=$con" %}
-ins_cost(150); // XXX
-ins_encode %{
-__ divsd($dst$$XMMRegister, $constantaddress($con));
-%}
-ins_pipe(pipe_slow);
-%}
-instruct sqrtF_reg(regF dst, regF src)
-%{
-match(Set dst (ConvD2F (SqrtD (ConvF2D src))));
-format %{ "sqrtss  $dst, $src" %}
-ins_cost(150); // XXX
-opcode(0xF3, 0x0F, 0x51);
-ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
-ins_pipe(pipe_slow);
-%}
-instruct sqrtF_mem(regF dst, memory src)
-%{
-match(Set dst (ConvD2F (SqrtD (ConvF2D (LoadF src)))));
-format %{ "sqrtss  $dst, $src" %}
-ins_cost(150); // XXX
-opcode(0xF3, 0x0F, 0x51);
-ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
-ins_pipe(pipe_slow);
-%}
-instruct sqrtF_imm(regF dst, immF con) %{
-match(Set dst (ConvD2F (SqrtD (ConvF2D con))));
-format %{ "sqrtss  $dst, [$constantaddress]\t# load from constant table: float=$con" %}
-ins_cost(150); // XXX
-ins_encode %{
-__ sqrtss($dst$$XMMRegister, $constantaddress($con));
-%}
-ins_pipe(pipe_slow);
-%}
-instruct sqrtD_reg(regD dst, regD src)
-%{
-match(Set dst (SqrtD src));
-format %{ "sqrtsd  $dst, $src" %}
-ins_cost(150); // XXX
-opcode(0xF2, 0x0F, 0x51);
-ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
-ins_pipe(pipe_slow);
-%}
-instruct sqrtD_mem(regD dst, memory src)
-%{
-match(Set dst (SqrtD (LoadD src)));
-format %{ "sqrtsd  $dst, $src" %}
-ins_cost(150); // XXX
-opcode(0xF2, 0x0F, 0x51);
-ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
-ins_pipe(pipe_slow);
-%}
-instruct sqrtD_imm(regD dst, immD con) %{
-match(Set dst (SqrtD con));
-format %{ "sqrtsd  $dst, [$constantaddress]\t# load from constant table: double=$con" %}
-ins_cost(150); // XXX
-ins_encode %{
-__ sqrtsd($dst$$XMMRegister, $constantaddress($con));
-%}
-ins_pipe(pipe_slow);
-%}
-instruct absF_reg(regF dst)
-%{
-match(Set dst (AbsF dst));
-format %{ "andps   $dst, [0x7fffffff]\t# abs float by sign masking" %}
-ins_encode(absF_encoding(dst));
-ins_pipe(pipe_slow);
-%}
-instruct absD_reg(regD dst)
-%{
-match(Set dst (AbsD dst));
-format %{ "andpd   $dst, [0x7fffffffffffffff]\t"
-"# abs double by sign masking" %}
-ins_encode(absD_encoding(dst));
-ins_pipe(pipe_slow);
-%}
-instruct negF_reg(regF dst)
-%{
-match(Set dst (NegF dst));
-format %{ "xorps   $dst, [0x80000000]\t# neg float by sign flipping" %}
-ins_encode(negF_encoding(dst));
-ins_pipe(pipe_slow);
-%}
-instruct negD_reg(regD dst)
-%{
-match(Set dst (NegD dst));
-format %{ "xorpd   $dst, [0x8000000000000000]\t"
-"# neg double by sign flipping" %}
-ins_encode(negD_encoding(dst));
 ins_pipe(pipe_slow);
 %}
 // -----------Trig and Trancendental Instructions------------------------------
 instruct cosD_reg(regD dst) %{
 instruct convF2D_reg_reg(regD dst, regF src)
 %{
 match(Set dst (ConvF2D src));
 format %{ "cvtss2sd $dst, $src" %}
-opcode(0xF3, 0x0F, 0x5A);
+ins_encode %{
-ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
+__ cvtss2sd ($dst$$XMMRegister, $src$$XMMRegister);
+%}
 ins_pipe(pipe_slow); // XXX
 %}
 instruct convF2D_reg_mem(regD dst, memory src)
 %{
 match(Set dst (ConvF2D (LoadF src)));
 format %{ "cvtss2sd $dst, $src" %}
-opcode(0xF3, 0x0F, 0x5A);
+ins_encode %{
-ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
+__ cvtss2sd ($dst$$XMMRegister, $src$$Address);
+%}
 ins_pipe(pipe_slow); // XXX
 %}
 instruct convD2F_reg_reg(regF dst, regD src)
 %{
 match(Set dst (ConvD2F src));
 format %{ "cvtsd2ss $dst, $src" %}
-opcode(0xF2, 0x0F, 0x5A);
+ins_encode %{
-ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
+__ cvtsd2ss ($dst$$XMMRegister, $src$$XMMRegister);
+%}
 ins_pipe(pipe_slow); // XXX
 %}
 instruct convD2F_reg_mem(regF dst, memory src)
 %{
 match(Set dst (ConvD2F (LoadD src)));
 format %{ "cvtsd2ss $dst, $src" %}
-opcode(0xF2, 0x0F, 0x5A);
+ins_encode %{
-ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
+__ cvtsd2ss ($dst$$XMMRegister, $src$$Address);
+%}
 ins_pipe(pipe_slow); // XXX
 %}
 // XXX do mem variants
 instruct convF2I_reg_reg(rRegI dst, regF src, rFlagsReg cr)
 "subq    rsp, #8\n\t"
 "movss   [rsp], $src\n\t"
 "call    f2i_fixup\n\t"
 "popq    $dst\n"
 "done:   "%}
-opcode(0xF3, 0x0F, 0x2C);
+ins_encode %{
-ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src),
+Label done;
-f2i_fixup(dst, src));
+__ cvttss2sil($dst$$Register, $src$$XMMRegister);
+__ cmpl($dst$$Register, 0x80000000);
+__ jccb(Assembler::notEqual, done);
+__ subptr(rsp, 8);
+__ movflt(Address(rsp, 0), $src$$XMMRegister);
+__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2i_fixup())));
+__ pop($dst$$Register);
+__ bind(done);
+%}
 ins_pipe(pipe_slow);
 %}
 instruct convF2L_reg_reg(rRegL dst, regF src, rFlagsReg cr)
 %{
 "subq    rsp, #8\n\t"
 "movss   [rsp], $src\n\t"
 "call    f2l_fixup\n\t"
 "popq    $dst\n"
 "done:   "%}
-opcode(0xF3, 0x0F, 0x2C);
+ins_encode %{
-ins_encode(OpcP, REX_reg_reg_wide(dst, src), OpcS, OpcT, reg_reg(dst, src),
+Label done;
-f2l_fixup(dst, src));
+__ cvttss2siq($dst$$Register, $src$$XMMRegister);
+__ cmp64($dst$$Register,
+ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
+__ jccb(Assembler::notEqual, done);
+__ subptr(rsp, 8);
+__ movflt(Address(rsp, 0), $src$$XMMRegister);
+__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::f2l_fixup())));
+__ pop($dst$$Register);
+__ bind(done);
+%}
 ins_pipe(pipe_slow);
 %}
 instruct convD2I_reg_reg(rRegI dst, regD src, rFlagsReg cr)
 %{
 "subq    rsp, #8\n\t"
 "movsd   [rsp], $src\n\t"
 "call    d2i_fixup\n\t"
 "popq    $dst\n"
 "done:   "%}
-opcode(0xF2, 0x0F, 0x2C);
+ins_encode %{
-ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src),
+Label done;
-d2i_fixup(dst, src));
+__ cvttsd2sil($dst$$Register, $src$$XMMRegister);
+__ cmpl($dst$$Register, 0x80000000);
+__ jccb(Assembler::notEqual, done);
+__ subptr(rsp, 8);
+__ movdbl(Address(rsp, 0), $src$$XMMRegister);
+__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2i_fixup())));
+__ pop($dst$$Register);
+__ bind(done);
+%}
 ins_pipe(pipe_slow);
 %}
 instruct convD2L_reg_reg(rRegL dst, regD src, rFlagsReg cr)
 %{
 "subq    rsp, #8\n\t"
 "movsd   [rsp], $src\n\t"
 "call    d2l_fixup\n\t"
 "popq    $dst\n"
 "done:   "%}
-opcode(0xF2, 0x0F, 0x2C);
+ins_encode %{
-ins_encode(OpcP, REX_reg_reg_wide(dst, src), OpcS, OpcT, reg_reg(dst, src),
+Label done;
-d2l_fixup(dst, src));
+__ cvttsd2siq($dst$$Register, $src$$XMMRegister);
+__ cmp64($dst$$Register,
+ExternalAddress((address) StubRoutines::x86::double_sign_flip()));
+__ jccb(Assembler::notEqual, done);
+__ subptr(rsp, 8);
+__ movdbl(Address(rsp, 0), $src$$XMMRegister);
+__ call(RuntimeAddress(CAST_FROM_FN_PTR(address, StubRoutines::x86::d2l_fixup())));
+__ pop($dst$$Register);
+__ bind(done);
+%}
 ins_pipe(pipe_slow);
 %}
 instruct convI2F_reg_reg(regF dst, rRegI src)
 %{
 predicate(!UseXmmI2F);
 match(Set dst (ConvI2F src));
 format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
-opcode(0xF3, 0x0F, 0x2A);
+ins_encode %{
-ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
+__ cvtsi2ssl ($dst$$XMMRegister, $src$$Register);
+%}
 ins_pipe(pipe_slow); // XXX
 %}
 instruct convI2F_reg_mem(regF dst, memory src)
 %{
 match(Set dst (ConvI2F (LoadI src)));
 format %{ "cvtsi2ssl $dst, $src\t# i2f" %}
-opcode(0xF3, 0x0F, 0x2A);
+ins_encode %{
-ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
+__ cvtsi2ssl ($dst$$XMMRegister, $src$$Address);
+%}
 ins_pipe(pipe_slow); // XXX
 %}
 instruct convI2D_reg_reg(regD dst, rRegI src)
 %{
 predicate(!UseXmmI2D);
 match(Set dst (ConvI2D src));
 format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
-opcode(0xF2, 0x0F, 0x2A);
+ins_encode %{
-ins_encode(OpcP, REX_reg_reg(dst, src), OpcS, OpcT, reg_reg(dst, src));
+__ cvtsi2sdl ($dst$$XMMRegister, $src$$Register);
+%}
 ins_pipe(pipe_slow); // XXX
 %}
 instruct convI2D_reg_mem(regD dst, memory src)
 %{
 match(Set dst (ConvI2D (LoadI src)));
 format %{ "cvtsi2sdl $dst, $src\t# i2d" %}
-opcode(0xF2, 0x0F, 0x2A);
+ins_encode %{
-ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
+__ cvtsi2sdl ($dst$$XMMRegister, $src$$Address);
+%}
 ins_pipe(pipe_slow); // XXX
 %}
 instruct convXI2F_reg(regF dst, rRegI src)
 %{
 instruct convL2F_reg_reg(regF dst, rRegL src)
 %{
 match(Set dst (ConvL2F src));
 format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
-opcode(0xF3, 0x0F, 0x2A);
+ins_encode %{
-ins_encode(OpcP, REX_reg_reg_wide(dst, src), OpcS, OpcT, reg_reg(dst, src));
+__ cvtsi2ssq ($dst$$XMMRegister, $src$$Register);
+%}
 ins_pipe(pipe_slow); // XXX
 %}
 instruct convL2F_reg_mem(regF dst, memory src)
 %{
 match(Set dst (ConvL2F (LoadL src)));
 format %{ "cvtsi2ssq $dst, $src\t# l2f" %}
-opcode(0xF3, 0x0F, 0x2A);
+ins_encode %{
-ins_encode(OpcP, REX_reg_mem_wide(dst, src), OpcS, OpcT, reg_mem(dst, src));
+__ cvtsi2ssq ($dst$$XMMRegister, $src$$Address);
+%}
 ins_pipe(pipe_slow); // XXX
 %}
 instruct convL2D_reg_reg(regD dst, rRegL src)
 %{
 match(Set dst (ConvL2D src));
 format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
-opcode(0xF2, 0x0F, 0x2A);
+ins_encode %{
-ins_encode(OpcP, REX_reg_reg_wide(dst, src), OpcS, OpcT, reg_reg(dst, src));
+__ cvtsi2sdq ($dst$$XMMRegister, $src$$Register);
+%}
 ins_pipe(pipe_slow); // XXX
 %}
 instruct convL2D_reg_mem(regD dst, memory src)
 %{
 match(Set dst (ConvL2D (LoadL src)));
 format %{ "cvtsi2sdq $dst, $src\t# l2d" %}
-opcode(0xF2, 0x0F, 0x2A);
+ins_encode %{
-ins_encode(OpcP, REX_reg_mem_wide(dst, src), OpcS, OpcT, reg_mem(dst, src));
+__ cvtsi2sdq ($dst$$XMMRegister, $src$$Address);
+%}
 ins_pipe(pipe_slow); // XXX
 %}
 instruct convI2L_reg_reg(rRegL dst, rRegI src)
 %{
 instruct convI2L_reg_reg_zex(rRegL dst, rRegI src, immL_32bits mask)
 %{
 match(Set dst (AndL (ConvI2L src) mask));
 format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
-ins_encode(enc_copy(dst, src));
+ins_encode %{
+if ($dst$$reg != $src$$reg) {
+__ movl($dst$$Register, $src$$Register);
+}
+%}
 ins_pipe(ialu_reg_reg);
 %}
 // Zero-extend convert int to long
 instruct convI2L_reg_mem_zex(rRegL dst, memory src, immL_32bits mask)
 %{
 match(Set dst (AndL (ConvI2L (LoadI src)) mask));
 format %{ "movl    $dst, $src\t# i2l zero-extend\n\t" %}
-opcode(0x8B);
+ins_encode %{
-ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
+__ movl($dst$$Register, $src$$Address);
+%}
 ins_pipe(ialu_reg_mem);
 %}
 instruct zerox_long_reg_reg(rRegL dst, rRegL src, immL_32bits mask)
 %{
 match(Set dst (AndL src mask));
 format %{ "movl    $dst, $src\t# zero-extend long" %}
-ins_encode(enc_copy_always(dst, src));
+ins_encode %{
+__ movl($dst$$Register, $src$$Register);
+%}
 ins_pipe(ialu_reg_reg);
 %}
 instruct convL2I_reg_reg(rRegI dst, rRegL src)
 %{
 match(Set dst (ConvL2I src));
 format %{ "movl    $dst, $src\t# l2i" %}
-ins_encode(enc_copy_always(dst, src));
+ins_encode %{
+__ movl($dst$$Register, $src$$Register);
+%}
 ins_pipe(ialu_reg_reg);
 %}
 instruct MoveF2I_stack_reg(rRegI dst, stackSlotF src) %{
 match(Set dst (MoveF2I src));
 effect(DEF dst, USE src);
 ins_cost(125);
 format %{ "movl    $dst, $src\t# MoveF2I_stack_reg" %}
-opcode(0x8B);
+ins_encode %{
-ins_encode(REX_reg_mem(dst, src), OpcP, reg_mem(dst, src));
+__ movl($dst$$Register, Address(rsp, $src$$disp));
+%}
 ins_pipe(ialu_reg_mem);
 %}
 instruct MoveI2F_stack_reg(regF dst, stackSlotI src) %{
 match(Set dst (MoveI2F src));
 effect(DEF dst, USE src);
 ins_cost(125);
 format %{ "movss   $dst, $src\t# MoveI2F_stack_reg" %}
-opcode(0xF3, 0x0F, 0x10);
+ins_encode %{
-ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
+__ movflt($dst$$XMMRegister, Address(rsp, $src$$disp));
+%}
 ins_pipe(pipe_slow);
 %}
 instruct MoveD2L_stack_reg(rRegL dst, stackSlotD src) %{
 match(Set dst (MoveD2L src));
 effect(DEF dst, USE src);
 ins_cost(125);
 format %{ "movq    $dst, $src\t# MoveD2L_stack_reg" %}
-opcode(0x8B);
+ins_encode %{
-ins_encode(REX_reg_mem_wide(dst, src), OpcP, reg_mem(dst, src));
+__ movq($dst$$Register, Address(rsp, $src$$disp));
+%}
 ins_pipe(ialu_reg_mem);
 %}
 instruct MoveL2D_stack_reg_partial(regD dst, stackSlotL src) %{
 predicate(!UseXmmLoadAndClearUpper);
 match(Set dst (MoveL2D src));
 effect(DEF dst, USE src);
 ins_cost(125);
 format %{ "movlpd  $dst, $src\t# MoveL2D_stack_reg" %}
-opcode(0x66, 0x0F, 0x12);
+ins_encode %{
-ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
+__ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
+%}
 ins_pipe(pipe_slow);
 %}
 instruct MoveL2D_stack_reg(regD dst, stackSlotL src) %{
 predicate(UseXmmLoadAndClearUpper);
 match(Set dst (MoveL2D src));
 effect(DEF dst, USE src);
 ins_cost(125);
 format %{ "movsd   $dst, $src\t# MoveL2D_stack_reg" %}
-opcode(0xF2, 0x0F, 0x10);
+ins_encode %{
-ins_encode(OpcP, REX_reg_mem(dst, src), OpcS, OpcT, reg_mem(dst, src));
+__ movdbl($dst$$XMMRegister, Address(rsp, $src$$disp));
+%}
 ins_pipe(pipe_slow);
 %}
 instruct MoveF2I_reg_stack(stackSlotI dst, regF src) %{
 match(Set dst (MoveF2I src));
 effect(DEF dst, USE src);
 ins_cost(95); // XXX
 format %{ "movss   $dst, $src\t# MoveF2I_reg_stack" %}
-opcode(0xF3, 0x0F, 0x11);
+ins_encode %{
-ins_encode(OpcP, REX_reg_mem(src, dst), OpcS, OpcT, reg_mem(src, dst));
+__ movflt(Address(rsp, $dst$$disp), $src$$XMMRegister);
+%}
 ins_pipe(pipe_slow);
 %}
 instruct MoveI2F_reg_stack(stackSlotF dst, rRegI src) %{
 match(Set dst (MoveI2F src));
 effect(DEF dst, USE src);
 ins_cost(100);
 format %{ "movl    $dst, $src\t# MoveI2F_reg_stack" %}
-opcode(0x89);
+ins_encode %{
-ins_encode(REX_reg_mem(src, dst), OpcP, reg_mem(src, dst));
+__ movl(Address(rsp, $dst$$disp), $src$$Register);
+%}
 ins_pipe( ialu_mem_reg );
 %}
 instruct MoveD2L_reg_stack(stackSlotL dst, regD src) %{
 match(Set dst (MoveD2L src));
 effect(DEF dst, USE src);
 ins_cost(95); // XXX
 format %{ "movsd   $dst, $src\t# MoveL2D_reg_stack" %}
-opcode(0xF2, 0x0F, 0x11);
+ins_encode %{
-ins_encode(OpcP, REX_reg_mem(src, dst), OpcS, OpcT, reg_mem(src, dst));
+__ movdbl(Address(rsp, $dst$$disp), $src$$XMMRegister);
+%}
 ins_pipe(pipe_slow);
 %}
 instruct MoveL2D_reg_stack(stackSlotD dst, rRegL src) %{
 match(Set dst (MoveL2D src));
 effect(DEF dst, USE src);
 ins_cost(100);
 format %{ "movq    $dst, $src\t# MoveL2D_reg_stack" %}
-opcode(0x89);
+ins_encode %{
-ins_encode(REX_reg_mem_wide(src, dst), OpcP, reg_mem(src, dst));
+__ movq(Address(rsp, $dst$$disp), $src$$Register);
+%}
 ins_pipe(ialu_mem_reg);
 %}
 instruct MoveF2I_reg_reg(rRegI dst, regF src) %{
 match(Set dst (MoveF2I src));
 effect(DEF dst, USE src);
 ins_cost(85);
 format %{ "movd    $dst,$src\t# MoveF2I" %}
-ins_encode %{ __ movdl($dst$$Register, $src$$XMMRegister); %}
+ins_encode %{
+__ movdl($dst$$Register, $src$$XMMRegister);
+%}
 ins_pipe( pipe_slow );
 %}
 instruct MoveD2L_reg_reg(rRegL dst, regD src) %{
 match(Set dst (MoveD2L src));
 effect(DEF dst, USE src);
 ins_cost(85);
 format %{ "movd    $dst,$src\t# MoveD2L" %}
-ins_encode %{ __ movdq($dst$$Register, $src$$XMMRegister); %}
+ins_encode %{
+__ movdq($dst$$Register, $src$$XMMRegister);
+%}
 ins_pipe( pipe_slow );
 %}
 // The next instructions have long latency and use Int unit. Set high cost.
 instruct MoveI2F_reg_reg(regF dst, rRegI src) %{
 match(Set dst (MoveI2F src));
 effect(DEF dst, USE src);
 ins_cost(300);
 format %{ "movd    $dst,$src\t# MoveI2F" %}
-ins_encode %{ __ movdl($dst$$XMMRegister, $src$$Register); %}
+ins_encode %{
+__ movdl($dst$$XMMRegister, $src$$Register);
+%}
 ins_pipe( pipe_slow );
 %}
 instruct MoveL2D_reg_reg(regD dst, rRegL src) %{
 match(Set dst (MoveL2D src));
 effect(DEF dst, USE src);
 ins_cost(300);
 format %{ "movd    $dst,$src\t# MoveL2D" %}
-ins_encode %{ __ movdq($dst$$XMMRegister, $src$$Register); %}
+ins_encode %{
+__ movdq($dst$$XMMRegister, $src$$Register);
+%}
 ins_pipe( pipe_slow );
 %}
 // Replicate scalar to packed byte (1 byte) values in xmm
 instruct Repl8B_reg(regD dst, regD src) %{
 match(Set dst (Replicate8B src));
 format %{ "MOVDQA  $dst,$src\n\t"
 "PUNPCKLBW $dst,$dst\n\t"
 "PSHUFLW $dst,$dst,0x00\t! replicate8B" %}
-ins_encode( pshufd_8x8(dst, src));
+ins_encode %{
+if ($dst$$reg != $src$$reg) {
+__ movdqa($dst$$XMMRegister, $src$$XMMRegister);
+}
+__ punpcklbw($dst$$XMMRegister, $dst$$XMMRegister);
+__ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
+%}
 ins_pipe( pipe_slow );
 %}
 // Replicate scalar to packed byte (1 byte) values in xmm
 instruct Repl8B_rRegI(regD dst, rRegI src) %{
 match(Set dst (Replicate8B src));
 format %{ "MOVD    $dst,$src\n\t"
 "PUNPCKLBW $dst,$dst\n\t"
 "PSHUFLW $dst,$dst,0x00\t! replicate8B" %}
-ins_encode( mov_i2x(dst, src), pshufd_8x8(dst, dst));
+ins_encode %{
+__ movdl($dst$$XMMRegister, $src$$Register);
+__ punpcklbw($dst$$XMMRegister, $dst$$XMMRegister);
+__ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
+%}
 ins_pipe( pipe_slow );
 %}
 // Replicate scalar zero to packed byte (1 byte) values in xmm
 instruct Repl8B_immI0(regD dst, immI0 zero) %{
 match(Set dst (Replicate8B zero));
 format %{ "PXOR  $dst,$dst\t! replicate8B" %}
-ins_encode( pxor(dst, dst));
+ins_encode %{
+__ pxor($dst$$XMMRegister, $dst$$XMMRegister);
+%}
 ins_pipe( fpu_reg_reg );
 %}
 // Replicate scalar to packed shore (2 byte) values in xmm
 instruct Repl4S_reg(regD dst, regD src) %{
 match(Set dst (Replicate4S src));
 format %{ "PSHUFLW $dst,$src,0x00\t! replicate4S" %}
-ins_encode( pshufd_4x16(dst, src));
+ins_encode %{
+__ pshuflw($dst$$XMMRegister, $src$$XMMRegister, 0x00);
+%}
 ins_pipe( fpu_reg_reg );
 %}
 // Replicate scalar to packed shore (2 byte) values in xmm
 instruct Repl4S_rRegI(regD dst, rRegI src) %{
 match(Set dst (Replicate4S src));
 format %{ "MOVD    $dst,$src\n\t"
 "PSHUFLW $dst,$dst,0x00\t! replicate4S" %}
-ins_encode( mov_i2x(dst, src), pshufd_4x16(dst, dst));
+ins_encode %{
+__ movdl($dst$$XMMRegister, $src$$Register);
+__ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
+%}
 ins_pipe( fpu_reg_reg );
 %}
 // Replicate scalar zero to packed short (2 byte) values in xmm
 instruct Repl4S_immI0(regD dst, immI0 zero) %{
 match(Set dst (Replicate4S zero));
 format %{ "PXOR  $dst,$dst\t! replicate4S" %}
-ins_encode( pxor(dst, dst));
+ins_encode %{
+__ pxor($dst$$XMMRegister, $dst$$XMMRegister);
+%}
 ins_pipe( fpu_reg_reg );
 %}
 // Replicate scalar to packed char (2 byte) values in xmm
 instruct Repl4C_reg(regD dst, regD src) %{
 match(Set dst (Replicate4C src));
 format %{ "PSHUFLW $dst,$src,0x00\t! replicate4C" %}
-ins_encode( pshufd_4x16(dst, src));
+ins_encode %{
+__ pshuflw($dst$$XMMRegister, $src$$XMMRegister, 0x00);
+%}
 ins_pipe( fpu_reg_reg );
 %}
 // Replicate scalar to packed char (2 byte) values in xmm
 instruct Repl4C_rRegI(regD dst, rRegI src) %{
 match(Set dst (Replicate4C src));
 format %{ "MOVD    $dst,$src\n\t"
 "PSHUFLW $dst,$dst,0x00\t! replicate4C" %}
-ins_encode( mov_i2x(dst, src), pshufd_4x16(dst, dst));
+ins_encode %{
+__ movdl($dst$$XMMRegister, $src$$Register);
+__ pshuflw($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
+%}
 ins_pipe( fpu_reg_reg );
 %}
 // Replicate scalar zero to packed char (2 byte) values in xmm
 instruct Repl4C_immI0(regD dst, immI0 zero) %{
 match(Set dst (Replicate4C zero));
 format %{ "PXOR  $dst,$dst\t! replicate4C" %}
-ins_encode( pxor(dst, dst));
+ins_encode %{
+__ pxor($dst$$XMMRegister, $dst$$XMMRegister);
+%}
 ins_pipe( fpu_reg_reg );
 %}
 // Replicate scalar to packed integer (4 byte) values in xmm
 instruct Repl2I_reg(regD dst, regD src) %{
 match(Set dst (Replicate2I src));
 format %{ "PSHUFD $dst,$src,0x00\t! replicate2I" %}
-ins_encode( pshufd(dst, src, 0x00));
+ins_encode %{
+__ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0x00);
+%}
 ins_pipe( fpu_reg_reg );
 %}
 // Replicate scalar to packed integer (4 byte) values in xmm
 instruct Repl2I_rRegI(regD dst, rRegI src) %{
 match(Set dst (Replicate2I src));
 format %{ "MOVD   $dst,$src\n\t"
 "PSHUFD $dst,$dst,0x00\t! replicate2I" %}
-ins_encode( mov_i2x(dst, src), pshufd(dst, dst, 0x00));
+ins_encode %{
+__ movdl($dst$$XMMRegister, $src$$Register);
+__ pshufd($dst$$XMMRegister, $dst$$XMMRegister, 0x00);
+%}
 ins_pipe( fpu_reg_reg );
 %}
 // Replicate scalar zero to packed integer (2 byte) values in xmm
 instruct Repl2I_immI0(regD dst, immI0 zero) %{
 match(Set dst (Replicate2I zero));
 format %{ "PXOR  $dst,$dst\t! replicate2I" %}
-ins_encode( pxor(dst, dst));
+ins_encode %{
+__ pxor($dst$$XMMRegister, $dst$$XMMRegister);
+%}
 ins_pipe( fpu_reg_reg );
 %}
 // Replicate scalar to packed single precision floating point values in xmm
 instruct Repl2F_reg(regD dst, regD src) %{
 match(Set dst (Replicate2F src));
 format %{ "PSHUFD $dst,$src,0xe0\t! replicate2F" %}
-ins_encode( pshufd(dst, src, 0xe0));
+ins_encode %{
+__ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0xe0);
+%}
 ins_pipe( fpu_reg_reg );
 %}
 // Replicate scalar to packed single precision floating point values in xmm
 instruct Repl2F_regF(regD dst, regF src) %{
 match(Set dst (Replicate2F src));
 format %{ "PSHUFD $dst,$src,0xe0\t! replicate2F" %}
-ins_encode( pshufd(dst, src, 0xe0));
+ins_encode %{
+__ pshufd($dst$$XMMRegister, $src$$XMMRegister, 0xe0);
+%}
 ins_pipe( fpu_reg_reg );
 %}
 // Replicate scalar to packed single precision floating point values in xmm
 instruct Repl2F_immF0(regD dst, immF0 zero) %{
 match(Set dst (Replicate2F zero));
 format %{ "PXOR  $dst,$dst\t! replicate2F" %}
-ins_encode( pxor(dst, dst));
+ins_encode %{
+__ pxor($dst$$XMMRegister, $dst$$XMMRegister);
+%}
 ins_pipe( fpu_reg_reg );
 %}
 // =======================================================================
 %{
 match(Set result (PartialSubtypeCheck sub super));
 effect(KILL rcx, KILL cr);
 ins_cost(1100);  // slightly larger than the next version
-format %{ "movq    rdi, [$sub + (sizeof(oopDesc) + Klass::secondary_supers_offset_in_bytes())]\n\t"
+format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
 "movl    rcx, [rdi + arrayOopDesc::length_offset_in_bytes()]\t# length to scan\n\t"
 "addq    rdi, arrayOopDex::base_offset_in_bytes(T_OBJECT)\t# Skip to start of data; set NZ in case count is zero\n\t"
 "repne   scasq\t# Scan *rdi++ for a match with rax while rcx--\n\t"
 "jne,s   miss\t\t# Missed: rdi not-zero\n\t"
-"movq    [$sub + (sizeof(oopDesc) + Klass::secondary_super_cache_offset_in_bytes())], $super\t# Hit: update cache\n\t"
+"movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
 "xorq    $result, $result\t\t Hit: rdi zero\n\t"
 "miss:\t" %}
 opcode(0x1); // Force a XOR of RDI
 ins_encode(enc_PartialSubtypeCheck());
 %{
 match(Set cr (CmpP (PartialSubtypeCheck sub super) zero));
 effect(KILL rcx, KILL result);
 ins_cost(1000);
-format %{ "movq    rdi, [$sub + (sizeof(oopDesc) + Klass::secondary_supers_offset_in_bytes())]\n\t"
+format %{ "movq    rdi, [$sub + in_bytes(Klass::secondary_supers_offset())]\n\t"
 "movl    rcx, [rdi + arrayOopDesc::length_offset_in_bytes()]\t# length to scan\n\t"
 "addq    rdi, arrayOopDex::base_offset_in_bytes(T_OBJECT)\t# Skip to start of data; set NZ in case count is zero\n\t"
 "repne   scasq\t# Scan *rdi++ for a match with rax while cx-- != 0\n\t"
 "jne,s   miss\t\t# Missed: flags nz\n\t"
-"movq    [$sub + (sizeof(oopDesc) + Klass::secondary_super_cache_offset_in_bytes())], $super\t# Hit: update cache\n\t"
+"movq    [$sub + in_bytes(Klass::secondary_super_cache_offset())], $super\t# Hit: update cache\n\t"
 "miss:\t" %}
 opcode(0x0); // No need to XOR RDI
 ins_encode(enc_PartialSubtypeCheck());
 ins_pipe(pipe_slow);
 // ============================================================================
 // inlined locking and unlocking
 instruct cmpFastLock(rFlagsReg cr,
-rRegP object, rRegP box, rax_RegI tmp, rRegP scr)
+rRegP object, rbx_RegP box, rax_RegI tmp, rRegP scr)
 %{
 match(Set cr (FastLock object box));
-effect(TEMP tmp, TEMP scr);
+effect(TEMP tmp, TEMP scr, USE_KILL box);
 ins_cost(300);
-format %{ "fastlock $object,$box,$tmp,$scr" %}
+format %{ "fastlock $object,$box\t! kills $box,$tmp,$scr" %}
 ins_encode(Fast_Lock(object, box, tmp, scr));
 ins_pipe(pipe_slow);
 %}
 instruct cmpFastUnlock(rFlagsReg cr,
 rRegP object, rax_RegP box, rRegP tmp)
 %{
 match(Set cr (FastUnlock object box));
-effect(TEMP tmp);
+effect(TEMP tmp, USE_KILL box);
 ins_cost(300);
-format %{ "fastunlock $object, $box, $tmp" %}
+format %{ "fastunlock $object,$box\t! kills $box,$tmp" %}
 ins_encode(Fast_Unlock(object, box, tmp));
 ins_pipe(pipe_slow);
 %}
 // use the following format syntax
 format %{ "jmp     rethrow_stub" %}
 ins_encode(enc_rethrow);
 ins_pipe(pipe_jmp);
+%}
+// ============================================================================
+// This name is KNOWN by the ADLC and cannot be changed.
+// The ADLC forces a 'TypeRawPtr::BOTTOM' output type
+// for this guy.
+instruct tlsLoadP(r15_RegP dst) %{
+match(Set dst (ThreadLocal));
+effect(DEF dst);
+size(0);
+format %{ "# TLS is in R15" %}
+ins_encode( /*empty encoding*/ );
+ins_pipe(ialu_reg_reg);
 %}
 //----------PEEPHOLE RULES-----------------------------------------------------
 // These must follow all instruction definitions as they use the names

Mercurial > hg > truffle

comparison src/cpu/x86/vm/x86_64.ad @ 4970:33df1aeaebbf