Source code

001/*
002 * Copyright (c) 2009, 2015, Oracle and/or its affiliates. All rights reserved.
003 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
004 *
005 * This code is free software; you can redistribute it and/or modify it
006 * under the terms of the GNU General Public License version 2 only, as
007 * published by the Free Software Foundation.
008 *
009 * This code is distributed in the hope that it will be useful, but WITHOUT
010 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
011 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
012 * version 2 for more details (a copy is included in the LICENSE file that
013 * accompanied this code).
014 *
015 * You should have received a copy of the GNU General Public License version
016 * 2 along with this work; if not, write to the Free Software Foundation,
017 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
018 *
019 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
020 * or visit www.oracle.com if you need additional information or have any
021 * questions.
022 */
023
024package com.oracle.graal.compiler.amd64;
025
026import static com.oracle.graal.asm.amd64.AMD64Assembler.AMD64BinaryArithmetic.*;
027import static com.oracle.graal.asm.amd64.AMD64Assembler.AMD64MOp.*;
028import static com.oracle.graal.asm.amd64.AMD64Assembler.AMD64RMOp.*;
029import static com.oracle.graal.asm.amd64.AMD64Assembler.AMD64Shift.*;
030import static com.oracle.graal.asm.amd64.AMD64Assembler.OperandSize.*;
031import static com.oracle.graal.lir.amd64.AMD64Arithmetic.*;
032import static com.oracle.graal.lir.amd64.AMD64MathIntrinsicOp.IntrinsicOpcode.*;
033import static jdk.internal.jvmci.code.ValueUtil.*;
034
035import java.util.*;
036
037import jdk.internal.jvmci.amd64.*;
038import jdk.internal.jvmci.code.*;
039import jdk.internal.jvmci.common.*;
040import jdk.internal.jvmci.meta.*;
041
042import com.oracle.graal.asm.*;
043import com.oracle.graal.asm.amd64.AMD64Assembler.AMD64BinaryArithmetic;
044import com.oracle.graal.asm.amd64.AMD64Assembler.AMD64MIOp;
045import com.oracle.graal.asm.amd64.AMD64Assembler.AMD64MOp;
046import com.oracle.graal.asm.amd64.AMD64Assembler.AMD64MROp;
047import com.oracle.graal.asm.amd64.AMD64Assembler.AMD64RMIOp;
048import com.oracle.graal.asm.amd64.AMD64Assembler.AMD64RMOp;
049import com.oracle.graal.asm.amd64.AMD64Assembler.AMD64Shift;
050import com.oracle.graal.asm.amd64.AMD64Assembler.ConditionFlag;
051import com.oracle.graal.asm.amd64.AMD64Assembler.OperandSize;
052import com.oracle.graal.asm.amd64.AMD64Assembler.SSEOp;
053import com.oracle.graal.compiler.common.calc.*;
054import com.oracle.graal.compiler.common.spi.*;
055import com.oracle.graal.compiler.common.util.*;
056import com.oracle.graal.lir.*;
057import com.oracle.graal.lir.StandardOp.JumpOp;
058import com.oracle.graal.lir.amd64.*;
059import com.oracle.graal.lir.amd64.AMD64Arithmetic.FPDivRemOp;
060import com.oracle.graal.lir.amd64.AMD64ControlFlow.BranchOp;
061import com.oracle.graal.lir.amd64.AMD64ControlFlow.CondMoveOp;
062import com.oracle.graal.lir.amd64.AMD64ControlFlow.FloatBranchOp;
063import com.oracle.graal.lir.amd64.AMD64ControlFlow.FloatCondMoveOp;
064import com.oracle.graal.lir.amd64.AMD64ControlFlow.ReturnOp;
065import com.oracle.graal.lir.amd64.AMD64ControlFlow.StrategySwitchOp;
066import com.oracle.graal.lir.amd64.AMD64ControlFlow.TableSwitchOp;
067import com.oracle.graal.lir.amd64.AMD64Move.AMD64PushPopStackMove;
068import com.oracle.graal.lir.amd64.AMD64Move.AMD64StackMove;
069import com.oracle.graal.lir.amd64.AMD64Move.CompareAndSwapOp;
070import com.oracle.graal.lir.amd64.AMD64Move.LeaDataOp;
071import com.oracle.graal.lir.amd64.AMD64Move.LeaOp;
072import com.oracle.graal.lir.amd64.AMD64Move.MembarOp;
073import com.oracle.graal.lir.amd64.AMD64Move.MoveFromRegOp;
074import com.oracle.graal.lir.amd64.AMD64Move.MoveToRegOp;
075import com.oracle.graal.lir.amd64.AMD64Move.StackLeaOp;
076import com.oracle.graal.lir.framemap.*;
077import com.oracle.graal.lir.gen.*;
078import com.oracle.graal.phases.util.*;
079
080/**
081 * This class implements the AMD64 specific portion of the LIR generator.
082 */
083public abstract class AMD64LIRGenerator extends LIRGenerator implements AMD64ArithmeticLIRGenerator {
084
085    private static final RegisterValue RCX_I = AMD64.rcx.asValue(LIRKind.value(Kind.Int));
086    private AMD64SpillMoveFactory moveFactory;
087    private Map<PlatformKind.Key, RegisterBackupPair> categorized;
088
089    private static class RegisterBackupPair {
090        public final Register register;
091        public final StackSlotValue backupSlot;
092
093        RegisterBackupPair(Register register, StackSlotValue backupSlot) {
094            this.register = register;
095            this.backupSlot = backupSlot;
096        }
097    }
098
099    private class AMD64SpillMoveFactory extends SpillMoveFactoryBase {
100
101        @Override
102        protected LIRInstruction createMoveIntern(AllocatableValue result, Value input) {
103            return AMD64LIRGenerator.this.createMove(result, input);
104        }
105
106        @Override
107        protected LIRInstruction createStackMoveIntern(AllocatableValue result, Value input) {
108            return AMD64LIRGenerator.this.createStackMove(result, input);
109        }
110
111    }
112
113    public AMD64LIRGenerator(LIRKindTool lirKindTool, Providers providers, CallingConvention cc, LIRGenerationResult lirGenRes) {
114        super(lirKindTool, providers, cc, lirGenRes);
115    }
116
117    public SpillMoveFactory getSpillMoveFactory() {
118        if (moveFactory == null) {
119            moveFactory = new AMD64SpillMoveFactory();
120        }
121        return moveFactory;
122    }
123
124    @Override
125    public boolean canInlineConstant(JavaConstant c) {
126        switch (c.getKind()) {
127            case Long:
128                return NumUtil.isInt(c.asLong()) && !getCodeCache().needsDataPatch(c);
129            case Object:
130                return c.isNull();
131            default:
132                return true;
133        }
134    }
135
136    /**
137     * Checks whether the supplied constant can be used without loading it into a register for store
138     * operations, i.e., on the right hand side of a memory access.
139     *
140     * @param c The constant to check.
141     * @return True if the constant can be used directly, false if the constant needs to be in a
142     *         register.
143     */
144    protected final boolean canStoreConstant(JavaConstant c) {
145        // there is no immediate move of 64-bit constants on Intel
146        switch (c.getKind()) {
147            case Long:
148                return Util.isInt(c.asLong()) && !getCodeCache().needsDataPatch(c);
149            case Double:
150                return false;
151            case Object:
152                return c.isNull();
153            default:
154                return true;
155        }
156    }
157
158    protected AMD64LIRInstruction createMove(AllocatableValue dst, Value src) {
159        if (src instanceof AMD64AddressValue) {
160            return new LeaOp(dst, (AMD64AddressValue) src);
161        } else if (isRegister(src) || isStackSlotValue(dst)) {
162            return new MoveFromRegOp(dst.getKind(), dst, src);
163        } else {
164            return new MoveToRegOp(dst.getKind(), dst, src);
165        }
166    }
167
168    protected LIRInstruction createStackMove(AllocatableValue result, Value input) {
169        Kind kind = result.getKind();
170        OperandSize size;
171        switch (kind) {
172            case Long:
173            case Double:
174                size = QWORD;
175                break;
176            case Short:
177                size = WORD;
178                break;
179            default:
180                RegisterBackupPair backup = getScratchRegister(input.getPlatformKind());
181                Register scratchRegister = backup.register;
182                StackSlotValue backupSlot = backup.backupSlot;
183                return createStackMove(result, input, scratchRegister, backupSlot);
184        }
185        return new AMD64PushPopStackMove(size, result, input);
186    }
187
188    protected LIRInstruction createStackMove(AllocatableValue result, Value input, Register scratchRegister, StackSlotValue backupSlot) {
189        return new AMD64StackMove(result, input, scratchRegister, backupSlot);
190    }
191
192    protected RegisterBackupPair getScratchRegister(PlatformKind kind) {
193        PlatformKind.Key key = kind.getKey();
194        if (categorized == null) {
195            categorized = new HashMap<>();
196        } else if (categorized.containsKey(key)) {
197            return categorized.get(key);
198        }
199
200        FrameMapBuilder frameMapBuilder = getResult().getFrameMapBuilder();
201        RegisterConfig registerConfig = frameMapBuilder.getRegisterConfig();
202
203        Register[] availableRegister = registerConfig.filterAllocatableRegisters(kind, registerConfig.getAllocatableRegisters());
204        assert availableRegister != null && availableRegister.length > 1;
205        Register scratchRegister = availableRegister[0];
206
207        Architecture arch = frameMapBuilder.getCodeCache().getTarget().arch;
208        LIRKind largestKind = LIRKind.value(arch.getLargestStorableKind(scratchRegister.getRegisterCategory()));
209        VirtualStackSlot backupSlot = frameMapBuilder.allocateSpillSlot(largestKind);
210
211        RegisterBackupPair value = new RegisterBackupPair(scratchRegister, backupSlot);
212        categorized.put(key, value);
213
214        return value;
215    }
216
217    @Override
218    public void emitMove(AllocatableValue dst, Value src) {
219        append(createMove(dst, src));
220    }
221
222    public void emitData(AllocatableValue dst, byte[] data) {
223        append(new LeaDataOp(dst, data));
224    }
225
226    public AMD64AddressValue asAddressValue(Value address) {
227        if (address instanceof AMD64AddressValue) {
228            return (AMD64AddressValue) address;
229        } else {
230            if (address instanceof JavaConstant) {
231                long displacement = ((JavaConstant) address).asLong();
232                if (NumUtil.isInt(displacement)) {
233                    return new AMD64AddressValue(address.getLIRKind(), Value.ILLEGAL, (int) displacement);
234                }
235            }
236            return new AMD64AddressValue(address.getLIRKind(), asAllocatable(address), 0);
237        }
238    }
239
240    @Override
241    public Variable emitAddress(StackSlotValue address) {
242        Variable result = newVariable(LIRKind.value(target().wordKind));
243        append(new StackLeaOp(result, address));
244        return result;
245    }
246
247    private static LIRKind toStackKind(LIRKind kind) {
248        if (kind.getPlatformKind() instanceof Kind) {
249            Kind stackKind = ((Kind) kind.getPlatformKind()).getStackKind();
250            return kind.changeType(stackKind);
251        } else {
252            return kind;
253        }
254    }
255
256    @Override
257    public Variable emitLoad(LIRKind kind, Value address, LIRFrameState state) {
258        AMD64AddressValue loadAddress = asAddressValue(address);
259        Variable result = newVariable(toStackKind(kind));
260        switch ((Kind) kind.getPlatformKind()) {
261            case Boolean:
262                append(new AMD64Unary.MemoryOp(MOVZXB, DWORD, result, loadAddress, state));
263                break;
264            case Byte:
265                append(new AMD64Unary.MemoryOp(MOVSXB, DWORD, result, loadAddress, state));
266                break;
267            case Char:
268                append(new AMD64Unary.MemoryOp(MOVZX, DWORD, result, loadAddress, state));
269                break;
270            case Short:
271                append(new AMD64Unary.MemoryOp(MOVSX, DWORD, result, loadAddress, state));
272                break;
273            case Int:
274                append(new AMD64Unary.MemoryOp(MOV, DWORD, result, loadAddress, state));
275                break;
276            case Long:
277            case Object:
278                append(new AMD64Unary.MemoryOp(MOV, QWORD, result, loadAddress, state));
279                break;
280            case Float:
281                append(new AMD64Unary.MemoryOp(MOVSS, SS, result, loadAddress, state));
282                break;
283            case Double:
284                append(new AMD64Unary.MemoryOp(MOVSD, SD, result, loadAddress, state));
285                break;
286            default:
287                throw JVMCIError.shouldNotReachHere();
288        }
289        return result;
290    }
291
292    protected void emitStoreConst(Kind kind, AMD64AddressValue address, JavaConstant value, LIRFrameState state) {
293        if (value.isNull()) {
294            assert kind == Kind.Int || kind == Kind.Long || kind == Kind.Object;
295            OperandSize size = kind == Kind.Int ? DWORD : QWORD;
296            append(new AMD64BinaryConsumer.MemoryConstOp(AMD64MIOp.MOV, size, address, 0, state));
297        } else {
298            AMD64MIOp op = AMD64MIOp.MOV;
299            OperandSize size;
300            long imm;
301
302            switch (kind) {
303                case Boolean:
304                case Byte:
305                    op = AMD64MIOp.MOVB;
306                    size = BYTE;
307                    imm = value.asInt();
308                    break;
309                case Char:
310                case Short:
311                    size = WORD;
312                    imm = value.asInt();
313                    break;
314                case Int:
315                    size = DWORD;
316                    imm = value.asInt();
317                    break;
318                case Long:
319                    size = QWORD;
320                    imm = value.asLong();
321                    break;
322                case Float:
323                    size = DWORD;
324                    imm = Float.floatToRawIntBits(value.asFloat());
325                    break;
326                case Double:
327                    size = QWORD;
328                    imm = Double.doubleToRawLongBits(value.asDouble());
329                    break;
330                default:
331                    throw JVMCIError.shouldNotReachHere("unexpected kind " + kind);
332            }
333
334            if (NumUtil.isInt(imm)) {
335                append(new AMD64BinaryConsumer.MemoryConstOp(op, size, address, (int) imm, state));
336            } else {
337                emitStore(kind, address, asAllocatable(value), state);
338            }
339        }
340    }
341
342    protected void emitStore(Kind kind, AMD64AddressValue address, AllocatableValue value, LIRFrameState state) {
343        switch (kind) {
344            case Boolean:
345            case Byte:
346                append(new AMD64BinaryConsumer.MemoryMROp(AMD64MROp.MOVB, BYTE, address, value, state));
347                break;
348            case Char:
349            case Short:
350                append(new AMD64BinaryConsumer.MemoryMROp(AMD64MROp.MOV, WORD, address, value, state));
351                break;
352            case Int:
353                append(new AMD64BinaryConsumer.MemoryMROp(AMD64MROp.MOV, DWORD, address, value, state));
354                break;
355            case Long:
356            case Object:
357                append(new AMD64BinaryConsumer.MemoryMROp(AMD64MROp.MOV, QWORD, address, value, state));
358                break;
359            case Float:
360                append(new AMD64BinaryConsumer.MemoryMROp(AMD64MROp.MOVSS, SS, address, value, state));
361                break;
362            case Double:
363                append(new AMD64BinaryConsumer.MemoryMROp(AMD64MROp.MOVSD, SD, address, value, state));
364                break;
365            default:
366                throw JVMCIError.shouldNotReachHere();
367        }
368    }
369
370    @Override
371    public void emitStore(LIRKind lirKind, Value address, Value input, LIRFrameState state) {
372        AMD64AddressValue storeAddress = asAddressValue(address);
373        Kind kind = (Kind) lirKind.getPlatformKind();
374        if (isConstant(input)) {
375            emitStoreConst(kind, storeAddress, asConstant(input), state);
376        } else {
377            emitStore(kind, storeAddress, asAllocatable(input), state);
378        }
379    }
380
381    @Override
382    public Variable emitCompareAndSwap(Value address, Value expectedValue, Value newValue, Value trueValue, Value falseValue) {
383        LIRKind kind = newValue.getLIRKind();
384        assert kind.equals(expectedValue.getLIRKind());
385        Kind memKind = (Kind) kind.getPlatformKind();
386
387        AMD64AddressValue addressValue = asAddressValue(address);
388        RegisterValue raxRes = AMD64.rax.asValue(kind);
389        emitMove(raxRes, expectedValue);
390        append(new CompareAndSwapOp(memKind, raxRes, addressValue, raxRes, asAllocatable(newValue)));
391
392        assert trueValue.getLIRKind().equals(falseValue.getLIRKind());
393        Variable result = newVariable(trueValue.getLIRKind());
394        append(new CondMoveOp(result, Condition.EQ, asAllocatable(trueValue), falseValue));
395        return result;
396    }
397
398    @Override
399    public Value emitAtomicReadAndAdd(Value address, Value delta) {
400        LIRKind kind = delta.getLIRKind();
401        Kind memKind = (Kind) kind.getPlatformKind();
402        Variable result = newVariable(kind);
403        AMD64AddressValue addressValue = asAddressValue(address);
404        append(new AMD64Move.AtomicReadAndAddOp(memKind, result, addressValue, asAllocatable(delta)));
405        return result;
406    }
407
408    @Override
409    public Value emitAtomicReadAndWrite(Value address, Value newValue) {
410        LIRKind kind = newValue.getLIRKind();
411        Kind memKind = (Kind) kind.getPlatformKind();
412        Variable result = newVariable(kind);
413        AMD64AddressValue addressValue = asAddressValue(address);
414        append(new AMD64Move.AtomicReadAndWriteOp(memKind, result, addressValue, asAllocatable(newValue)));
415        return result;
416    }
417
418    @Override
419    public void emitNullCheck(Value address, LIRFrameState state) {
420        assert address.getKind() == Kind.Object || address.getKind() == Kind.Long : address + " - " + address.getKind() + " not a pointer!";
421        append(new AMD64Move.NullCheckOp(asAddressValue(address), state));
422    }
423
424    @Override
425    public void emitJump(LabelRef label) {
426        assert label != null;
427        append(new JumpOp(label));
428    }
429
430    @Override
431    public void emitCompareBranch(PlatformKind cmpKind, Value left, Value right, Condition cond, boolean unorderedIsTrue, LabelRef trueLabel, LabelRef falseLabel, double trueLabelProbability) {
432        boolean mirrored = emitCompare(cmpKind, left, right);
433        Condition finalCondition = mirrored ? cond.mirror() : cond;
434        if (cmpKind == Kind.Float || cmpKind == Kind.Double) {
435            append(new FloatBranchOp(finalCondition, unorderedIsTrue, trueLabel, falseLabel, trueLabelProbability));
436        } else {
437            append(new BranchOp(finalCondition, trueLabel, falseLabel, trueLabelProbability));
438        }
439    }
440
441    public void emitCompareBranchMemory(Kind cmpKind, Value left, AMD64AddressValue right, LIRFrameState state, Condition cond, boolean unorderedIsTrue, LabelRef trueLabel, LabelRef falseLabel,
442                    double trueLabelProbability) {
443        boolean mirrored = emitCompareMemory(cmpKind, left, right, state);
444        Condition finalCondition = mirrored ? cond.mirror() : cond;
445        if (cmpKind == Kind.Float || cmpKind == Kind.Double) {
446            append(new FloatBranchOp(finalCondition, unorderedIsTrue, trueLabel, falseLabel, trueLabelProbability));
447        } else {
448            append(new BranchOp(finalCondition, trueLabel, falseLabel, trueLabelProbability));
449        }
450    }
451
452    @Override
453    public void emitOverflowCheckBranch(LabelRef overflow, LabelRef noOverflow, LIRKind cmpLIRKind, double overflowProbability) {
454        append(new BranchOp(ConditionFlag.Overflow, overflow, noOverflow, overflowProbability));
455    }
456
457    @Override
458    public void emitIntegerTestBranch(Value left, Value right, LabelRef trueDestination, LabelRef falseDestination, double trueDestinationProbability) {
459        emitIntegerTest(left, right);
460        append(new BranchOp(Condition.EQ, trueDestination, falseDestination, trueDestinationProbability));
461    }
462
463    @Override
464    public Variable emitConditionalMove(PlatformKind cmpKind, Value left, Value right, Condition cond, boolean unorderedIsTrue, Value trueValue, Value falseValue) {
465        boolean mirrored = emitCompare(cmpKind, left, right);
466        Condition finalCondition = mirrored ? cond.mirror() : cond;
467
468        Variable result = newVariable(trueValue.getLIRKind());
469        if (cmpKind == Kind.Float || cmpKind == Kind.Double) {
470            append(new FloatCondMoveOp(result, finalCondition, unorderedIsTrue, load(trueValue), load(falseValue)));
471        } else {
472            append(new CondMoveOp(result, finalCondition, load(trueValue), loadNonConst(falseValue)));
473        }
474        return result;
475    }
476
477    @Override
478    public Variable emitIntegerTestMove(Value left, Value right, Value trueValue, Value falseValue) {
479        emitIntegerTest(left, right);
480        Variable result = newVariable(trueValue.getLIRKind());
481        append(new CondMoveOp(result, Condition.EQ, load(trueValue), loadNonConst(falseValue)));
482        return result;
483    }
484
485    private void emitIntegerTest(Value a, Value b) {
486        assert a.getKind().isNumericInteger();
487        OperandSize size = a.getKind() == Kind.Long ? QWORD : DWORD;
488        if (isConstant(b) && NumUtil.is32bit(asConstant(b).asLong())) {
489            append(new AMD64BinaryConsumer.ConstOp(AMD64MIOp.TEST, size, asAllocatable(a), (int) asConstant(b).asLong()));
490        } else if (isConstant(a) && NumUtil.is32bit(asConstant(a).asLong())) {
491            append(new AMD64BinaryConsumer.ConstOp(AMD64MIOp.TEST, size, asAllocatable(b), (int) asConstant(a).asLong()));
492        } else if (isAllocatableValue(b)) {
493            append(new AMD64BinaryConsumer.Op(AMD64RMOp.TEST, size, asAllocatable(b), asAllocatable(a)));
494        } else {
495            append(new AMD64BinaryConsumer.Op(AMD64RMOp.TEST, size, asAllocatable(a), asAllocatable(b)));
496        }
497    }
498
499    protected void emitCompareOp(PlatformKind cmpKind, Variable left, Value right) {
500        OperandSize size;
501        switch ((Kind) cmpKind) {
502            case Byte:
503            case Boolean:
504                size = BYTE;
505                break;
506            case Short:
507            case Char:
508                size = WORD;
509                break;
510            case Int:
511                size = DWORD;
512                break;
513            case Long:
514            case Object:
515                size = QWORD;
516                break;
517            case Float:
518                append(new AMD64BinaryConsumer.Op(SSEOp.UCOMIS, PS, left, asAllocatable(right)));
519                return;
520            case Double:
521                append(new AMD64BinaryConsumer.Op(SSEOp.UCOMIS, PD, left, asAllocatable(right)));
522                return;
523            default:
524                throw JVMCIError.shouldNotReachHere("unexpected kind: " + cmpKind);
525        }
526
527        if (isConstant(right)) {
528            JavaConstant c = asConstant(right);
529            if (c.isDefaultForKind()) {
530                AMD64RMOp op = size == BYTE ? TESTB : TEST;
531                append(new AMD64BinaryConsumer.Op(op, size, left, left));
532                return;
533            } else if (NumUtil.is32bit(c.asLong())) {
534                append(new AMD64BinaryConsumer.ConstOp(CMP, size, left, (int) c.asLong()));
535                return;
536            }
537        }
538
539        AMD64RMOp op = CMP.getRMOpcode(size);
540        append(new AMD64BinaryConsumer.Op(op, size, left, asAllocatable(right)));
541    }
542
543    /**
544     * This method emits the compare against memory instruction, and may reorder the operands. It
545     * returns true if it did so.
546     *
547     * @param b the right operand of the comparison
548     * @return true if the left and right operands were switched, false otherwise
549     */
550    private boolean emitCompareMemory(Kind cmpKind, Value a, AMD64AddressValue b, LIRFrameState state) {
551        OperandSize size;
552        switch (cmpKind) {
553            case Byte:
554            case Boolean:
555                size = BYTE;
556                break;
557            case Short:
558            case Char:
559                size = WORD;
560                break;
561            case Int:
562                size = DWORD;
563                break;
564            case Long:
565            case Object:
566                size = QWORD;
567                break;
568            case Float:
569                append(new AMD64BinaryConsumer.MemoryRMOp(SSEOp.UCOMIS, PS, asAllocatable(a), b, state));
570                return false;
571            case Double:
572                append(new AMD64BinaryConsumer.MemoryRMOp(SSEOp.UCOMIS, PD, asAllocatable(a), b, state));
573                return false;
574            default:
575                throw JVMCIError.shouldNotReachHere("unexpected kind: " + cmpKind);
576        }
577
578        if (isConstant(a)) {
579            return emitCompareMemoryConOp(size, asConstant(a), b, state);
580        } else {
581            return emitCompareRegMemoryOp(size, a, b, state);
582        }
583    }
584
585    protected boolean emitCompareMemoryConOp(OperandSize size, JavaConstant a, AMD64AddressValue b, LIRFrameState state) {
586        if (NumUtil.is32bit(a.asLong())) {
587            append(new AMD64BinaryConsumer.MemoryConstOp(CMP, size, b, (int) a.asLong(), state));
588            return true;
589        } else {
590            return emitCompareRegMemoryOp(size, a, b, state);
591        }
592    }
593
594    private boolean emitCompareRegMemoryOp(OperandSize size, Value a, AMD64AddressValue b, LIRFrameState state) {
595        AMD64RMOp op = CMP.getRMOpcode(size);
596        append(new AMD64BinaryConsumer.MemoryRMOp(op, size, asAllocatable(a), b, state));
597        return false;
598    }
599
600    /**
601     * This method emits the compare instruction, and may reorder the operands. It returns true if
602     * it did so.
603     *
604     * @param a the left operand of the comparison
605     * @param b the right operand of the comparison
606     * @return true if the left and right operands were switched, false otherwise
607     */
608    private boolean emitCompare(PlatformKind cmpKind, Value a, Value b) {
609        Variable left;
610        Value right;
611        boolean mirrored;
612        if (LIRValueUtil.isVariable(b)) {
613            left = load(b);
614            right = loadNonConst(a);
615            mirrored = true;
616        } else {
617            left = load(a);
618            right = loadNonConst(b);
619            mirrored = false;
620        }
621        emitCompareOp(cmpKind, left, right);
622        return mirrored;
623    }
624
625    @Override
626    public Variable emitNegate(Value inputVal) {
627        AllocatableValue input = asAllocatable(inputVal);
628        Variable result = newVariable(LIRKind.combine(input));
629        switch (input.getKind()) {
630            case Int:
631                append(new AMD64Unary.MOp(NEG, DWORD, result, input));
632                break;
633            case Long:
634                append(new AMD64Unary.MOp(NEG, QWORD, result, input));
635                break;
636            case Float:
637                append(new AMD64Binary.DataOp(SSEOp.XOR, PS, result, input, JavaConstant.forFloat(Float.intBitsToFloat(0x80000000)), 16));
638                break;
639            case Double:
640                append(new AMD64Binary.DataOp(SSEOp.XOR, PD, result, input, JavaConstant.forDouble(Double.longBitsToDouble(0x8000000000000000L)), 16));
641                break;
642            default:
643                throw JVMCIError.shouldNotReachHere();
644        }
645        return result;
646    }
647
648    @Override
649    public Variable emitNot(Value inputVal) {
650        AllocatableValue input = asAllocatable(inputVal);
651        Variable result = newVariable(LIRKind.combine(input));
652        switch (input.getKind()) {
653            case Int:
654                append(new AMD64Unary.MOp(NOT, DWORD, result, input));
655                break;
656            case Long:
657                append(new AMD64Unary.MOp(NOT, QWORD, result, input));
658                break;
659            default:
660                throw JVMCIError.shouldNotReachHere();
661        }
662        return result;
663    }
664
665    private Variable emitBinary(LIRKind resultKind, AMD64BinaryArithmetic op, OperandSize size, boolean commutative, Value a, Value b, boolean setFlags) {
666        if (isConstant(b)) {
667            return emitBinaryConst(resultKind, op, size, commutative, asAllocatable(a), asConstant(b), setFlags);
668        } else if (commutative && isConstant(a)) {
669            return emitBinaryConst(resultKind, op, size, commutative, asAllocatable(b), asConstant(a), setFlags);
670        } else {
671            return emitBinaryVar(resultKind, op.getRMOpcode(size), size, commutative, asAllocatable(a), asAllocatable(b));
672        }
673    }
674
675    private Variable emitBinary(LIRKind resultKind, AMD64RMOp op, OperandSize size, boolean commutative, Value a, Value b) {
676        if (isConstant(b)) {
677            return emitBinaryConst(resultKind, op, size, asAllocatable(a), asConstant(b));
678        } else if (commutative && isConstant(a)) {
679            return emitBinaryConst(resultKind, op, size, asAllocatable(b), asConstant(a));
680        } else {
681            return emitBinaryVar(resultKind, op, size, commutative, asAllocatable(a), asAllocatable(b));
682        }
683    }
684
685    private Variable emitBinaryConst(LIRKind resultKind, AMD64BinaryArithmetic op, OperandSize size, boolean commutative, AllocatableValue a, JavaConstant b, boolean setFlags) {
686        if (NumUtil.isInt(b.asLong())) {
687            Variable result = newVariable(resultKind);
688            int constant = (int) b.asLong();
689
690            if (!setFlags) {
691                AMD64MOp mop = getMOp(op, constant);
692                if (mop != null) {
693                    append(new AMD64Unary.MOp(mop, size, result, a));
694                    return result;
695                }
696            }
697
698            append(new AMD64Binary.ConstOp(op, size, result, a, constant));
699            return result;
700        } else {
701            return emitBinaryVar(resultKind, op.getRMOpcode(size), size, commutative, a, asAllocatable(b));
702        }
703    }
704
705    private static AMD64MOp getMOp(AMD64BinaryArithmetic op, int constant) {
706        if (constant == 1) {
707            if (op.equals(AMD64BinaryArithmetic.ADD)) {
708                return AMD64MOp.INC;
709            }
710            if (op.equals(AMD64BinaryArithmetic.SUB)) {
711                return AMD64MOp.DEC;
712            }
713        } else if (constant == -1) {
714            if (op.equals(AMD64BinaryArithmetic.ADD)) {
715                return AMD64MOp.DEC;
716            }
717            if (op.equals(AMD64BinaryArithmetic.SUB)) {
718                return AMD64MOp.INC;
719            }
720        }
721        return null;
722    }
723
724    private Variable emitBinaryConst(LIRKind resultKind, AMD64RMOp op, OperandSize size, AllocatableValue a, JavaConstant b) {
725        Variable result = newVariable(resultKind);
726        append(new AMD64Binary.DataOp(op, size, result, a, b));
727        return result;
728    }
729
730    private Variable emitBinaryVar(LIRKind resultKind, AMD64RMOp op, OperandSize size, boolean commutative, AllocatableValue a, AllocatableValue b) {
731        Variable result = newVariable(resultKind);
732        if (commutative) {
733            append(new AMD64Binary.CommutativeOp(op, size, result, a, b));
734        } else {
735            append(new AMD64Binary.Op(op, size, result, a, b));
736        }
737        return result;
738    }
739
740    @Override
741    public Variable emitAdd(LIRKind resultKind, Value a, Value b, boolean setFlags) {
742        switch (a.getKind().getStackKind()) {
743            case Int:
744                return emitBinary(resultKind, ADD, DWORD, true, a, b, setFlags);
745            case Long:
746                return emitBinary(resultKind, ADD, QWORD, true, a, b, setFlags);
747            case Float:
748                return emitBinary(resultKind, SSEOp.ADD, SS, true, a, b);
749            case Double:
750                return emitBinary(resultKind, SSEOp.ADD, SD, true, a, b);
751            default:
752                throw JVMCIError.shouldNotReachHere();
753        }
754    }
755
756    @Override
757    public Variable emitSub(LIRKind resultKind, Value a, Value b, boolean setFlags) {
758        switch (a.getKind().getStackKind()) {
759            case Int:
760                return emitBinary(resultKind, SUB, DWORD, false, a, b, setFlags);
761            case Long:
762                return emitBinary(resultKind, SUB, QWORD, false, a, b, setFlags);
763            case Float:
764                return emitBinary(resultKind, SSEOp.SUB, SS, false, a, b);
765            case Double:
766                return emitBinary(resultKind, SSEOp.SUB, SD, false, a, b);
767            default:
768                throw JVMCIError.shouldNotReachHere();
769        }
770    }
771
772    private Variable emitIMULConst(OperandSize size, AllocatableValue a, JavaConstant b) {
773        if (NumUtil.isInt(b.asLong())) {
774            int imm = (int) b.asLong();
775            AMD64RMIOp op;
776            if (NumUtil.isByte(imm)) {
777                op = AMD64RMIOp.IMUL_SX;
778            } else {
779                op = AMD64RMIOp.IMUL;
780            }
781
782            Variable ret = newVariable(LIRKind.combine(a, b));
783            append(new AMD64Binary.RMIOp(op, size, ret, a, imm));
784            return ret;
785        } else {
786            return emitBinaryVar(LIRKind.combine(a, b), AMD64RMOp.IMUL, size, true, a, asAllocatable(b));
787        }
788    }
789
790    private Variable emitIMUL(OperandSize size, Value a, Value b) {
791        if (isConstant(b)) {
792            return emitIMULConst(size, asAllocatable(a), asConstant(b));
793        } else if (isConstant(a)) {
794            return emitIMULConst(size, asAllocatable(b), asConstant(a));
795        } else {
796            return emitBinaryVar(LIRKind.combine(a, b), AMD64RMOp.IMUL, size, true, asAllocatable(a), asAllocatable(b));
797        }
798    }
799
800    @Override
801    public Variable emitMul(Value a, Value b, boolean setFlags) {
802        switch (a.getKind().getStackKind()) {
803            case Int:
804                return emitIMUL(DWORD, a, b);
805            case Long:
806                return emitIMUL(QWORD, a, b);
807            case Float:
808                return emitBinary(LIRKind.combine(a, b), SSEOp.MUL, SS, true, a, b);
809            case Double:
810                return emitBinary(LIRKind.combine(a, b), SSEOp.MUL, SD, true, a, b);
811            default:
812                throw JVMCIError.shouldNotReachHere();
813        }
814    }
815
816    private RegisterValue moveToReg(Register reg, Value v) {
817        RegisterValue ret = reg.asValue(v.getLIRKind());
818        emitMove(ret, v);
819        return ret;
820    }
821
822    private Value emitMulHigh(AMD64MOp opcode, OperandSize size, Value a, Value b) {
823        AMD64MulDivOp mulHigh = append(new AMD64MulDivOp(opcode, size, LIRKind.combine(a, b), moveToReg(AMD64.rax, a), asAllocatable(b)));
824        return emitMove(mulHigh.getHighResult());
825    }
826
827    @Override
828    public Value emitMulHigh(Value a, Value b) {
829        switch (a.getKind().getStackKind()) {
830            case Int:
831                return emitMulHigh(AMD64MOp.IMUL, DWORD, a, b);
832            case Long:
833                return emitMulHigh(AMD64MOp.IMUL, QWORD, a, b);
834            default:
835                throw JVMCIError.shouldNotReachHere();
836        }
837    }
838
839    @Override
840    public Value emitUMulHigh(Value a, Value b) {
841        switch (a.getKind().getStackKind()) {
842            case Int:
843                return emitMulHigh(AMD64MOp.MUL, DWORD, a, b);
844            case Long:
845                return emitMulHigh(AMD64MOp.MUL, QWORD, a, b);
846            default:
847                throw JVMCIError.shouldNotReachHere();
848        }
849    }
850
851    public Value emitBinaryMemory(AMD64RMOp op, OperandSize size, AllocatableValue a, AMD64AddressValue location, LIRFrameState state) {
852        Variable result = newVariable(LIRKind.combine(a));
853        append(new AMD64Binary.MemoryOp(op, size, result, a, location, state));
854        return result;
855    }
856
857    protected Value emitConvertMemoryOp(PlatformKind kind, AMD64RMOp op, OperandSize size, AMD64AddressValue address, LIRFrameState state) {
858        Variable result = newVariable(LIRKind.value(kind));
859        append(new AMD64Unary.MemoryOp(op, size, result, address, state));
860        return result;
861    }
862
863    protected Value emitZeroExtendMemory(Kind memoryKind, int resultBits, AMD64AddressValue address, LIRFrameState state) {
864        // Issue a zero extending load of the proper bit size and set the result to
865        // the proper kind.
866        Variable result = newVariable(LIRKind.value(resultBits == 32 ? Kind.Int : Kind.Long));
867        switch (memoryKind) {
868            case Boolean:
869            case Byte:
870                append(new AMD64Unary.MemoryOp(MOVZXB, DWORD, result, address, state));
871                break;
872            case Char:
873            case Short:
874                append(new AMD64Unary.MemoryOp(MOVZX, DWORD, result, address, state));
875                break;
876            case Int:
877                append(new AMD64Unary.MemoryOp(MOV, DWORD, result, address, state));
878                break;
879            case Long:
880                append(new AMD64Unary.MemoryOp(MOV, QWORD, result, address, state));
881                break;
882            default:
883                throw JVMCIError.shouldNotReachHere();
884        }
885        return result;
886    }
887
888    private AMD64MulDivOp emitIDIV(OperandSize size, Value a, Value b, LIRFrameState state) {
889        LIRKind kind = LIRKind.combine(a, b);
890
891        AMD64SignExtendOp sx = append(new AMD64SignExtendOp(size, kind, moveToReg(AMD64.rax, a)));
892        return append(new AMD64MulDivOp(AMD64MOp.IDIV, size, kind, sx.getHighResult(), sx.getLowResult(), asAllocatable(b), state));
893    }
894
895    private AMD64MulDivOp emitDIV(OperandSize size, Value a, Value b, LIRFrameState state) {
896        LIRKind kind = LIRKind.combine(a, b);
897
898        RegisterValue rax = moveToReg(AMD64.rax, a);
899        RegisterValue rdx = AMD64.rdx.asValue(kind);
900        append(new AMD64ClearRegisterOp(size, rdx));
901        return append(new AMD64MulDivOp(AMD64MOp.DIV, size, kind, rdx, rax, asAllocatable(b), state));
902    }
903
904    public Value[] emitIntegerDivRem(Value a, Value b, LIRFrameState state) {
905        AMD64MulDivOp op;
906        switch (a.getKind().getStackKind()) {
907            case Int:
908                op = emitIDIV(DWORD, a, b, state);
909                break;
910            case Long:
911                op = emitIDIV(QWORD, a, b, state);
912                break;
913            default:
914                throw JVMCIError.shouldNotReachHere();
915        }
916        return new Value[]{emitMove(op.getQuotient()), emitMove(op.getRemainder())};
917    }
918
919    @Override
920    public Value emitDiv(Value a, Value b, LIRFrameState state) {
921        switch (a.getKind().getStackKind()) {
922            case Int:
923                AMD64MulDivOp op = emitIDIV(DWORD, a, b, state);
924                return emitMove(op.getQuotient());
925            case Long:
926                AMD64MulDivOp lop = emitIDIV(QWORD, a, b, state);
927                return emitMove(lop.getQuotient());
928            case Float:
929                return emitBinary(LIRKind.combine(a, b), SSEOp.DIV, SS, false, a, b);
930            case Double:
931                return emitBinary(LIRKind.combine(a, b), SSEOp.DIV, SD, false, a, b);
932            default:
933                throw JVMCIError.shouldNotReachHere();
934        }
935    }
936
937    @Override
938    public Value emitRem(Value a, Value b, LIRFrameState state) {
939        switch (a.getKind().getStackKind()) {
940            case Int:
941                AMD64MulDivOp op = emitIDIV(DWORD, a, b, state);
942                return emitMove(op.getRemainder());
943            case Long:
944                AMD64MulDivOp lop = emitIDIV(QWORD, a, b, state);
945                return emitMove(lop.getRemainder());
946            case Float: {
947                Variable result = newVariable(LIRKind.combine(a, b));
948                append(new FPDivRemOp(FREM, result, load(a), load(b)));
949                return result;
950            }
951            case Double: {
952                Variable result = newVariable(LIRKind.combine(a, b));
953                append(new FPDivRemOp(DREM, result, load(a), load(b)));
954                return result;
955            }
956            default:
957                throw JVMCIError.shouldNotReachHere();
958        }
959    }
960
961    @Override
962    public Variable emitUDiv(Value a, Value b, LIRFrameState state) {
963        AMD64MulDivOp op;
964        switch (a.getKind().getStackKind()) {
965            case Int:
966                op = emitDIV(DWORD, a, b, state);
967                break;
968            case Long:
969                op = emitDIV(QWORD, a, b, state);
970                break;
971            default:
972                throw JVMCIError.shouldNotReachHere();
973        }
974        return emitMove(op.getQuotient());
975    }
976
977    @Override
978    public Variable emitURem(Value a, Value b, LIRFrameState state) {
979        AMD64MulDivOp op;
980        switch (a.getKind().getStackKind()) {
981            case Int:
982                op = emitDIV(DWORD, a, b, state);
983                break;
984            case Long:
985                op = emitDIV(QWORD, a, b, state);
986                break;
987            default:
988                throw JVMCIError.shouldNotReachHere();
989        }
990        return emitMove(op.getRemainder());
991    }
992
993    @Override
994    public Variable emitAnd(Value a, Value b) {
995        LIRKind resultKind = LIRKind.combine(a, b);
996        switch (a.getKind().getStackKind()) {
997            case Int:
998                return emitBinary(resultKind, AND, DWORD, true, a, b, false);
999            case Long:
1000                return emitBinary(resultKind, AND, QWORD, true, a, b, false);
1001            case Float:
1002                return emitBinary(resultKind, SSEOp.AND, PS, true, a, b);
1003            case Double:
1004                return emitBinary(resultKind, SSEOp.AND, PD, true, a, b);
1005            default:
1006                throw JVMCIError.shouldNotReachHere();
1007        }
1008    }
1009
1010    @Override
1011    public Variable emitOr(Value a, Value b) {
1012        LIRKind resultKind = LIRKind.combine(a, b);
1013        switch (a.getKind().getStackKind()) {
1014            case Int:
1015                return emitBinary(resultKind, OR, DWORD, true, a, b, false);
1016            case Long:
1017                return emitBinary(resultKind, OR, QWORD, true, a, b, false);
1018            case Float:
1019                return emitBinary(resultKind, SSEOp.OR, PS, true, a, b);
1020            case Double:
1021                return emitBinary(resultKind, SSEOp.OR, PD, true, a, b);
1022            default:
1023                throw JVMCIError.shouldNotReachHere();
1024        }
1025    }
1026
1027    @Override
1028    public Variable emitXor(Value a, Value b) {
1029        LIRKind resultKind = LIRKind.combine(a, b);
1030        switch (a.getKind().getStackKind()) {
1031            case Int:
1032                return emitBinary(resultKind, XOR, DWORD, true, a, b, false);
1033            case Long:
1034                return emitBinary(resultKind, XOR, QWORD, true, a, b, false);
1035            case Float:
1036                return emitBinary(resultKind, SSEOp.XOR, PS, true, a, b);
1037            case Double:
1038                return emitBinary(resultKind, SSEOp.XOR, PD, true, a, b);
1039            default:
1040                throw JVMCIError.shouldNotReachHere();
1041        }
1042    }
1043
1044    private Variable emitShift(AMD64Shift op, OperandSize size, Value a, Value b) {
1045        Variable result = newVariable(LIRKind.combine(a, b).changeType(a.getPlatformKind()));
1046        AllocatableValue input = asAllocatable(a);
1047        if (isConstant(b)) {
1048            JavaConstant c = asConstant(b);
1049            if (c.asLong() == 1) {
1050                append(new AMD64Unary.MOp(op.m1Op, size, result, input));
1051            } else {
1052                /*
1053                 * c is implicitly masked to 5 or 6 bits by the CPU, so casting it to (int) is
1054                 * always correct, even without the NumUtil.is32bit() test.
1055                 */
1056                append(new AMD64Binary.ConstOp(op.miOp, size, result, input, (int) c.asLong()));
1057            }
1058        } else {
1059            emitMove(RCX_I, b);
1060            append(new AMD64ShiftOp(op.mcOp, size, result, input, RCX_I));
1061        }
1062        return result;
1063    }
1064
1065    @Override
1066    public Variable emitShl(Value a, Value b) {
1067        switch (a.getKind().getStackKind()) {
1068            case Int:
1069                return emitShift(SHL, DWORD, a, b);
1070            case Long:
1071                return emitShift(SHL, QWORD, a, b);
1072            default:
1073                throw JVMCIError.shouldNotReachHere();
1074        }
1075    }
1076
1077    @Override
1078    public Variable emitShr(Value a, Value b) {
1079        switch (a.getKind().getStackKind()) {
1080            case Int:
1081                return emitShift(SAR, DWORD, a, b);
1082            case Long:
1083                return emitShift(SAR, QWORD, a, b);
1084            default:
1085                throw JVMCIError.shouldNotReachHere();
1086        }
1087    }
1088
1089    @Override
1090    public Variable emitUShr(Value a, Value b) {
1091        switch (a.getKind().getStackKind()) {
1092            case Int:
1093                return emitShift(SHR, DWORD, a, b);
1094            case Long:
1095                return emitShift(SHR, QWORD, a, b);
1096            default:
1097                throw JVMCIError.shouldNotReachHere();
1098        }
1099    }
1100
1101    public Variable emitRol(Value a, Value b) {
1102        switch (a.getKind().getStackKind()) {
1103            case Int:
1104                return emitShift(ROL, DWORD, a, b);
1105            case Long:
1106                return emitShift(ROL, QWORD, a, b);
1107            default:
1108                throw JVMCIError.shouldNotReachHere();
1109        }
1110    }
1111
1112    public Variable emitRor(Value a, Value b) {
1113        switch (a.getKind().getStackKind()) {
1114            case Int:
1115                return emitShift(ROR, DWORD, a, b);
1116            case Long:
1117                return emitShift(ROR, QWORD, a, b);
1118            default:
1119                throw JVMCIError.shouldNotReachHere();
1120        }
1121    }
1122
1123    private AllocatableValue emitConvertOp(LIRKind kind, AMD64RMOp op, OperandSize size, Value input) {
1124        Variable result = newVariable(kind);
1125        append(new AMD64Unary.RMOp(op, size, result, asAllocatable(input)));
1126        return result;
1127    }
1128
1129    private AllocatableValue emitConvertOp(LIRKind kind, AMD64MROp op, OperandSize size, Value input) {
1130        Variable result = newVariable(kind);
1131        append(new AMD64Unary.MROp(op, size, result, asAllocatable(input)));
1132        return result;
1133    }
1134
1135    @Override
1136    public Value emitReinterpret(LIRKind to, Value inputVal) {
1137        LIRKind from = inputVal.getLIRKind();
1138        if (to.equals(from)) {
1139            return inputVal;
1140        }
1141
1142        AllocatableValue input = asAllocatable(inputVal);
1143        /*
1144         * Conversions between integer to floating point types require moves between CPU and FPU
1145         * registers.
1146         */
1147        Kind fromKind = (Kind) from.getPlatformKind();
1148        switch ((Kind) to.getPlatformKind()) {
1149            case Int:
1150                switch (fromKind) {
1151                    case Float:
1152                        return emitConvertOp(to, AMD64MROp.MOVD, DWORD, input);
1153                }
1154                break;
1155            case Long:
1156                switch (fromKind) {
1157                    case Double:
1158                        return emitConvertOp(to, AMD64MROp.MOVQ, QWORD, input);
1159                }
1160                break;
1161            case Float:
1162                switch (fromKind) {
1163                    case Int:
1164                        return emitConvertOp(to, AMD64RMOp.MOVD, DWORD, input);
1165                }
1166                break;
1167            case Double:
1168                switch (fromKind) {
1169                    case Long:
1170                        return emitConvertOp(to, AMD64RMOp.MOVQ, QWORD, input);
1171                }
1172                break;
1173        }
1174        throw JVMCIError.shouldNotReachHere();
1175    }
1176
1177    public Value emitFloatConvert(FloatConvert op, Value input) {
1178        switch (op) {
1179            case D2F:
1180                return emitConvertOp(LIRKind.combine(input).changeType(Kind.Float), SSEOp.CVTSD2SS, SD, input);
1181            case D2I:
1182                return emitConvertOp(LIRKind.combine(input).changeType(Kind.Int), SSEOp.CVTTSD2SI, DWORD, input);
1183            case D2L:
1184                return emitConvertOp(LIRKind.combine(input).changeType(Kind.Long), SSEOp.CVTTSD2SI, QWORD, input);
1185            case F2D:
1186                return emitConvertOp(LIRKind.combine(input).changeType(Kind.Double), SSEOp.CVTSS2SD, SS, input);
1187            case F2I:
1188                return emitConvertOp(LIRKind.combine(input).changeType(Kind.Int), SSEOp.CVTTSS2SI, DWORD, input);
1189            case F2L:
1190                return emitConvertOp(LIRKind.combine(input).changeType(Kind.Long), SSEOp.CVTTSS2SI, QWORD, input);
1191            case I2D:
1192                return emitConvertOp(LIRKind.combine(input).changeType(Kind.Double), SSEOp.CVTSI2SD, DWORD, input);
1193            case I2F:
1194                return emitConvertOp(LIRKind.combine(input).changeType(Kind.Float), SSEOp.CVTSI2SS, DWORD, input);
1195            case L2D:
1196                return emitConvertOp(LIRKind.combine(input).changeType(Kind.Double), SSEOp.CVTSI2SD, QWORD, input);
1197            case L2F:
1198                return emitConvertOp(LIRKind.combine(input).changeType(Kind.Float), SSEOp.CVTSI2SS, QWORD, input);
1199            default:
1200                throw JVMCIError.shouldNotReachHere();
1201        }
1202    }
1203
1204    @Override
1205    public Value emitNarrow(Value inputVal, int bits) {
1206        if (inputVal.getKind() == Kind.Long && bits <= 32) {
1207            // TODO make it possible to reinterpret Long as Int in LIR without move
1208            return emitConvertOp(LIRKind.combine(inputVal).changeType(Kind.Int), AMD64RMOp.MOV, DWORD, inputVal);
1209        } else {
1210            return inputVal;
1211        }
1212    }
1213
1214    @Override
1215    public Value emitSignExtend(Value inputVal, int fromBits, int toBits) {
1216        assert fromBits <= toBits && toBits <= 64;
1217        if (fromBits == toBits) {
1218            return inputVal;
1219        } else if (toBits > 32) {
1220            // sign extend to 64 bits
1221            switch (fromBits) {
1222                case 8:
1223                    return emitConvertOp(LIRKind.combine(inputVal).changeType(Kind.Long), MOVSXB, QWORD, inputVal);
1224                case 16:
1225                    return emitConvertOp(LIRKind.combine(inputVal).changeType(Kind.Long), MOVSX, QWORD, inputVal);
1226                case 32:
1227                    return emitConvertOp(LIRKind.combine(inputVal).changeType(Kind.Long), MOVSXD, QWORD, inputVal);
1228                default:
1229                    throw JVMCIError.unimplemented("unsupported sign extension (" + fromBits + " bit -> " + toBits + " bit)");
1230            }
1231        } else {
1232            // sign extend to 32 bits (smaller values are internally represented as 32 bit values)
1233            switch (fromBits) {
1234                case 8:
1235                    return emitConvertOp(LIRKind.combine(inputVal).changeType(Kind.Int), MOVSXB, DWORD, inputVal);
1236                case 16:
1237                    return emitConvertOp(LIRKind.combine(inputVal).changeType(Kind.Int), MOVSX, DWORD, inputVal);
1238                case 32:
1239                    return inputVal;
1240                default:
1241                    throw JVMCIError.unimplemented("unsupported sign extension (" + fromBits + " bit -> " + toBits + " bit)");
1242            }
1243        }
1244    }
1245
1246    @Override
1247    public Value emitZeroExtend(Value inputVal, int fromBits, int toBits) {
1248        assert fromBits <= toBits && toBits <= 64;
1249        if (fromBits == toBits) {
1250            return inputVal;
1251        } else if (fromBits > 32) {
1252            assert inputVal.getKind() == Kind.Long;
1253            Variable result = newVariable(LIRKind.combine(inputVal).changeType(Kind.Long));
1254            long mask = CodeUtil.mask(fromBits);
1255            append(new AMD64Binary.DataOp(AND.getRMOpcode(QWORD), QWORD, result, asAllocatable(inputVal), JavaConstant.forLong(mask)));
1256            return result;
1257        } else {
1258            assert inputVal.getKind().getStackKind() == Kind.Int;
1259
1260            LIRKind resultKind = LIRKind.combine(inputVal);
1261            if (toBits > 32) {
1262                resultKind = resultKind.changeType(Kind.Long);
1263            } else {
1264                resultKind = resultKind.changeType(Kind.Int);
1265            }
1266
1267            /*
1268             * Always emit DWORD operations, even if the resultKind is Long. On AMD64, all DWORD
1269             * operations implicitly set the upper half of the register to 0, which is what we want
1270             * anyway. Compared to the QWORD oparations, the encoding of the DWORD operations is
1271             * sometimes one byte shorter.
1272             */
1273            switch (fromBits) {
1274                case 8:
1275                    return emitConvertOp(resultKind, MOVZXB, DWORD, inputVal);
1276                case 16:
1277                    return emitConvertOp(resultKind, MOVZX, DWORD, inputVal);
1278                case 32:
1279                    return emitConvertOp(resultKind, MOV, DWORD, inputVal);
1280            }
1281
1282            // odd bit count, fall back on manual masking
1283            Variable result = newVariable(resultKind);
1284            JavaConstant mask;
1285            if (toBits > 32) {
1286                mask = JavaConstant.forLong(CodeUtil.mask(fromBits));
1287            } else {
1288                mask = JavaConstant.forInt((int) CodeUtil.mask(fromBits));
1289            }
1290            append(new AMD64Binary.DataOp(AND.getRMOpcode(DWORD), DWORD, result, asAllocatable(inputVal), mask));
1291            return result;
1292        }
1293    }
1294
1295    @Override
1296    public void emitMembar(int barriers) {
1297        int necessaryBarriers = target().arch.requiredBarriers(barriers);
1298        if (target().isMP && necessaryBarriers != 0) {
1299            append(new MembarOp(necessaryBarriers));
1300        }
1301    }
1302
1303    public abstract void emitCCall(long address, CallingConvention nativeCallingConvention, Value[] args, int numberOfFloatingPointArguments);
1304
1305    @Override
1306    protected void emitForeignCallOp(ForeignCallLinkage linkage, Value result, Value[] arguments, Value[] temps, LIRFrameState info) {
1307        long maxOffset = linkage.getMaxCallTargetOffset();
1308        if (maxOffset != (int) maxOffset) {
1309            append(new AMD64Call.DirectFarForeignCallOp(linkage, result, arguments, temps, info));
1310        } else {
1311            append(new AMD64Call.DirectNearForeignCallOp(linkage, result, arguments, temps, info));
1312        }
1313    }
1314
1315    @Override
1316    public Variable emitBitCount(Value value) {
1317        Variable result = newVariable(LIRKind.combine(value).changeType(Kind.Int));
1318        if (value.getKind().getStackKind() == Kind.Int) {
1319            append(new AMD64Unary.RMOp(POPCNT, DWORD, result, asAllocatable(value)));
1320        } else {
1321            append(new AMD64Unary.RMOp(POPCNT, QWORD, result, asAllocatable(value)));
1322        }
1323        return result;
1324    }
1325
1326    @Override
1327    public Variable emitBitScanForward(Value value) {
1328        Variable result = newVariable(LIRKind.combine(value).changeType(Kind.Int));
1329        append(new AMD64Unary.RMOp(BSF, QWORD, result, asAllocatable(value)));
1330        return result;
1331    }
1332
1333    @Override
1334    public Variable emitBitScanReverse(Value value) {
1335        Variable result = newVariable(LIRKind.combine(value).changeType(Kind.Int));
1336        if (value.getKind().getStackKind() == Kind.Int) {
1337            append(new AMD64Unary.RMOp(BSR, DWORD, result, asAllocatable(value)));
1338        } else {
1339            append(new AMD64Unary.RMOp(BSR, QWORD, result, asAllocatable(value)));
1340        }
1341        return result;
1342    }
1343
1344    public Value emitCountLeadingZeros(Value value) {
1345        Variable result = newVariable(LIRKind.combine(value).changeType(Kind.Int));
1346        if (value.getKind().getStackKind() == Kind.Int) {
1347            append(new AMD64Unary.RMOp(LZCNT, DWORD, result, asAllocatable(value)));
1348        } else {
1349            append(new AMD64Unary.RMOp(LZCNT, QWORD, result, asAllocatable(value)));
1350        }
1351        return result;
1352    }
1353
1354    public Value emitCountTrailingZeros(Value value) {
1355        Variable result = newVariable(LIRKind.combine(value).changeType(Kind.Int));
1356        if (value.getKind().getStackKind() == Kind.Int) {
1357            append(new AMD64Unary.RMOp(TZCNT, DWORD, result, asAllocatable(value)));
1358        } else {
1359            append(new AMD64Unary.RMOp(TZCNT, QWORD, result, asAllocatable(value)));
1360        }
1361        return result;
1362    }
1363
1364    @Override
1365    public Value emitMathAbs(Value input) {
1366        Variable result = newVariable(LIRKind.combine(input));
1367        switch (input.getKind()) {
1368            case Float:
1369                append(new AMD64Binary.DataOp(SSEOp.AND, PS, result, asAllocatable(input), JavaConstant.forFloat(Float.intBitsToFloat(0x7FFFFFFF)), 16));
1370                break;
1371            case Double:
1372                append(new AMD64Binary.DataOp(SSEOp.AND, PD, result, asAllocatable(input), JavaConstant.forDouble(Double.longBitsToDouble(0x7FFFFFFFFFFFFFFFL)), 16));
1373                break;
1374            default:
1375                throw JVMCIError.shouldNotReachHere();
1376        }
1377        return result;
1378    }
1379
1380    @Override
1381    public Value emitMathSqrt(Value input) {
1382        Variable result = newVariable(LIRKind.combine(input));
1383        switch (input.getKind()) {
1384            case Float:
1385                append(new AMD64Unary.RMOp(SSEOp.SQRT, SS, result, asAllocatable(input)));
1386                break;
1387            case Double:
1388                append(new AMD64Unary.RMOp(SSEOp.SQRT, SD, result, asAllocatable(input)));
1389                break;
1390            default:
1391                throw JVMCIError.shouldNotReachHere();
1392        }
1393        return result;
1394    }
1395
1396    @Override
1397    public Value emitMathLog(Value input, boolean base10) {
1398        Variable result = newVariable(LIRKind.combine(input));
1399        append(new AMD64MathIntrinsicOp(base10 ? LOG10 : LOG, result, asAllocatable(input)));
1400        return result;
1401    }
1402
1403    @Override
1404    public Value emitMathCos(Value input) {
1405        Variable result = newVariable(LIRKind.combine(input));
1406        append(new AMD64MathIntrinsicOp(COS, result, asAllocatable(input)));
1407        return result;
1408    }
1409
1410    @Override
1411    public Value emitMathSin(Value input) {
1412        Variable result = newVariable(LIRKind.combine(input));
1413        append(new AMD64MathIntrinsicOp(SIN, result, asAllocatable(input)));
1414        return result;
1415    }
1416
1417    @Override
1418    public Value emitMathTan(Value input) {
1419        Variable result = newVariable(LIRKind.combine(input));
1420        append(new AMD64MathIntrinsicOp(TAN, result, asAllocatable(input)));
1421        return result;
1422    }
1423
1424    @Override
1425    public Variable emitByteSwap(Value input) {
1426        Variable result = newVariable(LIRKind.combine(input));
1427        append(new AMD64ByteSwapOp(result, input));
1428        return result;
1429    }
1430
1431    @Override
1432    public Variable emitArrayEquals(Kind kind, Value array1, Value array2, Value length) {
1433        Variable result = newVariable(LIRKind.value(Kind.Int));
1434        append(new AMD64ArrayEqualsOp(this, kind, result, array1, array2, asAllocatable(length)));
1435        return result;
1436    }
1437
1438    @Override
1439    public void emitReturn(Value input) {
1440        AllocatableValue operand = Value.ILLEGAL;
1441        if (input != null) {
1442            operand = resultOperandFor(input.getLIRKind());
1443            emitMove(operand, input);
1444        }
1445        append(new ReturnOp(operand));
1446    }
1447
1448    @Override
1449    public void emitStrategySwitch(SwitchStrategy strategy, Variable key, LabelRef[] keyTargets, LabelRef defaultTarget) {
1450        // a temp is needed for loading object constants
1451        boolean needsTemp = key.getKind() == Kind.Object;
1452        append(new StrategySwitchOp(strategy, keyTargets, defaultTarget, key, needsTemp ? newVariable(key.getLIRKind()) : Value.ILLEGAL));
1453    }
1454
1455    @Override
1456    protected void emitTableSwitch(int lowKey, LabelRef defaultTarget, LabelRef[] targets, Value key) {
1457        append(new TableSwitchOp(lowKey, defaultTarget, targets, key, newVariable(LIRKind.value(target().wordKind)), newVariable(key.getLIRKind())));
1458    }
1459
1460}