# HG changeset patch
# User Tom Rodriguez <tom.rodriguez@oracle.com>
# Date 1386441016 -3600
# Node ID 785bbb6192384d072c2f2e6d680231b52c3d8945
# Parent  42aaf73067079821438c3e101c6994ad0acbff55
Basic allocation prefetching support

diff -r 42aaf7306707 -r 785bbb619238 graal/com.oracle.graal.asm.amd64/src/com/oracle/graal/asm/amd64/AMD64Assembler.java
--- a/graal/com.oracle.graal.asm.amd64/src/com/oracle/graal/asm/amd64/AMD64Assembler.java	Thu Dec 05 18:13:04 2013 -0800
+++ b/graal/com.oracle.graal.asm.amd64/src/com/oracle/graal/asm/amd64/AMD64Assembler.java	Sat Dec 07 19:30:16 2013 +0100
@@ -2524,4 +2524,53 @@
     public AMD64Address getPlaceholder() {
         return Placeholder;
     }
+
+    private void prefetchPrefix(AMD64Address src) {
+        prefix(src);
+        emitByte(0x0F);
+    }
+
+    public void prefetchnta(AMD64Address src) {
+        prefetchPrefix(src);
+        emitByte(0x18);
+        emitOperandHelper(0, src);
+    }
+
+    void prefetchr(AMD64Address src) {
+        // assert(VM_Version::supports_3dnow_prefetch(), "must support");
+        prefetchPrefix(src);
+        emitByte(0x0D);
+        emitOperandHelper(0, src);
+    }
+
+    public void prefetcht0(AMD64Address src) {
+        // NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
+        prefetchPrefix(src);
+        emitByte(0x18);
+        emitOperandHelper(1, src);
+    }
+
+    public void prefetcht1(AMD64Address src) {
+        // NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
+        prefetchPrefix(src);
+        emitByte(0x18);
+        emitOperandHelper(2, src);
+    }
+
+    public void prefetcht2(AMD64Address src) {
+        // NOT_LP64(assert(VM_Version::supports_sse(), "must support"));
+        prefix(src);
+        emitByte(0x0f);
+        emitByte(0x18);
+        emitOperandHelper(3, src);
+    }
+
+    public void prefetchw(AMD64Address src) {
+        // assert(VM_Version::supports_3dnow_prefetch(), "must support");
+        prefix(src);
+        emitByte(0x0f);
+        emitByte(0x0D);
+        emitOperandHelper(1, src);
+    }
+
 }
diff -r 42aaf7306707 -r 785bbb619238 graal/com.oracle.graal.compiler.amd64/src/com/oracle/graal/compiler/amd64/AMD64LIRGenerator.java
--- a/graal/com.oracle.graal.compiler.amd64/src/com/oracle/graal/compiler/amd64/AMD64LIRGenerator.java	Thu Dec 05 18:13:04 2013 -0800
+++ b/graal/com.oracle.graal.compiler.amd64/src/com/oracle/graal/compiler/amd64/AMD64LIRGenerator.java	Sat Dec 07 19:30:16 2013 +0100
@@ -142,6 +142,7 @@
 
     @Override
     public AMD64AddressValue emitAddress(Value base, long displacement, Value index, int scale) {
+        assert (scale >= 1 && scale <= 8) || index.equals(Value.ILLEGAL) : "invalid scale";
         AllocatableValue baseRegister;
         long finalDisp = displacement;
         if (isConstant(base)) {
diff -r 42aaf7306707 -r 785bbb619238 graal/com.oracle.graal.hotspot.amd64/src/com/oracle/graal/hotspot/amd64/AMD64HotSpotLIRGenerator.java
--- a/graal/com.oracle.graal.hotspot.amd64/src/com/oracle/graal/hotspot/amd64/AMD64HotSpotLIRGenerator.java	Thu Dec 05 18:13:04 2013 -0800
+++ b/graal/com.oracle.graal.hotspot.amd64/src/com/oracle/graal/hotspot/amd64/AMD64HotSpotLIRGenerator.java	Sat Dec 07 19:30:16 2013 +0100
@@ -574,4 +574,9 @@
             super.visitInfopointNode(i);
         }
     }
+
+    public void emitPrefetchAllocate(ValueNode address, ValueNode distance) {
+        AMD64AddressValue addr = emitAddress(operand(address), 0, loadNonConst(operand(distance)), 1);
+        append(new AMD64PrefetchOp(addr, config.allocatePrefetchInstr));
+    }
 }
diff -r 42aaf7306707 -r 785bbb619238 graal/com.oracle.graal.hotspot.amd64/src/com/oracle/graal/hotspot/amd64/AMD64PrefetchOp.java
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/graal/com.oracle.graal.hotspot.amd64/src/com/oracle/graal/hotspot/amd64/AMD64PrefetchOp.java	Sat Dec 07 19:30:16 2013 +0100
@@ -0,0 +1,63 @@
+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package com.oracle.graal.hotspot.amd64;
+
+import static com.oracle.graal.lir.LIRInstruction.OperandFlag.*;
+
+import com.oracle.graal.asm.amd64.*;
+import com.oracle.graal.graph.*;
+import com.oracle.graal.lir.amd64.*;
+import com.oracle.graal.lir.asm.*;
+
+public class AMD64PrefetchOp extends AMD64LIRInstruction {
+
+    private final int instr;  // AllocatePrefecthInstr
+    @Alive({COMPOSITE}) protected AMD64AddressValue address;
+
+    public AMD64PrefetchOp(AMD64AddressValue address, int instr) {
+        this.address = address;
+        this.instr = instr;
+    }
+
+    @Override
+    public void emitCode(CompilationResultBuilder crb, AMD64MacroAssembler masm) {
+        switch (instr) {
+            case 0:
+                masm.prefetchnta(address.toAddress());
+                break;
+            case 1:
+                masm.prefetcht0(address.toAddress());
+                break;
+            case 2:
+                masm.prefetcht2(address.toAddress());
+                break;
+            case 3:
+                masm.prefetchw(address.toAddress());
+                break;
+            default:
+                throw GraalInternalError.shouldNotReachHere("unspported prefetch op " + instr);
+
+        }
+    }
+}
diff -r 42aaf7306707 -r 785bbb619238 graal/com.oracle.graal.hotspot.sparc/src/com/oracle/graal/hotspot/sparc/SPARCHotSpotLIRGenerator.java
--- a/graal/com.oracle.graal.hotspot.sparc/src/com/oracle/graal/hotspot/sparc/SPARCHotSpotLIRGenerator.java	Thu Dec 05 18:13:04 2013 -0800
+++ b/graal/com.oracle.graal.hotspot.sparc/src/com/oracle/graal/hotspot/sparc/SPARCHotSpotLIRGenerator.java	Sat Dec 07 19:30:16 2013 +0100
@@ -311,4 +311,9 @@
         GraalInternalError.shouldNotReachHere("binary negation not implemented");
         return null;
     }
+
+    public void emitPrefetchAllocate(ValueNode address, ValueNode distance) {
+        SPARCAddressValue addr = emitAddress(operand(address), 0, loadNonConst(operand(distance)), 1);
+        append(new SPARCPrefetchOp(addr, config.allocatePrefetchInstr));
+    }
 }
diff -r 42aaf7306707 -r 785bbb619238 graal/com.oracle.graal.hotspot.sparc/src/com/oracle/graal/hotspot/sparc/SPARCPrefetchOp.java
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/graal/com.oracle.graal.hotspot.sparc/src/com/oracle/graal/hotspot/sparc/SPARCPrefetchOp.java	Sat Dec 07 19:30:16 2013 +0100
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package com.oracle.graal.hotspot.sparc;
+
+import static com.oracle.graal.lir.LIRInstruction.OperandFlag.*;
+
+import com.oracle.graal.asm.sparc.*;
+import com.oracle.graal.graph.*;
+import com.oracle.graal.lir.asm.*;
+import com.oracle.graal.lir.sparc.*;
+
+public class SPARCPrefetchOp extends SPARCLIRInstruction {
+
+    private final int instr;  // AllocatePrefecthInstr
+    @Alive({COMPOSITE}) protected SPARCAddressValue address;
+
+    public SPARCPrefetchOp(SPARCAddressValue address, int instr) {
+        this.address = address;
+        this.instr = instr;
+    }
+
+    @Override
+    public void emitCode(CompilationResultBuilder crb, SPARCMacroAssembler masm) {
+        assert instr == 0 : "only supported value is 0";
+        // masm.prefetch(address.toAddress(), 2);
+        throw GraalInternalError.unimplemented("prefetch instruction");
+    }
+
+}
diff -r 42aaf7306707 -r 785bbb619238 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/HotSpotGraalRuntime.java
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/HotSpotGraalRuntime.java	Thu Dec 05 18:13:04 2013 -0800
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/HotSpotGraalRuntime.java	Sat Dec 07 19:30:16 2013 +0100
@@ -42,6 +42,8 @@
 import com.oracle.graal.phases.*;
 import com.oracle.graal.runtime.*;
 
+//JaCoCo Exclude
+
 /**
  * Singleton class holding the instance of the {@link GraalRuntime}.
  */
diff -r 42aaf7306707 -r 785bbb619238 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/HotSpotLIRGenerator.java
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/HotSpotLIRGenerator.java	Thu Dec 05 18:13:04 2013 -0800
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/HotSpotLIRGenerator.java	Sat Dec 07 19:30:16 2013 +0100
@@ -50,6 +50,8 @@
 
     void emitJumpToExceptionHandlerInCaller(ValueNode handlerInCallerPc, ValueNode exception, ValueNode exceptionPc);
 
+    void emitPrefetchAllocate(ValueNode address, ValueNode distance);
+
     void visitDirectCompareAndSwap(DirectCompareAndSwapNode x);
 
     /**
diff -r 42aaf7306707 -r 785bbb619238 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/HotSpotVMConfig.java
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/HotSpotVMConfig.java	Thu Dec 05 18:13:04 2013 -0800
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/HotSpotVMConfig.java	Sat Dec 07 19:30:16 2013 +0100
@@ -673,6 +673,13 @@
     @HotSpotVMFlag(name = "UseCRC32Intrinsics") @Stable public boolean useCRC32Intrinsics;
     @HotSpotVMFlag(name = "UseG1GC") @Stable public boolean useG1GC;
 
+    @HotSpotVMFlag(name = "AllocatePrefetchStyle") @Stable public int allocatePrefetchStyle;
+    @HotSpotVMFlag(name = "AllocatePrefetchInstr") @Stable public int allocatePrefetchInstr;
+    @HotSpotVMFlag(name = "AllocatePrefetchLines") @Stable public int allocatePrefetchLines;
+    @HotSpotVMFlag(name = "AllocateInstancePrefetchLines") @Stable public int allocateInstancePrefetchLines;
+    @HotSpotVMFlag(name = "AllocatePrefetchStepSize") @Stable public int allocatePrefetchStepSize;
+    @HotSpotVMFlag(name = "AllocatePrefetchDistance") @Stable public int allocatePrefetchDistance;
+
     @HotSpotVMField(name = "Universe::_collectedHeap", type = "CollectedHeap*", get = HotSpotVMField.Type.VALUE) @Stable private long universeCollectedHeap;
     @HotSpotVMField(name = "CollectedHeap::_total_collections", type = "unsigned int", get = HotSpotVMField.Type.OFFSET) @Stable private int collectedHeapTotalCollectionsOffset;
 
diff -r 42aaf7306707 -r 785bbb619238 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/nodes/PrefetchAllocateNode.java
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/nodes/PrefetchAllocateNode.java	Sat Dec 07 19:30:16 2013 +0100
@@ -0,0 +1,51 @@
+/*
+ * Copyright (c) 2009, 2011, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+
+package com.oracle.graal.hotspot.nodes;
+
+import com.oracle.graal.compiler.gen.*;
+import com.oracle.graal.compiler.target.*;
+import com.oracle.graal.hotspot.*;
+import com.oracle.graal.nodes.*;
+import com.oracle.graal.nodes.type.*;
+import com.oracle.graal.word.*;
+
+public class PrefetchAllocateNode extends FixedWithNextNode implements LIRGenLowerable {
+
+    @Input private ValueNode distance;
+    @Input private ValueNode address;
+
+    public PrefetchAllocateNode(ValueNode address, ValueNode distance) {
+        super(StampFactory.forVoid());
+        this.address = address;
+        this.distance = distance;
+    }
+
+    @Override
+    public void generate(LIRGenerator gen) {
+        ((HotSpotLIRGenerator) gen).emitPrefetchAllocate(address, distance);
+    }
+
+    @NodeIntrinsic
+    public static native void prefetch(Word address, Word distance);
+}
diff -r 42aaf7306707 -r 785bbb619238 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/NewObjectSnippets.java
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/NewObjectSnippets.java	Thu Dec 05 18:13:04 2013 -0800
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/NewObjectSnippets.java	Sat Dec 07 19:30:16 2013 +0100
@@ -24,6 +24,7 @@
 
 import static com.oracle.graal.api.code.UnsignedMath.*;
 import static com.oracle.graal.api.meta.MetaUtil.*;
+import static com.oracle.graal.hotspot.HotSpotGraalRuntime.*;
 import static com.oracle.graal.hotspot.replacements.HotSpotReplacementsUtil.*;
 import static com.oracle.graal.hotspot.replacements.NewObjectSnippets.Options.*;
 import static com.oracle.graal.nodes.PiArrayNode.*;
@@ -112,6 +113,21 @@
         }
     }
 
+    private static void emitPrefetchAllocate(Word address, boolean isArray) {
+        if (config().allocatePrefetchStyle > 0) {
+            // Insert a prefetch for each allocation only on the fast-path
+            // Generate several prefetch instructions.
+            int lines = isArray ? config().allocatePrefetchLines : config().allocateInstancePrefetchLines;
+            int stepSize = config().allocatePrefetchStepSize;
+            int distance = config().allocatePrefetchDistance;
+            ExplodeLoopNode.explodeLoop();
+            for (int i = 0; i < lines; i++) {
+                PrefetchAllocateNode.prefetch(address, Word.signed(distance));
+                distance += stepSize;
+            }
+        }
+    }
+
     @Snippet
     public static Object allocateInstance(@ConstantParameter int size, Word hub, Word prototypeMarkWord, @ConstantParameter boolean fillContents, @ConstantParameter Register threadRegister,
                     @ConstantParameter String typeContext) {
@@ -122,6 +138,7 @@
         Word newTop = top.add(size);
         if (useTLAB() && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
             writeTlabTop(thread, newTop);
+            emitPrefetchAllocate(newTop, false);
             result = formatObject(hub, size, top, prototypeMarkWord, fillContents);
         } else {
             new_stub.inc();
@@ -157,6 +174,7 @@
         Word newTop = top.add(allocationSize);
         if (useTLAB() && probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) {
             writeTlabTop(thread, newTop);
+            emitPrefetchAllocate(newTop, true);
             newarray_loopInit.inc();
             result = formatArray(hub, allocationSize, length, headerSize, top, prototypeMarkWord, fillContents);
         } else {