# HG changeset patch
# User Thomas Wuerthinger <thomas@wuerthinger.net>
# Date 1309971011 -7200
# Node ID 5ad8481bebfc297c9eec9f9c511a94b62698640c
# Parent  80af9762097a74c04bd548f1dc0de5d91c7f7d01
Added intrinsification of Object arraycopy.

diff -r 80af9762097a -r 5ad8481bebfc graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/graph/BlockMap.java
--- a/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/graph/BlockMap.java	Wed Jul 06 17:53:05 2011 +0200
+++ b/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/graph/BlockMap.java	Wed Jul 06 18:50:11 2011 +0200
@@ -523,7 +523,7 @@
             }
 
             assert block.loops == 0;
-            block.loops = 1 << nextLoop;
+            block.loops = (long) 1 << (long) nextLoop;
             nextLoop++;
         }
         assert Long.bitCount(block.loops) == 1;
diff -r 80af9762097a -r 5ad8481bebfc graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/ir/Invoke.java
--- a/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/ir/Invoke.java	Wed Jul 06 17:53:05 2011 +0200
+++ b/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/ir/Invoke.java	Wed Jul 06 18:50:11 2011 +0200
@@ -41,11 +41,21 @@
     private static final int SUCCESSOR_COUNT = 1;
     private static final int SUCCESSOR_EXCEPTION_EDGE = 0;
 
+    private boolean canInline = true;
+
     @Override
     protected int inputCount() {
         return super.inputCount() + argumentCount;
     }
 
+    public boolean canInline() {
+        return canInline;
+    }
+
+    public void setCanInline(boolean b) {
+        canInline = b;
+    }
+
     @Override
     protected int successorCount() {
         return super.successorCount() + SUCCESSOR_COUNT;
@@ -204,6 +214,7 @@
     @Override
     public Node copy(Graph into) {
         Invoke x = new Invoke(bci, opcode, kind, new Value[argumentCount], target, returnType, into);
+        x.setCanInline(canInline);
         return x;
     }
 }
diff -r 80af9762097a -r 5ad8481bebfc graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/phases/InliningPhase.java
--- a/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/phases/InliningPhase.java	Wed Jul 06 17:53:05 2011 +0200
+++ b/graal/com.oracle.max.graal.compiler/src/com/oracle/max/graal/compiler/phases/InliningPhase.java	Wed Jul 06 18:50:11 2011 +0200
@@ -118,7 +118,7 @@
     private RiMethod inlineInvoke(Invoke invoke, int iterations, float ratio) {
         RiMethod parent = invoke.stateAfter().method();
         RiTypeProfile profile = parent.typeProfile(invoke.bci);
-        if (GraalOptions.Intrinsify && compilation.runtime.intrinsicGraph(invoke.target, invoke.arguments()) != null) {
+        if (GraalOptions.Intrinsify && compilation.runtime.intrinsicGraph(parent, invoke.bci, invoke.target, invoke.arguments()) != null) {
             // Always intrinsify.
             return invoke.target;
         }
@@ -209,6 +209,12 @@
     }
 
     private boolean checkInvokeConditions(Invoke invoke) {
+        if (!invoke.canInline()) {
+            if (GraalOptions.TraceInlining) {
+                TTY.println("not inlining %s because the invoke is manually set to be non-inlinable", methodName(invoke.target, invoke));
+            }
+            return false;
+        }
         if (invoke.stateAfter() == null) {
             if (GraalOptions.TraceInlining) {
                 TTY.println("not inlining %s because the invoke has no after state", methodName(invoke.target, invoke));
@@ -317,6 +323,7 @@
     }
 
     private void inlineMethod(Invoke invoke, RiMethod method) {
+        RiMethod parent = invoke.stateAfter().method();
         FrameState stateAfter = invoke.stateAfter();
         FixedNode exceptionEdge = invoke.exceptionEdge();
         if (exceptionEdge instanceof Placeholder) {
@@ -339,7 +346,7 @@
 
         CompilerGraph graph = null;
         if (GraalOptions.Intrinsify) {
-            graph = (CompilerGraph) compilation.runtime.intrinsicGraph(method, invoke.arguments());
+            graph = (CompilerGraph) compilation.runtime.intrinsicGraph(parent, invoke.bci, method, invoke.arguments());
         }
         if (graph != null) {
             if (GraalOptions.TraceInlining) {
@@ -405,7 +412,7 @@
 
         FrameState stateBefore = null;
         for (Node node : duplicates.values()) {
-            if (node instanceof Invoke) {
+            if (node instanceof Invoke && ((Invoke) node).canInline()) {
                 newInvokes.add((Invoke) node);
             } else if (node instanceof FrameState) {
                 FrameState frameState = (FrameState) node;
diff -r 80af9762097a -r 5ad8481bebfc graal/com.oracle.max.graal.runtime/src/com/oracle/max/graal/runtime/HotSpotRuntime.java
--- a/graal/com.oracle.max.graal.runtime/src/com/oracle/max/graal/runtime/HotSpotRuntime.java	Wed Jul 06 17:53:05 2011 +0200
+++ b/graal/com.oracle.max.graal.runtime/src/com/oracle/max/graal/runtime/HotSpotRuntime.java	Wed Jul 06 18:50:11 2011 +0200
@@ -27,11 +27,13 @@
 import java.util.*;
 
 import com.oracle.max.graal.compiler.*;
+import com.oracle.max.graal.compiler.debug.*;
 import com.oracle.max.graal.compiler.graph.*;
 import com.oracle.max.graal.compiler.ir.*;
 import com.oracle.max.graal.compiler.value.*;
 import com.oracle.max.graal.graph.*;
 import com.oracle.max.graal.runtime.nodes.*;
+import com.sun.cri.bytecode.*;
 import com.sun.cri.ci.*;
 import com.sun.cri.ci.CiTargetMethod.Call;
 import com.sun.cri.ci.CiTargetMethod.DataPatch;
@@ -307,10 +309,11 @@
             LocationNode arrayLocation = createArrayLocation(graph, elementKind);
             arrayLocation.setIndex(storeIndexed.index());
             Value value = storeIndexed.value();
+            Value array = storeIndexed.array();
             if (elementKind == CiKind.Object && !value.isNullConstant()) {
                 // Store check!
-                if (storeIndexed.array().exactType() != null) {
-                    RiType elementType = storeIndexed.array().exactType().componentType();
+                if (array.exactType() != null) {
+                    RiType elementType = array.exactType().componentType();
                     if (elementType.superType() != null) {
                         Constant type = new Constant(elementType.getEncoding(Representation.ObjectHub), graph);
                         value = new CheckCast(type, value, graph);
@@ -318,17 +321,16 @@
                         assert elementType.name().equals("Ljava/lang/Object;") : elementType.name();
                     }
                 } else {
-                    ReadNode arrayKlass = new ReadNode(CiKind.Object, storeIndexed.array(), LocationNode.create(LocationNode.FINAL_LOCATION, CiKind.Object, config.hubOffset, graph), graph);
-                    ReadNode arrayElementKlass = new ReadNode(CiKind.Object, arrayKlass, LocationNode.create(LocationNode.FINAL_LOCATION, CiKind.Object, config.arrayClassElementOffset, graph), graph);
+                    ReadNode arrayElementKlass = readArrayElementKlass(graph, array);
                     value = new CheckCast(arrayElementKlass, value, graph);
                 }
             }
-            WriteNode memoryWrite = new WriteNode(elementKind.stackKind(), storeIndexed.array(), value, arrayLocation, graph);
+            WriteNode memoryWrite = new WriteNode(elementKind.stackKind(), array, value, arrayLocation, graph);
             memoryWrite.setGuard(boundsCheck);
             memoryWrite.setStateAfter(storeIndexed.stateAfter());
             anchor.setNext(memoryWrite);
             if (elementKind == CiKind.Object && !value.isNullConstant()) {
-                ArrayWriteBarrier writeBarrier = new ArrayWriteBarrier(storeIndexed.array(), arrayLocation, graph);
+                ArrayWriteBarrier writeBarrier = new ArrayWriteBarrier(array, arrayLocation, graph);
                 memoryWrite.setNext(writeBarrier);
                 writeBarrier.setNext(storeIndexed.next());
             } else {
@@ -339,6 +341,12 @@
         }
     }
 
+    private ReadNode readArrayElementKlass(Graph graph, Value array) {
+        ReadNode arrayKlass = readHub(graph, array);
+        ReadNode arrayElementKlass = new ReadNode(CiKind.Object, arrayKlass, LocationNode.create(LocationNode.FINAL_LOCATION, CiKind.Object, config.arrayClassElementOffset, graph), graph);
+        return arrayElementKlass;
+    }
+
     private LocationNode createArrayLocation(Graph graph, CiKind elementKind) {
         return LocationNode.create(LocationNode.getArrayLocation(elementKind), elementKind, config.getArrayOffset(elementKind), graph);
     }
@@ -348,7 +356,7 @@
     }
 
     @Override
-    public Graph intrinsicGraph(RiMethod method, List<? extends Node> parameters) {
+    public Graph intrinsicGraph(RiMethod caller, int bci, RiMethod method, List<? extends Node> parameters) {
         if (!intrinsicGraphs.containsKey(method)) {
             RiType holder = method.holder();
             String fullName = method.name() + method.signature().asString();
@@ -357,7 +365,7 @@
                 if (fullName.equals("getClass()Ljava/lang/Class;")) {
                     CompilerGraph graph = new CompilerGraph(this);
                     Local receiver = new Local(CiKind.Object, 0, graph);
-                    ReadNode klassOop = new ReadNode(CiKind.Object, receiver, LocationNode.create(LocationNode.FINAL_LOCATION, CiKind.Object, config.hubOffset, graph), graph);
+                    ReadNode klassOop = readHub(graph, receiver);
                     Return ret = new Return(new ReadNode(CiKind.Object, klassOop, LocationNode.create(LocationNode.FINAL_LOCATION, CiKind.Object, config.classMirrorOffset, graph), graph), graph);
                     graph.start().setNext(ret);
                     graph.setReturn(ret);
@@ -388,10 +396,6 @@
 
                     CiKind componentType = src.declaredType().componentType().kind();
 
-                    if (componentType == CiKind.Object) {
-                        return null;
-                    }
-
                     FrameState stateBefore = new FrameState(method, FrameState.BEFORE_BCI, 0, 0, 0, false, graph);
                     FrameState stateAfter = new FrameState(method, FrameState.AFTER_BCI, 0, 0, 0, false, graph);
 
@@ -435,7 +439,22 @@
                     merge1.addEnd(new EndNode(graph));
                     merge1.setStateAfter(stateBefore);
 
-                    ifNode.setFalseSuccessor(merge1.endAt(0));
+
+                    Invoke newInvoke = null;
+                    if (componentType == CiKind.Object) {
+                        Value srcClass = readHub(graph, src);
+                        Value destClass = readHub(graph, dest);
+                        If elementClassIf = new If(new Compare(srcClass, Condition.EQ, destClass, graph), graph);
+                        ifNode.setFalseSuccessor(elementClassIf);
+                        newInvoke = new Invoke(bci, Bytecodes.INVOKESTATIC, CiKind.Void, new Value[]{src, srcPos, dest, destPos, length}, method, method.signature().returnType(method.holder()), graph);
+                        newInvoke.setCanInline(false);
+                        newInvoke.setStateAfter(stateAfter);
+                        elementClassIf.setFalseSuccessor(newInvoke);
+                        elementClassIf.setTrueSuccessor(merge1.endAt(0));
+                    } else {
+                        ifNode.setFalseSuccessor(merge1.endAt(0));
+                    }
+
                     secondIf.setFalseSuccessor(merge1.endAt(1));
                     merge1.setNext(normalVector);
 
@@ -447,6 +466,11 @@
                     normalVector.setNext(merge2.endAt(0));
                     reverseVector.setNext(merge2.endAt(1));
 
+                    if (newInvoke != null) {
+                        merge2.addEnd(new EndNode(graph));
+                        newInvoke.setNext(merge2.endAt(2));
+                    }
+
                     Return ret = new Return(null, graph);
                     merge2.setNext(ret);
                     graph.setReturn(ret);
@@ -496,4 +520,8 @@
         }
         return intrinsicGraphs.get(method);
     }
+
+    private ReadNode readHub(Graph graph, Value value) {
+        return new ReadNode(CiKind.Object, value, LocationNode.create(LocationNode.FINAL_LOCATION, CiKind.Object, config.hubOffset, graph), graph);
+    }
 }