Mercurial > hg > truffle

changeset 7902:14fedab0419e

Find changesets by keywords (author, files, the commit message), revision number or hash, or revset expression.
improved arraycopy and Object.clone snippets
author Lukas Stadler <lukas.stadler@jku.at>
date Wed, 27 Feb 2013 18:37:08 +0100
parents 93a133fc03ce
children f19c4d447e73
files graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ArrayCopyNode.java graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ArrayCopySnippets.java graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ClassSubstitutions.java graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/HotSpotSnippetUtils.java graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ObjectCloneNode.java graal/com.oracle.graal.snippets/src/com/oracle/graal/snippets/nodes/MacroNode.java
diffstat 6 files changed, 175 insertions(+), 75 deletions(-) [+]
line wrap: on
line diff
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ArrayCopyNode.java	Wed Feb 27 18:28:09 2013 +0100
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ArrayCopyNode.java	Wed Feb 27 18:37:08 2013 +0100
@@ -30,6 +30,7 @@
 import com.oracle.graal.nodes.*;
 import com.oracle.graal.nodes.spi.*;
 import com.oracle.graal.nodes.virtual.*;
+import com.oracle.graal.phases.*;
 import com.oracle.graal.phases.common.*;
 import com.oracle.graal.snippets.nodes.*;
 
@@ -59,21 +60,19 @@
         return arguments.get(4);
     }
 
-    private ResolvedJavaMethod selectSnippet(LoweringTool tool) {
+    private StructuredGraph selectSnippet(LoweringTool tool) {
         ResolvedJavaType srcType = getSource().objectStamp().type();
         ResolvedJavaType destType = getDestination().objectStamp().type();
 
-        if (srcType != null && srcType.isArray() && destType != null && destType.isArray()) {
-            Kind componentKind = srcType.getComponentType().getKind();
-            if (componentKind != Kind.Object) {
-                if (srcType.getComponentType() == destType.getComponentType()) {
-                    return tool.getRuntime().lookupJavaMethod(ArrayCopySnippets.getSnippetForKind(componentKind));
-                }
-            } else if (destType.getComponentType().isAssignableFrom(srcType.getComponentType()) && getDestination().objectStamp().isExactType()) {
-                return tool.getRuntime().lookupJavaMethod(ArrayCopySnippets.getSnippetForKind(Kind.Object));
-            }
+        if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) {
+            return null;
         }
-        return null;
+        if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType()) || !getDestination().objectStamp().isExactType()) {
+            return null;
+        }
+        Kind componentKind = srcType.getComponentType().getKind();
+        ResolvedJavaMethod snippetMethod = tool.getRuntime().lookupJavaMethod(ArrayCopySnippets.getSnippetForKind(componentKind));
+        return (StructuredGraph) snippetMethod.getCompilerStorage().get(Graph.class);
     }
 
     private static void unrollFixedLengthLoop(StructuredGraph snippetGraph, int length, LoweringTool tool) {
@@ -88,27 +87,43 @@
         new CanonicalizerPhase(tool.getRuntime(), tool.assumptions()).apply(snippetGraph);
     }
 
+    private static void replaceSnippetInvokes(StructuredGraph snippetGraph, ResolvedJavaMethod targetMethod, int bci) {
+        for (InvokeNode invoke : snippetGraph.getNodes(InvokeNode.class)) {
+            if (invoke.methodCallTarget().targetMethod() != targetMethod) {
+                throw new GraalInternalError("unexpected invoke in arraycopy snippet");
+            }
+            if (invoke.stateAfter().bci == FrameState.INVALID_FRAMESTATE_BCI) {
+                InvokeNode newInvoke = snippetGraph.add(new InvokeNode(invoke.methodCallTarget(), bci));
+                newInvoke.setStateAfter(snippetGraph.add(new FrameState(FrameState.AFTER_BCI)));
+                snippetGraph.replaceFixedWithFixed(invoke, newInvoke);
+            } else {
+                assert invoke.stateAfter().bci == FrameState.AFTER_BCI : invoke;
+            }
+        }
+    }
+
     @Override
-    public void lower(LoweringTool tool) {
-        ResolvedJavaMethod snippetMethod = selectSnippet(tool);
-        if (snippetMethod == null) {
-            snippetMethod = tool.getRuntime().lookupJavaMethod(ArrayCopySnippets.increaseGenericCallCounterMethod);
-            // we will call the generic method. the generic snippet will only increase the counter,
-            // not call the actual method. therefore we create a second invoke here.
-            ((StructuredGraph) graph()).addAfterFixed(this, createInvoke());
-        }
-        if (Debug.isLogEnabled()) {
-            Debug.log("%s > Intrinsify (%s)", Debug.currentScope(), snippetMethod.getSignature().getParameterType(0, snippetMethod.getDeclaringClass()).getComponentType());
+    protected StructuredGraph getSnippetGraph(LoweringTool tool) {
+        if (!GraalOptions.IntrinsifyArrayCopy) {
+            return null;
         }
 
-        StructuredGraph snippetGraph = (StructuredGraph) snippetMethod.getCompilerStorage().get(Graph.class);
-        assert snippetGraph != null : "ArrayCopySnippets should be installed";
-        if (getLength().isConstant()) {
-            snippetGraph = snippetGraph.copy();
-            unrollFixedLengthLoop(snippetGraph, getLength().asConstant().asInt(), tool);
+        StructuredGraph snippetGraph = selectSnippet(tool);
+        if (snippetGraph == null) {
+            ResolvedJavaMethod snippetMethod = tool.getRuntime().lookupJavaMethod(ArrayCopySnippets.genericArraycopySnippet);
+            snippetGraph = ((StructuredGraph) snippetMethod.getCompilerStorage().get(Graph.class)).copy();
+            assert snippetGraph != null : "ArrayCopySnippets should be installed";
+
+            replaceSnippetInvokes(snippetGraph, getTargetMethod(), getBci());
+        } else {
+            assert snippetGraph != null : "ArrayCopySnippets should be installed";
+
+            if (getLength().isConstant()) {
+                snippetGraph = snippetGraph.copy();
+                unrollFixedLengthLoop(snippetGraph, getLength().asConstant().asInt(), tool);
+            }
         }
-        InvokeNode invoke = replaceWithInvoke();
-        InliningUtil.inline(invoke, snippetGraph, false);
+        return snippetGraph;
     }
 
     private static boolean checkBounds(int position, int length, VirtualObjectNode virtualObject) {
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ArrayCopySnippets.java	Wed Feb 27 18:28:09 2013 +0100
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ArrayCopySnippets.java	Wed Feb 27 18:37:08 2013 +0100
@@ -39,12 +39,13 @@
 import com.oracle.graal.snippets.*;
 import com.oracle.graal.snippets.Snippet.ConstantParameter;
 import com.oracle.graal.snippets.nodes.*;
+import com.oracle.graal.word.*;
 
 @SuppressWarnings("unused")
 public class ArrayCopySnippets implements SnippetsInterface {
 
     private static final EnumMap<Kind, Method> arraycopyMethods = new EnumMap<>(Kind.class);
-    public static final Method increaseGenericCallCounterMethod;
+    public static final Method genericArraycopySnippet;
 
     private static void addArraycopySnippetMethod(Kind kind, Class<?> arrayClass) throws NoSuchMethodException {
         arraycopyMethods.put(kind, ArrayCopySnippets.class.getDeclaredMethod("arraycopy", arrayClass, int.class, arrayClass, int.class, int.class));
@@ -61,7 +62,7 @@
             addArraycopySnippetMethod(Kind.Float, float[].class);
             addArraycopySnippetMethod(Kind.Double, double[].class);
             addArraycopySnippetMethod(Kind.Object, Object[].class);
-            increaseGenericCallCounterMethod = ArrayCopySnippets.class.getDeclaredMethod("increaseGenericCallCounter", Object.class, int.class, Object.class, int.class, int.class);
+            genericArraycopySnippet = ArrayCopySnippets.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class);
         } catch (SecurityException | NoSuchMethodException e) {
             throw new GraalInternalError(e);
         }
@@ -75,7 +76,9 @@
     private static final long VECTOR_SIZE = arrayIndexScale(Kind.Long);
 
     public static void vectorizedCopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter("baseKind") Kind baseKind) {
-        checkInputs(src, srcPos, dest, destPos, length);
+        checkNonNull(src);
+        checkNonNull(dest);
+        checkLimits(src, srcPos, dest, destPos, length);
         int header = arrayBaseOffset(baseKind);
         int elementSize = arrayIndexScale(baseKind);
         long byteLength = (long) length * elementSize;
@@ -103,17 +106,24 @@
         }
     }
 
-    public static void checkInputs(Object src, int srcPos, Object dest, int destPos, int length) {
-        if (src == null) {
+    public static void checkNonNull(Object obj) {
+        if (obj == null) {
             probability(DEOPT_PATH_PROBABILITY);
             checkNPECounter.inc();
             DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
         }
-        if (dest == null) {
+    }
+
+    public static int checkArrayType(Word hub) {
+        int layoutHelper = readLayoutHelper(hub);
+        if (layoutHelper >= 0) {
             probability(DEOPT_PATH_PROBABILITY);
-            checkNPECounter.inc();
             DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
         }
+        return layoutHelper;
+    }
+
+    public static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length) {
         if (srcPos < 0) {
             probability(DEOPT_PATH_PROBABILITY);
             checkAIOOBECounter.inc();
@@ -181,7 +191,9 @@
     @Snippet
     public static void arraycopy(long[] src, int srcPos, long[] dest, int destPos, int length) {
         longCounter.inc();
-        checkInputs(src, srcPos, dest, destPos, length);
+        checkNonNull(src);
+        checkNonNull(dest);
+        checkLimits(src, srcPos, dest, destPos, length);
         Kind baseKind = Kind.Long;
         int header = arrayBaseOffset(baseKind);
         long byteLength = (long) length * arrayIndexScale(baseKind);
@@ -203,7 +215,9 @@
     @Snippet
     public static void arraycopy(double[] src, int srcPos, double[] dest, int destPos, int length) {
         doubleCounter.inc();
-        checkInputs(src, srcPos, dest, destPos, length);
+        checkNonNull(src);
+        checkNonNull(dest);
+        checkLimits(src, srcPos, dest, destPos, length);
         Kind baseKind = Kind.Double;
         int header = arrayBaseOffset(baseKind);
         long byteLength = (long) length * arrayIndexScale(baseKind);
@@ -226,7 +240,9 @@
     @Snippet
     public static void arraycopy(Object[] src, int srcPos, Object[] dest, int destPos, int length) {
         objectCounter.inc();
-        checkInputs(src, srcPos, dest, destPos, length);
+        checkNonNull(src);
+        checkNonNull(dest);
+        checkLimits(src, srcPos, dest, destPos, length);
         final int scale = arrayIndexScale(Kind.Object);
         int header = arrayBaseOffset(Kind.Object);
         if (src == dest && srcPos < destPos) { // bad aliased case
@@ -256,12 +272,67 @@
     }
 
     @Snippet
-    public static void increaseGenericCallCounter(Object src, int srcPos, Object dest, int destPos, int length) {
-        if (GraalOptions.SnippetCounters) {
-            if (src.getClass().getComponentType().isPrimitive()) {
-                genericPrimitiveCallCounter.inc();
-            } else {
-                genericObjectCallCounter.inc();
+    public static void arraycopy(Object src, int srcPos, Object dest, int destPos, int length) {
+
+        // loading the hubs also checks for nullness
+        Word srcHub = loadHub(src);
+        Word destHub = loadHub(dest);
+
+        int layoutHelper = checkArrayType(srcHub);
+        if (srcHub.equal(destHub) && src != dest) {
+            probability(FAST_PATH_PROBABILITY);
+
+            checkLimits(src, srcPos, dest, destPos, length);
+
+            arraycopyInnerloop(src, srcPos, dest, destPos, length, layoutHelper);
+        } else {
+            genericObjectCallCounter.inc();
+            System.arraycopy(src, srcPos, dest, destPos, length);
+        }
+    }
+
+    public static void arraycopyInnerloop(Object src, int srcPos, Object dest, int destPos, int length, int layoutHelper) {
+        int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift()) & layoutHelperLog2ElementSizeMask();
+        int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift()) & layoutHelperHeaderSizeMask();
+
+        Word memory = (Word) Word.fromObject(src);
+
+        Word srcOffset = (Word) Word.fromObject(src).add(headerSize).add(srcPos << log2ElementSize);
+        Word destOffset = (Word) Word.fromObject(dest).add(headerSize).add(destPos << log2ElementSize);
+        Word destStart = destOffset;
+        long sizeInBytes = ((long) length) << log2ElementSize;
+        Word destEnd = destOffset.add(Word.unsigned(length).shiftLeft(log2ElementSize));
+
+        int nonVectorBytes = (int) (sizeInBytes % VECTOR_SIZE);
+        Word destNonVectorEnd = destStart.add(nonVectorBytes);
+
+        while (destOffset.belowThan(destNonVectorEnd)) {
+            destOffset.writeByte(0, srcOffset.readByte(0, UNKNOWN_LOCATION), ANY_LOCATION);
+            destOffset = destOffset.add(1);
+            srcOffset = srcOffset.add(1);
+        }
+        while (destOffset.belowThan(destEnd)) {
+            destOffset.writeWord(0, srcOffset.readWord(0, UNKNOWN_LOCATION), ANY_LOCATION);
+            destOffset = destOffset.add(wordSize());
+            srcOffset = srcOffset.add(wordSize());
+        }
+
+        if ((layoutHelper & layoutHelperElementTypePrimitiveInPlace()) != 0) {
+            genericPrimitiveCallCounter.inc();
+
+        } else {
+            probability(LIKELY_PROBABILITY);
+            genericObjectExactCallCounter.inc();
+
+            if (length > 0) {
+                int cardShift = cardTableShift();
+                long cardStart = cardTableStart();
+                Word destCardOffset = destStart.unsignedShiftRight(cardShift).add(Word.unsigned(cardStart));
+                Word destCardEnd = destEnd.subtract(1).unsignedShiftRight(cardShift).add(Word.unsigned(cardStart));
+                while (destCardOffset.belowOrEqual(destCardEnd)) {
+                    DirectStoreNode.store(destCardOffset.rawValue(), false, Kind.Boolean);
+                    destCardOffset = destCardOffset.add(1);
+                }
             }
         }
     }
@@ -281,7 +352,8 @@
     private static final SnippetCounter objectCounter = new SnippetCounter(counters, "Object[]", "arraycopy for Object[] arrays");
     private static final SnippetCounter floatCounter = new SnippetCounter(counters, "float[]", "arraycopy for float[] arrays");
     private static final SnippetCounter doubleCounter = new SnippetCounter(counters, "double[]", "arraycopy for double[] arrays");
-    private static final SnippetCounter genericPrimitiveCallCounter = new SnippetCounter(counters, "genericPrimitive", "call to the generic, native arraycopy method");
+    private static final SnippetCounter genericPrimitiveCallCounter = new SnippetCounter(counters, "genericPrimitive", "generic arraycopy snippet for primitive arrays");
+    private static final SnippetCounter genericObjectExactCallCounter = new SnippetCounter(counters, "genericObjectExact", "generic arraycopy snippet for special object arrays");
     private static final SnippetCounter genericObjectCallCounter = new SnippetCounter(counters, "genericObject", "call to the generic, native arraycopy method");
 
 }
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ClassSubstitutions.java	Wed Feb 27 18:28:09 2013 +0100
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ClassSubstitutions.java	Wed Feb 27 18:37:08 2013 +0100
@@ -110,6 +110,6 @@
 
     @MethodSubstitution(isStatic = false)
     public static boolean isInstance(final Class<?> thisObj, Object obj) {
-        return !thisObj.isPrimitive() && ConditionalNode.materializeIsInstance(thisObj, obj);
+        return !isPrimitive(thisObj) && ConditionalNode.materializeIsInstance(thisObj, obj);
     }
 }
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/HotSpotSnippetUtils.java	Wed Feb 27 18:28:09 2013 +0100
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/HotSpotSnippetUtils.java	Wed Feb 27 18:37:08 2013 +0100
@@ -161,8 +161,8 @@
         return config().klassLayoutHelperOffset;
     }
 
-    public static int readLayoutHelper(Word klass) {
-        return klass.readInt(klassLayoutHelperOffset(), FINAL_LOCATION);
+    public static int readLayoutHelper(Word hub) {
+        return hub.readInt(klassLayoutHelperOffset(), FINAL_LOCATION);
     }
 
     @Fold
@@ -516,7 +516,6 @@
 
     @Fold
     public static int layoutHelperElementTypePrimitiveInPlace() {
-        System.out.println(String.format("%x", config().layoutHelperElementTypePrimitiveInPlace));
         return config().layoutHelperElementTypePrimitiveInPlace;
     }
 
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ObjectCloneNode.java	Wed Feb 27 18:28:09 2013 +0100
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ObjectCloneNode.java	Wed Feb 27 18:37:08 2013 +0100
@@ -26,7 +26,6 @@
 
 import com.oracle.graal.api.code.*;
 import com.oracle.graal.api.meta.*;
-import com.oracle.graal.debug.*;
 import com.oracle.graal.graph.*;
 import com.oracle.graal.nodes.*;
 import com.oracle.graal.nodes.java.*;
@@ -34,7 +33,6 @@
 import com.oracle.graal.nodes.type.*;
 import com.oracle.graal.nodes.virtual.*;
 import com.oracle.graal.phases.*;
-import com.oracle.graal.phases.common.*;
 import com.oracle.graal.snippets.nodes.*;
 
 public class ObjectCloneNode extends MacroNode implements VirtualizableAllocation, ArrayLengthProvider {
@@ -52,33 +50,30 @@
         return arguments.get(0);
     }
 
-    private Method selectSnippetMethod(LoweringTool tool) {
-        ResolvedJavaType type = getObject().objectStamp().type();
-        if (type.isArray()) {
-            return ObjectCloneSnippets.arrayCloneMethod;
-        } else if (type.isAssignableFrom(tool.getRuntime().lookupJavaType(Object[].class))) {
-            // arrays are assignable to Object, Cloneable and Serializable
-            return ObjectCloneSnippets.genericCloneMethod;
-        } else {
-            return ObjectCloneSnippets.instanceCloneMethod;
-        }
-    }
-
     @Override
-    public void lower(LoweringTool tool) {
+    protected StructuredGraph getSnippetGraph(LoweringTool tool) {
         if (!GraalOptions.IntrinsifyObjectClone) {
-            super.lower(tool);
-            return;
-        }
-        ResolvedJavaMethod snippetMethod = tool.getRuntime().lookupJavaMethod(selectSnippetMethod(tool));
-        if (Debug.isLogEnabled()) {
-            Debug.log("%s > Intrinsify (%s)", Debug.currentScope(), snippetMethod.getSignature().getParameterType(0, snippetMethod.getDeclaringClass()).getComponentType());
+            return null;
         }
 
+        ResolvedJavaType type = getObject().objectStamp().type();
+        Method method;
+        /*
+         * The first condition tests if the parameter is an array, the second condition tests if the
+         * parameter can be an array. Otherwise, the parameter is known to be a non-array object.
+         */
+        if (type.isArray()) {
+            method = ObjectCloneSnippets.arrayCloneMethod;
+        } else if (type == null || type.isAssignableFrom(tool.getRuntime().lookupJavaType(Object[].class))) {
+            method = ObjectCloneSnippets.genericCloneMethod;
+        } else {
+            method = ObjectCloneSnippets.instanceCloneMethod;
+        }
+        ResolvedJavaMethod snippetMethod = tool.getRuntime().lookupJavaMethod(method);
         StructuredGraph snippetGraph = (StructuredGraph) snippetMethod.getCompilerStorage().get(Graph.class);
+
         assert snippetGraph != null : "ObjectCloneSnippets should be installed";
-        InvokeNode invoke = replaceWithInvoke();
-        InliningUtil.inline(invoke, snippetGraph, false);
+        return snippetGraph;
     }
 
     private static boolean isCloneableType(ResolvedJavaType type, MetaAccessProvider metaAccess) {
--- a/graal/com.oracle.graal.snippets/src/com/oracle/graal/snippets/nodes/MacroNode.java	Wed Feb 27 18:28:09 2013 +0100
+++ b/graal/com.oracle.graal.snippets/src/com/oracle/graal/snippets/nodes/MacroNode.java	Wed Feb 27 18:37:08 2013 +0100
@@ -30,6 +30,7 @@
 import com.oracle.graal.nodes.java.*;
 import com.oracle.graal.nodes.java.MethodCallTargetNode.InvokeKind;
 import com.oracle.graal.nodes.spi.*;
+import com.oracle.graal.phases.common.*;
 
 public class MacroNode extends AbstractStateSplit implements Lowerable {
 
@@ -47,14 +48,32 @@
         this.returnType = invoke.methodCallTarget().returnType();
     }
 
+    public int getBci() {
+        return bci;
+    }
+
+    public ResolvedJavaMethod getTargetMethod() {
+        return targetMethod;
+    }
+
+    @SuppressWarnings("unused")
+    protected StructuredGraph getSnippetGraph(LoweringTool tool) {
+        return null;
+    }
+
     @Override
     public void lower(LoweringTool tool) {
-        replaceWithInvoke();
+        StructuredGraph snippetGraph = getSnippetGraph(tool);
+
+        InvokeNode invoke = replaceWithInvoke();
+
+        if (snippetGraph != null) {
+            InliningUtil.inline(invoke, snippetGraph, false);
+        }
     }
 
-    public InvokeNode replaceWithInvoke() {
+    private InvokeNode replaceWithInvoke() {
         InvokeNode invoke = createInvoke();
-
         ((StructuredGraph) graph()).replaceFixedWithFixed(this, invoke);
         return invoke;
     }