# HG changeset patch # User Lukas Stadler # Date 1361986628 -3600 # Node ID 14fedab0419e6be59bc24309f44cea089f1e5100 # Parent 93a133fc03ce5fb0651b1780fee63435a6bac55e improved arraycopy and Object.clone snippets diff -r 93a133fc03ce -r 14fedab0419e graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ArrayCopyNode.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ArrayCopyNode.java Wed Feb 27 18:28:09 2013 +0100 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ArrayCopyNode.java Wed Feb 27 18:37:08 2013 +0100 @@ -30,6 +30,7 @@ import com.oracle.graal.nodes.*; import com.oracle.graal.nodes.spi.*; import com.oracle.graal.nodes.virtual.*; +import com.oracle.graal.phases.*; import com.oracle.graal.phases.common.*; import com.oracle.graal.snippets.nodes.*; @@ -59,21 +60,19 @@ return arguments.get(4); } - private ResolvedJavaMethod selectSnippet(LoweringTool tool) { + private StructuredGraph selectSnippet(LoweringTool tool) { ResolvedJavaType srcType = getSource().objectStamp().type(); ResolvedJavaType destType = getDestination().objectStamp().type(); - if (srcType != null && srcType.isArray() && destType != null && destType.isArray()) { - Kind componentKind = srcType.getComponentType().getKind(); - if (componentKind != Kind.Object) { - if (srcType.getComponentType() == destType.getComponentType()) { - return tool.getRuntime().lookupJavaMethod(ArrayCopySnippets.getSnippetForKind(componentKind)); - } - } else if (destType.getComponentType().isAssignableFrom(srcType.getComponentType()) && getDestination().objectStamp().isExactType()) { - return tool.getRuntime().lookupJavaMethod(ArrayCopySnippets.getSnippetForKind(Kind.Object)); - } + if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) { + return null; } - return null; + if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType()) || !getDestination().objectStamp().isExactType()) { + return null; + } + Kind componentKind = srcType.getComponentType().getKind(); + ResolvedJavaMethod snippetMethod = tool.getRuntime().lookupJavaMethod(ArrayCopySnippets.getSnippetForKind(componentKind)); + return (StructuredGraph) snippetMethod.getCompilerStorage().get(Graph.class); } private static void unrollFixedLengthLoop(StructuredGraph snippetGraph, int length, LoweringTool tool) { @@ -88,27 +87,43 @@ new CanonicalizerPhase(tool.getRuntime(), tool.assumptions()).apply(snippetGraph); } + private static void replaceSnippetInvokes(StructuredGraph snippetGraph, ResolvedJavaMethod targetMethod, int bci) { + for (InvokeNode invoke : snippetGraph.getNodes(InvokeNode.class)) { + if (invoke.methodCallTarget().targetMethod() != targetMethod) { + throw new GraalInternalError("unexpected invoke in arraycopy snippet"); + } + if (invoke.stateAfter().bci == FrameState.INVALID_FRAMESTATE_BCI) { + InvokeNode newInvoke = snippetGraph.add(new InvokeNode(invoke.methodCallTarget(), bci)); + newInvoke.setStateAfter(snippetGraph.add(new FrameState(FrameState.AFTER_BCI))); + snippetGraph.replaceFixedWithFixed(invoke, newInvoke); + } else { + assert invoke.stateAfter().bci == FrameState.AFTER_BCI : invoke; + } + } + } + @Override - public void lower(LoweringTool tool) { - ResolvedJavaMethod snippetMethod = selectSnippet(tool); - if (snippetMethod == null) { - snippetMethod = tool.getRuntime().lookupJavaMethod(ArrayCopySnippets.increaseGenericCallCounterMethod); - // we will call the generic method. the generic snippet will only increase the counter, - // not call the actual method. therefore we create a second invoke here. - ((StructuredGraph) graph()).addAfterFixed(this, createInvoke()); - } - if (Debug.isLogEnabled()) { - Debug.log("%s > Intrinsify (%s)", Debug.currentScope(), snippetMethod.getSignature().getParameterType(0, snippetMethod.getDeclaringClass()).getComponentType()); + protected StructuredGraph getSnippetGraph(LoweringTool tool) { + if (!GraalOptions.IntrinsifyArrayCopy) { + return null; } - StructuredGraph snippetGraph = (StructuredGraph) snippetMethod.getCompilerStorage().get(Graph.class); - assert snippetGraph != null : "ArrayCopySnippets should be installed"; - if (getLength().isConstant()) { - snippetGraph = snippetGraph.copy(); - unrollFixedLengthLoop(snippetGraph, getLength().asConstant().asInt(), tool); + StructuredGraph snippetGraph = selectSnippet(tool); + if (snippetGraph == null) { + ResolvedJavaMethod snippetMethod = tool.getRuntime().lookupJavaMethod(ArrayCopySnippets.genericArraycopySnippet); + snippetGraph = ((StructuredGraph) snippetMethod.getCompilerStorage().get(Graph.class)).copy(); + assert snippetGraph != null : "ArrayCopySnippets should be installed"; + + replaceSnippetInvokes(snippetGraph, getTargetMethod(), getBci()); + } else { + assert snippetGraph != null : "ArrayCopySnippets should be installed"; + + if (getLength().isConstant()) { + snippetGraph = snippetGraph.copy(); + unrollFixedLengthLoop(snippetGraph, getLength().asConstant().asInt(), tool); + } } - InvokeNode invoke = replaceWithInvoke(); - InliningUtil.inline(invoke, snippetGraph, false); + return snippetGraph; } private static boolean checkBounds(int position, int length, VirtualObjectNode virtualObject) { diff -r 93a133fc03ce -r 14fedab0419e graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ArrayCopySnippets.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ArrayCopySnippets.java Wed Feb 27 18:28:09 2013 +0100 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ArrayCopySnippets.java Wed Feb 27 18:37:08 2013 +0100 @@ -39,12 +39,13 @@ import com.oracle.graal.snippets.*; import com.oracle.graal.snippets.Snippet.ConstantParameter; import com.oracle.graal.snippets.nodes.*; +import com.oracle.graal.word.*; @SuppressWarnings("unused") public class ArrayCopySnippets implements SnippetsInterface { private static final EnumMap arraycopyMethods = new EnumMap<>(Kind.class); - public static final Method increaseGenericCallCounterMethod; + public static final Method genericArraycopySnippet; private static void addArraycopySnippetMethod(Kind kind, Class arrayClass) throws NoSuchMethodException { arraycopyMethods.put(kind, ArrayCopySnippets.class.getDeclaredMethod("arraycopy", arrayClass, int.class, arrayClass, int.class, int.class)); @@ -61,7 +62,7 @@ addArraycopySnippetMethod(Kind.Float, float[].class); addArraycopySnippetMethod(Kind.Double, double[].class); addArraycopySnippetMethod(Kind.Object, Object[].class); - increaseGenericCallCounterMethod = ArrayCopySnippets.class.getDeclaredMethod("increaseGenericCallCounter", Object.class, int.class, Object.class, int.class, int.class); + genericArraycopySnippet = ArrayCopySnippets.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class); } catch (SecurityException | NoSuchMethodException e) { throw new GraalInternalError(e); } @@ -75,7 +76,9 @@ private static final long VECTOR_SIZE = arrayIndexScale(Kind.Long); public static void vectorizedCopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter("baseKind") Kind baseKind) { - checkInputs(src, srcPos, dest, destPos, length); + checkNonNull(src); + checkNonNull(dest); + checkLimits(src, srcPos, dest, destPos, length); int header = arrayBaseOffset(baseKind); int elementSize = arrayIndexScale(baseKind); long byteLength = (long) length * elementSize; @@ -103,17 +106,24 @@ } } - public static void checkInputs(Object src, int srcPos, Object dest, int destPos, int length) { - if (src == null) { + public static void checkNonNull(Object obj) { + if (obj == null) { probability(DEOPT_PATH_PROBABILITY); checkNPECounter.inc(); DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); } - if (dest == null) { + } + + public static int checkArrayType(Word hub) { + int layoutHelper = readLayoutHelper(hub); + if (layoutHelper >= 0) { probability(DEOPT_PATH_PROBABILITY); - checkNPECounter.inc(); DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); } + return layoutHelper; + } + + public static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length) { if (srcPos < 0) { probability(DEOPT_PATH_PROBABILITY); checkAIOOBECounter.inc(); @@ -181,7 +191,9 @@ @Snippet public static void arraycopy(long[] src, int srcPos, long[] dest, int destPos, int length) { longCounter.inc(); - checkInputs(src, srcPos, dest, destPos, length); + checkNonNull(src); + checkNonNull(dest); + checkLimits(src, srcPos, dest, destPos, length); Kind baseKind = Kind.Long; int header = arrayBaseOffset(baseKind); long byteLength = (long) length * arrayIndexScale(baseKind); @@ -203,7 +215,9 @@ @Snippet public static void arraycopy(double[] src, int srcPos, double[] dest, int destPos, int length) { doubleCounter.inc(); - checkInputs(src, srcPos, dest, destPos, length); + checkNonNull(src); + checkNonNull(dest); + checkLimits(src, srcPos, dest, destPos, length); Kind baseKind = Kind.Double; int header = arrayBaseOffset(baseKind); long byteLength = (long) length * arrayIndexScale(baseKind); @@ -226,7 +240,9 @@ @Snippet public static void arraycopy(Object[] src, int srcPos, Object[] dest, int destPos, int length) { objectCounter.inc(); - checkInputs(src, srcPos, dest, destPos, length); + checkNonNull(src); + checkNonNull(dest); + checkLimits(src, srcPos, dest, destPos, length); final int scale = arrayIndexScale(Kind.Object); int header = arrayBaseOffset(Kind.Object); if (src == dest && srcPos < destPos) { // bad aliased case @@ -256,12 +272,67 @@ } @Snippet - public static void increaseGenericCallCounter(Object src, int srcPos, Object dest, int destPos, int length) { - if (GraalOptions.SnippetCounters) { - if (src.getClass().getComponentType().isPrimitive()) { - genericPrimitiveCallCounter.inc(); - } else { - genericObjectCallCounter.inc(); + public static void arraycopy(Object src, int srcPos, Object dest, int destPos, int length) { + + // loading the hubs also checks for nullness + Word srcHub = loadHub(src); + Word destHub = loadHub(dest); + + int layoutHelper = checkArrayType(srcHub); + if (srcHub.equal(destHub) && src != dest) { + probability(FAST_PATH_PROBABILITY); + + checkLimits(src, srcPos, dest, destPos, length); + + arraycopyInnerloop(src, srcPos, dest, destPos, length, layoutHelper); + } else { + genericObjectCallCounter.inc(); + System.arraycopy(src, srcPos, dest, destPos, length); + } + } + + public static void arraycopyInnerloop(Object src, int srcPos, Object dest, int destPos, int length, int layoutHelper) { + int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift()) & layoutHelperLog2ElementSizeMask(); + int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift()) & layoutHelperHeaderSizeMask(); + + Word memory = (Word) Word.fromObject(src); + + Word srcOffset = (Word) Word.fromObject(src).add(headerSize).add(srcPos << log2ElementSize); + Word destOffset = (Word) Word.fromObject(dest).add(headerSize).add(destPos << log2ElementSize); + Word destStart = destOffset; + long sizeInBytes = ((long) length) << log2ElementSize; + Word destEnd = destOffset.add(Word.unsigned(length).shiftLeft(log2ElementSize)); + + int nonVectorBytes = (int) (sizeInBytes % VECTOR_SIZE); + Word destNonVectorEnd = destStart.add(nonVectorBytes); + + while (destOffset.belowThan(destNonVectorEnd)) { + destOffset.writeByte(0, srcOffset.readByte(0, UNKNOWN_LOCATION), ANY_LOCATION); + destOffset = destOffset.add(1); + srcOffset = srcOffset.add(1); + } + while (destOffset.belowThan(destEnd)) { + destOffset.writeWord(0, srcOffset.readWord(0, UNKNOWN_LOCATION), ANY_LOCATION); + destOffset = destOffset.add(wordSize()); + srcOffset = srcOffset.add(wordSize()); + } + + if ((layoutHelper & layoutHelperElementTypePrimitiveInPlace()) != 0) { + genericPrimitiveCallCounter.inc(); + + } else { + probability(LIKELY_PROBABILITY); + genericObjectExactCallCounter.inc(); + + if (length > 0) { + int cardShift = cardTableShift(); + long cardStart = cardTableStart(); + Word destCardOffset = destStart.unsignedShiftRight(cardShift).add(Word.unsigned(cardStart)); + Word destCardEnd = destEnd.subtract(1).unsignedShiftRight(cardShift).add(Word.unsigned(cardStart)); + while (destCardOffset.belowOrEqual(destCardEnd)) { + DirectStoreNode.store(destCardOffset.rawValue(), false, Kind.Boolean); + destCardOffset = destCardOffset.add(1); + } } } } @@ -281,7 +352,8 @@ private static final SnippetCounter objectCounter = new SnippetCounter(counters, "Object[]", "arraycopy for Object[] arrays"); private static final SnippetCounter floatCounter = new SnippetCounter(counters, "float[]", "arraycopy for float[] arrays"); private static final SnippetCounter doubleCounter = new SnippetCounter(counters, "double[]", "arraycopy for double[] arrays"); - private static final SnippetCounter genericPrimitiveCallCounter = new SnippetCounter(counters, "genericPrimitive", "call to the generic, native arraycopy method"); + private static final SnippetCounter genericPrimitiveCallCounter = new SnippetCounter(counters, "genericPrimitive", "generic arraycopy snippet for primitive arrays"); + private static final SnippetCounter genericObjectExactCallCounter = new SnippetCounter(counters, "genericObjectExact", "generic arraycopy snippet for special object arrays"); private static final SnippetCounter genericObjectCallCounter = new SnippetCounter(counters, "genericObject", "call to the generic, native arraycopy method"); } diff -r 93a133fc03ce -r 14fedab0419e graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ClassSubstitutions.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ClassSubstitutions.java Wed Feb 27 18:28:09 2013 +0100 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ClassSubstitutions.java Wed Feb 27 18:37:08 2013 +0100 @@ -110,6 +110,6 @@ @MethodSubstitution(isStatic = false) public static boolean isInstance(final Class thisObj, Object obj) { - return !thisObj.isPrimitive() && ConditionalNode.materializeIsInstance(thisObj, obj); + return !isPrimitive(thisObj) && ConditionalNode.materializeIsInstance(thisObj, obj); } } diff -r 93a133fc03ce -r 14fedab0419e graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/HotSpotSnippetUtils.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/HotSpotSnippetUtils.java Wed Feb 27 18:28:09 2013 +0100 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/HotSpotSnippetUtils.java Wed Feb 27 18:37:08 2013 +0100 @@ -161,8 +161,8 @@ return config().klassLayoutHelperOffset; } - public static int readLayoutHelper(Word klass) { - return klass.readInt(klassLayoutHelperOffset(), FINAL_LOCATION); + public static int readLayoutHelper(Word hub) { + return hub.readInt(klassLayoutHelperOffset(), FINAL_LOCATION); } @Fold @@ -516,7 +516,6 @@ @Fold public static int layoutHelperElementTypePrimitiveInPlace() { - System.out.println(String.format("%x", config().layoutHelperElementTypePrimitiveInPlace)); return config().layoutHelperElementTypePrimitiveInPlace; } diff -r 93a133fc03ce -r 14fedab0419e graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ObjectCloneNode.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ObjectCloneNode.java Wed Feb 27 18:28:09 2013 +0100 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ObjectCloneNode.java Wed Feb 27 18:37:08 2013 +0100 @@ -26,7 +26,6 @@ import com.oracle.graal.api.code.*; import com.oracle.graal.api.meta.*; -import com.oracle.graal.debug.*; import com.oracle.graal.graph.*; import com.oracle.graal.nodes.*; import com.oracle.graal.nodes.java.*; @@ -34,7 +33,6 @@ import com.oracle.graal.nodes.type.*; import com.oracle.graal.nodes.virtual.*; import com.oracle.graal.phases.*; -import com.oracle.graal.phases.common.*; import com.oracle.graal.snippets.nodes.*; public class ObjectCloneNode extends MacroNode implements VirtualizableAllocation, ArrayLengthProvider { @@ -52,33 +50,30 @@ return arguments.get(0); } - private Method selectSnippetMethod(LoweringTool tool) { - ResolvedJavaType type = getObject().objectStamp().type(); - if (type.isArray()) { - return ObjectCloneSnippets.arrayCloneMethod; - } else if (type.isAssignableFrom(tool.getRuntime().lookupJavaType(Object[].class))) { - // arrays are assignable to Object, Cloneable and Serializable - return ObjectCloneSnippets.genericCloneMethod; - } else { - return ObjectCloneSnippets.instanceCloneMethod; - } - } - @Override - public void lower(LoweringTool tool) { + protected StructuredGraph getSnippetGraph(LoweringTool tool) { if (!GraalOptions.IntrinsifyObjectClone) { - super.lower(tool); - return; - } - ResolvedJavaMethod snippetMethod = tool.getRuntime().lookupJavaMethod(selectSnippetMethod(tool)); - if (Debug.isLogEnabled()) { - Debug.log("%s > Intrinsify (%s)", Debug.currentScope(), snippetMethod.getSignature().getParameterType(0, snippetMethod.getDeclaringClass()).getComponentType()); + return null; } + ResolvedJavaType type = getObject().objectStamp().type(); + Method method; + /* + * The first condition tests if the parameter is an array, the second condition tests if the + * parameter can be an array. Otherwise, the parameter is known to be a non-array object. + */ + if (type.isArray()) { + method = ObjectCloneSnippets.arrayCloneMethod; + } else if (type == null || type.isAssignableFrom(tool.getRuntime().lookupJavaType(Object[].class))) { + method = ObjectCloneSnippets.genericCloneMethod; + } else { + method = ObjectCloneSnippets.instanceCloneMethod; + } + ResolvedJavaMethod snippetMethod = tool.getRuntime().lookupJavaMethod(method); StructuredGraph snippetGraph = (StructuredGraph) snippetMethod.getCompilerStorage().get(Graph.class); + assert snippetGraph != null : "ObjectCloneSnippets should be installed"; - InvokeNode invoke = replaceWithInvoke(); - InliningUtil.inline(invoke, snippetGraph, false); + return snippetGraph; } private static boolean isCloneableType(ResolvedJavaType type, MetaAccessProvider metaAccess) { diff -r 93a133fc03ce -r 14fedab0419e graal/com.oracle.graal.snippets/src/com/oracle/graal/snippets/nodes/MacroNode.java --- a/graal/com.oracle.graal.snippets/src/com/oracle/graal/snippets/nodes/MacroNode.java Wed Feb 27 18:28:09 2013 +0100 +++ b/graal/com.oracle.graal.snippets/src/com/oracle/graal/snippets/nodes/MacroNode.java Wed Feb 27 18:37:08 2013 +0100 @@ -30,6 +30,7 @@ import com.oracle.graal.nodes.java.*; import com.oracle.graal.nodes.java.MethodCallTargetNode.InvokeKind; import com.oracle.graal.nodes.spi.*; +import com.oracle.graal.phases.common.*; public class MacroNode extends AbstractStateSplit implements Lowerable { @@ -47,14 +48,32 @@ this.returnType = invoke.methodCallTarget().returnType(); } + public int getBci() { + return bci; + } + + public ResolvedJavaMethod getTargetMethod() { + return targetMethod; + } + + @SuppressWarnings("unused") + protected StructuredGraph getSnippetGraph(LoweringTool tool) { + return null; + } + @Override public void lower(LoweringTool tool) { - replaceWithInvoke(); + StructuredGraph snippetGraph = getSnippetGraph(tool); + + InvokeNode invoke = replaceWithInvoke(); + + if (snippetGraph != null) { + InliningUtil.inline(invoke, snippetGraph, false); + } } - public InvokeNode replaceWithInvoke() { + private InvokeNode replaceWithInvoke() { InvokeNode invoke = createInvoke(); - ((StructuredGraph) graph()).replaceFixedWithFixed(this, invoke); return invoke; }