# HG changeset patch # User Bernhard Urban # Date 1417595742 -7200 # Node ID 5afa718edb8b5bce9acc0b65ad78753551fc8e17 # Parent be63951fa2bcbed3d7e5b5da8ad58603c4a19189 arraycopy: move related classes to dedicated package diff -r be63951fa2bc -r 5afa718edb8b graal/com.oracle.graal.hotspot.hsail/src/com/oracle/graal/hotspot/hsail/HSAILHotSpotReplacementsImpl.java --- a/graal/com.oracle.graal.hotspot.hsail/src/com/oracle/graal/hotspot/hsail/HSAILHotSpotReplacementsImpl.java Tue Dec 02 20:48:12 2014 +0100 +++ b/graal/com.oracle.graal.hotspot.hsail/src/com/oracle/graal/hotspot/hsail/HSAILHotSpotReplacementsImpl.java Wed Dec 03 10:35:42 2014 +0200 @@ -28,7 +28,7 @@ import com.oracle.graal.api.code.*; import com.oracle.graal.api.meta.*; import com.oracle.graal.api.replacements.*; -import com.oracle.graal.hotspot.replacements.*; +import com.oracle.graal.hotspot.replacements.arraycopy.*; import com.oracle.graal.nodes.*; import com.oracle.graal.nodes.spi.*; import com.oracle.graal.phases.util.*; diff -r be63951fa2bc -r 5afa718edb8b graal/com.oracle.graal.hotspot.test/src/com/oracle/graal/hotspot/test/WriteBarrierVerificationTest.java --- a/graal/com.oracle.graal.hotspot.test/src/com/oracle/graal/hotspot/test/WriteBarrierVerificationTest.java Tue Dec 02 20:48:12 2014 +0100 +++ b/graal/com.oracle.graal.hotspot.test/src/com/oracle/graal/hotspot/test/WriteBarrierVerificationTest.java Wed Dec 03 10:35:42 2014 +0200 @@ -35,7 +35,7 @@ import com.oracle.graal.hotspot.*; import com.oracle.graal.hotspot.nodes.*; import com.oracle.graal.hotspot.phases.*; -import com.oracle.graal.hotspot.replacements.*; +import com.oracle.graal.hotspot.replacements.arraycopy.*; import com.oracle.graal.nodes.*; import com.oracle.graal.nodes.extended.*; import com.oracle.graal.nodes.spi.*; diff -r be63951fa2bc -r 5afa718edb8b graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/DefaultHotSpotLoweringProvider.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/DefaultHotSpotLoweringProvider.java Tue Dec 02 20:48:12 2014 +0100 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/DefaultHotSpotLoweringProvider.java Wed Dec 03 10:35:42 2014 +0200 @@ -40,6 +40,7 @@ import com.oracle.graal.hotspot.nodes.*; import com.oracle.graal.hotspot.nodes.type.*; import com.oracle.graal.hotspot.replacements.*; +import com.oracle.graal.hotspot.replacements.arraycopy.*; import com.oracle.graal.java.*; import com.oracle.graal.nodes.*; import com.oracle.graal.nodes.CallTargetNode.InvokeKind; diff -r be63951fa2bc -r 5afa718edb8b graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ArrayCopyCallNode.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ArrayCopyCallNode.java Tue Dec 02 20:48:12 2014 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,195 +0,0 @@ -/* - * Copyright (c) 2014, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -//JaCoCo Exclude -package com.oracle.graal.hotspot.replacements; - -import static com.oracle.graal.api.meta.LocationIdentity.*; - -import com.oracle.graal.api.meta.*; -import com.oracle.graal.api.runtime.*; -import com.oracle.graal.compiler.common.type.*; -import com.oracle.graal.hotspot.*; -import com.oracle.graal.hotspot.meta.*; -import com.oracle.graal.hotspot.nodes.*; -import com.oracle.graal.nodeinfo.*; -import com.oracle.graal.nodes.*; -import com.oracle.graal.nodes.calc.*; -import com.oracle.graal.nodes.extended.*; -import com.oracle.graal.nodes.spi.*; -import com.oracle.graal.runtime.*; - -@NodeInfo(allowedUsageTypes = {InputType.Memory}) -public class ArrayCopyCallNode extends AbstractMemoryCheckpoint implements Lowerable, MemoryCheckpoint.Single { - - @Input ValueNode src; - @Input ValueNode srcPos; - @Input ValueNode dest; - @Input ValueNode destPos; - @Input ValueNode length; - - protected Kind elementKind; - - /** - * Aligned means that the offset of the copy is heap word aligned. - */ - protected boolean aligned; - protected boolean disjoint; - protected boolean uninitialized; - - protected final HotSpotGraalRuntimeProvider runtime; - - public static ArrayCopyCallNode create(@InjectedNodeParameter HotSpotGraalRuntimeProvider runtime, ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, - Kind elementKind, boolean aligned, boolean disjoint, boolean uninitialized) { - return new ArrayCopyCallNode(src, srcPos, dest, destPos, length, elementKind, aligned, disjoint, uninitialized, runtime); - } - - protected ArrayCopyCallNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, Kind elementKind, boolean aligned, boolean disjoint, boolean uninitialized, - HotSpotGraalRuntimeProvider runtime) { - super(StampFactory.forVoid()); - assert elementKind != null; - this.src = src; - this.srcPos = srcPos; - this.dest = dest; - this.destPos = destPos; - this.length = length; - this.elementKind = elementKind; - this.aligned = aligned; - this.disjoint = disjoint; - this.uninitialized = uninitialized; - this.runtime = runtime; - } - - public static ArrayCopyCallNode create(@InjectedNodeParameter HotSpotGraalRuntimeProvider runtime, ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, - Kind elementKind, boolean disjoint) { - return new ArrayCopyCallNode(src, srcPos, dest, destPos, length, elementKind, false, disjoint, false, runtime); - } - - public ValueNode getSource() { - return src; - } - - public ValueNode getSourcePosition() { - return srcPos; - } - - public ValueNode getDestination() { - return dest; - } - - public ValueNode getDestinationPosition() { - return destPos; - } - - public ValueNode getLength() { - return length; - } - - public Kind getElementKind() { - return elementKind; - } - - private ValueNode computeBase(ValueNode base, ValueNode pos) { - FixedWithNextNode basePtr = graph().add(GetObjectAddressNode.create(base)); - graph().addBeforeFixed(this, basePtr); - ValueNode loc = IndexedLocationNode.create(getLocationIdentity(), elementKind, runtime.getArrayBaseOffset(elementKind), pos, graph(), runtime.getArrayIndexScale(elementKind)); - return graph().unique(ComputeAddressNode.create(basePtr, loc, StampFactory.forKind(Kind.Long))); - } - - @Override - public void lower(LoweringTool tool) { - if (graph().getGuardsStage() == StructuredGraph.GuardsStage.AFTER_FSA) { - updateAlignedDisjoint(); - ForeignCallDescriptor desc = HotSpotHostForeignCallsProvider.lookupArraycopyDescriptor(elementKind, isAligned(), isDisjoint(), isUninitialized()); - StructuredGraph graph = graph(); - ValueNode srcAddr = computeBase(getSource(), getSourcePosition()); - ValueNode destAddr = computeBase(getDestination(), getDestinationPosition()); - ValueNode len = getLength(); - if (len.stamp().getStackKind() != Kind.Long) { - len = IntegerConvertNode.convert(len, StampFactory.forKind(Kind.Long), graph()); - } - ForeignCallNode call = graph.add(ForeignCallNode.create(Graal.getRequiredCapability(RuntimeProvider.class).getHostBackend().getForeignCalls(), desc, srcAddr, destAddr, len)); - call.setStateAfter(stateAfter()); - graph.replaceFixedWithFixed(this, call); - - } - } - - @Override - public LocationIdentity getLocationIdentity() { - if (elementKind != null) { - return NamedLocationIdentity.getArrayLocation(elementKind); - } - return ANY_LOCATION; - } - - @NodeIntrinsic - private static native void arraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter Kind elementKind, @ConstantNodeParameter boolean aligned, - @ConstantNodeParameter boolean disjoint, @ConstantNodeParameter boolean uninitialized); - - public static void arraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter Kind elementKind, boolean aligned, boolean disjoint) { - arraycopy(src, srcPos, dest, destPos, length, elementKind, aligned, disjoint, false); - } - - public static void disjointArraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter Kind elementKind) { - arraycopy(src, srcPos, dest, destPos, length, elementKind, false, true, false); - } - - public static void disjointUninitializedArraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter Kind elementKind) { - arraycopy(src, srcPos, dest, destPos, length, elementKind, false, true, true); - } - - public boolean isAligned() { - return aligned; - } - - public boolean isDisjoint() { - return disjoint; - } - - public boolean isUninitialized() { - return uninitialized; - } - - boolean isHeapWordAligned(JavaConstant value, Kind kind) { - return (runtime.getArrayBaseOffset(kind) + (long) value.asInt() * runtime.getArrayIndexScale(kind)) % runtime.getConfig().heapWordSize == 0; - } - - public void updateAlignedDisjoint() { - Kind componentKind = elementKind; - if (srcPos == destPos) { - // Can treat as disjoint - disjoint = true; - } - PrimitiveConstant constantSrc = (PrimitiveConstant) srcPos.stamp().asConstant(); - PrimitiveConstant constantDst = (PrimitiveConstant) destPos.stamp().asConstant(); - if (constantSrc != null && constantDst != null) { - if (!aligned) { - aligned = isHeapWordAligned(constantSrc, componentKind) && isHeapWordAligned(constantDst, componentKind); - } - if (constantSrc.asInt() >= constantDst.asInt()) { - // low to high copy so treat as disjoint - disjoint = true; - } - } - } -} diff -r be63951fa2bc -r 5afa718edb8b graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ArrayCopyNode.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ArrayCopyNode.java Tue Dec 02 20:48:12 2014 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,148 +0,0 @@ -/* - * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package com.oracle.graal.hotspot.replacements; - -import static com.oracle.graal.compiler.GraalCompiler.*; - -import com.oracle.graal.api.meta.*; -import com.oracle.graal.compiler.common.*; -import com.oracle.graal.debug.*; -import com.oracle.graal.debug.Debug.Scope; -import com.oracle.graal.loop.phases.*; -import com.oracle.graal.nodeinfo.*; -import com.oracle.graal.nodes.*; -import com.oracle.graal.nodes.spi.*; -import com.oracle.graal.nodes.type.*; -import com.oracle.graal.phases.common.*; -import com.oracle.graal.phases.tiers.*; -import com.oracle.graal.replacements.nodes.*; - -@NodeInfo -public class ArrayCopyNode extends BasicArrayCopyNode implements Virtualizable, Lowerable { - - public static ArrayCopyNode create(Invoke invoke) { - return new ArrayCopyNode(invoke); - } - - protected ArrayCopyNode(Invoke invoke) { - super(invoke); - } - - private StructuredGraph selectSnippet(LoweringTool tool, final Replacements replacements) { - ResolvedJavaType srcType = StampTool.typeOrNull(getSource().stamp()); - ResolvedJavaType destType = StampTool.typeOrNull(getDestination().stamp()); - - if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) { - return null; - } - if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) { - return null; - } - if (!isExact()) { - return null; - } - Kind componentKind = srcType.getComponentType().getKind(); - final ResolvedJavaMethod snippetMethod = tool.getMetaAccess().lookupJavaMethod(ArrayCopySnippets.getSnippetForKind(componentKind, shouldUnroll(), isExact())); - try (Scope s = Debug.scope("ArrayCopySnippet", snippetMethod)) { - return replacements.getSnippet(snippetMethod); - } catch (Throwable e) { - throw Debug.handle(e); - } - } - - private static void unrollFixedLengthLoop(StructuredGraph snippetGraph, int length, LoweringTool tool) { - ParameterNode lengthParam = snippetGraph.getParameter(4); - if (lengthParam != null) { - snippetGraph.replaceFloating(lengthParam, ConstantNode.forInt(length, snippetGraph)); - } - // the canonicalization before loop unrolling is needed to propagate the length into - // additions, etc. - PhaseContext context = new PhaseContext(tool.getMetaAccess(), tool.getConstantReflection(), tool.getLowerer(), tool.getReplacements(), tool.assumptions(), tool.getStampProvider()); - new CanonicalizerPhase(true).apply(snippetGraph, context); - new LoopFullUnrollPhase(new CanonicalizerPhase(true)).apply(snippetGraph, context); - new CanonicalizerPhase(true).apply(snippetGraph, context); - } - - @Override - protected StructuredGraph getLoweredSnippetGraph(final LoweringTool tool) { - if (!shouldIntrinsify(getTargetMethod())) { - return null; - } - - final Replacements replacements = tool.getReplacements(); - StructuredGraph snippetGraph = selectSnippet(tool, replacements); - if (snippetGraph == null) { - ResolvedJavaType srcType = StampTool.typeOrNull(getSource().stamp()); - ResolvedJavaType destType = StampTool.typeOrNull(getDestination().stamp()); - ResolvedJavaType srcComponentType = srcType == null ? null : srcType.getComponentType(); - ResolvedJavaType destComponentType = destType == null ? null : destType.getComponentType(); - ResolvedJavaMethod snippetMethod = null; - if (srcComponentType != null && destComponentType != null && srcComponentType.getKind().equals(Kind.Object) && destComponentType.getKind().equals(Kind.Object)) { - snippetMethod = tool.getMetaAccess().lookupJavaMethod(ArrayCopySnippets.checkcastArraycopySnippet); - } else { - snippetMethod = tool.getMetaAccess().lookupJavaMethod(ArrayCopySnippets.genericArraycopySnippet); - } - snippetGraph = null; - try (Scope s = Debug.scope("ArrayCopySnippet", snippetMethod)) { - snippetGraph = replacements.getSnippet(snippetMethod, getTargetMethod()).copy(); - } catch (Throwable e) { - throw Debug.handle(e); - } - replaceSnippetInvokes(snippetGraph); - } else { - assert snippetGraph != null : "ArrayCopySnippets should be installed"; - snippetGraph = snippetGraph.copy(); - if (shouldUnroll()) { - final StructuredGraph copy = snippetGraph; - try (Scope s = Debug.scope("ArrayCopySnippetSpecialization", snippetGraph.method())) { - unrollFixedLengthLoop(copy, getLength().asJavaConstant().asInt(), tool); - } catch (Throwable e) { - throw Debug.handle(e); - } - } - } - return lowerReplacement(snippetGraph, tool); - } - - private boolean shouldUnroll() { - return getLength().isConstant() && getLength().asJavaConstant().asInt() <= GraalOptions.MaximumEscapeAnalysisArrayLength.getValue(); - } - - /* - * Returns true if this copy doesn't require store checks. Trivially true for primitive arrays. - */ - private boolean isExact() { - ResolvedJavaType srcType = StampTool.typeOrNull(getSource().stamp()); - if (srcType.getComponentType().getKind().isPrimitive() || getSource() == getDestination()) { - return true; - } - - ResolvedJavaType destType = StampTool.typeOrNull(getDestination().stamp()); - if (StampTool.isExactType(getDestination().stamp())) { - if (destType != null && destType.isAssignableFrom(srcType)) { - return true; - } - } - return false; - } -} diff -r be63951fa2bc -r 5afa718edb8b graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ArrayCopySnippets.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ArrayCopySnippets.java Tue Dec 02 20:48:12 2014 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,318 +0,0 @@ -/* - * Copyright (c) 2011, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package com.oracle.graal.hotspot.replacements; - -import static com.oracle.graal.compiler.common.GraalOptions.*; -import static com.oracle.graal.hotspot.replacements.HotSpotReplacementsUtil.*; -import static com.oracle.graal.nodes.GuardingPiNode.*; -import static com.oracle.graal.nodes.extended.BranchProbabilityNode.*; - -import java.lang.reflect.*; -import java.util.*; - -import com.oracle.graal.api.meta.*; -import com.oracle.graal.compiler.common.*; -import com.oracle.graal.graph.Node.ConstantNodeParameter; -import com.oracle.graal.graph.Node.NodeIntrinsic; -import com.oracle.graal.hotspot.word.*; -import com.oracle.graal.nodes.*; -import com.oracle.graal.nodes.extended.*; -import com.oracle.graal.nodes.java.*; -import com.oracle.graal.replacements.*; -import com.oracle.graal.word.*; - -public class ArrayCopySnippets implements Snippets { - - private static final EnumMap arraycopyMethods = new EnumMap<>(Kind.class); - private static final EnumMap arraycopyCalls = new EnumMap<>(Kind.class); - - public static final Method checkcastArraycopySnippet; - public static final Method genericArraycopySnippet; - - private static void addArraycopySnippetMethod(Kind kind, Class arrayClass) throws NoSuchMethodException { - arraycopyMethods.put(kind, ArrayCopySnippets.class.getDeclaredMethod("arraycopy", arrayClass, int.class, arrayClass, int.class, int.class)); - if (CallArrayCopy.getValue()) { - if (kind == Kind.Object) { - arraycopyCalls.put(kind, ArrayCopySnippets.class.getDeclaredMethod("objectArraycopyUnchecked", arrayClass, int.class, arrayClass, int.class, int.class)); - } else { - arraycopyCalls.put(kind, ArrayCopySnippets.class.getDeclaredMethod(kind + "Arraycopy", arrayClass, int.class, arrayClass, int.class, int.class)); - } - } - } - - static { - try { - addArraycopySnippetMethod(Kind.Byte, byte[].class); - addArraycopySnippetMethod(Kind.Boolean, boolean[].class); - addArraycopySnippetMethod(Kind.Char, char[].class); - addArraycopySnippetMethod(Kind.Short, short[].class); - addArraycopySnippetMethod(Kind.Int, int[].class); - addArraycopySnippetMethod(Kind.Long, long[].class); - addArraycopySnippetMethod(Kind.Float, float[].class); - addArraycopySnippetMethod(Kind.Double, double[].class); - addArraycopySnippetMethod(Kind.Object, Object[].class); - checkcastArraycopySnippet = ArrayCopySnippets.class.getDeclaredMethod("checkcastArraycopy", Object[].class, int.class, Object[].class, int.class, int.class); - genericArraycopySnippet = ArrayCopySnippets.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class); - } catch (SecurityException | NoSuchMethodException e) { - throw new GraalInternalError(e); - } - } - - public static Method getSnippetForKind(Kind kind, boolean shouldUnroll, boolean exact) { - Method m = null; - if (!shouldUnroll && exact) { - // use hotspot stubs - m = arraycopyCalls.get(kind); - if (m != null) { - return m; - } - } - // use snippets - return arraycopyMethods.get(kind); - } - - private static void checkedCopy(Object src, int srcPos, Object dest, int destPos, int length, Kind baseKind) { - Object nonNullSrc = guardingNonNull(src); - Object nonNullDest = guardingNonNull(dest); - checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); - UnsafeArrayCopyNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, baseKind); - } - - private static int checkArrayType(KlassPointer hub) { - int layoutHelper = readLayoutHelper(hub); - if (probability(SLOW_PATH_PROBABILITY, layoutHelper >= 0)) { - DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); - } - return layoutHelper; - } - - private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length) { - if (probability(SLOW_PATH_PROBABILITY, srcPos < 0)) { - checkAIOOBECounter.inc(); - DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); - } - if (probability(SLOW_PATH_PROBABILITY, destPos < 0)) { - checkAIOOBECounter.inc(); - DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); - } - if (probability(SLOW_PATH_PROBABILITY, length < 0)) { - checkAIOOBECounter.inc(); - DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); - } - if (probability(SLOW_PATH_PROBABILITY, srcPos + length > ArrayLengthNode.arrayLength(src))) { - checkAIOOBECounter.inc(); - DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); - } - if (probability(SLOW_PATH_PROBABILITY, destPos + length > ArrayLengthNode.arrayLength(dest))) { - checkAIOOBECounter.inc(); - DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); - } - checkSuccessCounter.inc(); - } - - @Snippet - public static void arraycopy(byte[] src, int srcPos, byte[] dest, int destPos, int length) { - byteCounter.inc(); - checkedCopy(src, srcPos, dest, destPos, length, Kind.Byte); - } - - @Snippet - public static void arraycopy(boolean[] src, int srcPos, boolean[] dest, int destPos, int length) { - booleanCounter.inc(); - checkedCopy(src, srcPos, dest, destPos, length, Kind.Boolean); - } - - @Snippet - public static void arraycopy(char[] src, int srcPos, char[] dest, int destPos, int length) { - charCounter.inc(); - checkedCopy(src, srcPos, dest, destPos, length, Kind.Char); - } - - @Snippet - public static void arraycopy(short[] src, int srcPos, short[] dest, int destPos, int length) { - shortCounter.inc(); - checkedCopy(src, srcPos, dest, destPos, length, Kind.Short); - } - - @Snippet - public static void arraycopy(int[] src, int srcPos, int[] dest, int destPos, int length) { - intCounter.inc(); - checkedCopy(src, srcPos, dest, destPos, length, Kind.Int); - } - - @Snippet - public static void arraycopy(float[] src, int srcPos, float[] dest, int destPos, int length) { - floatCounter.inc(); - checkedCopy(src, srcPos, dest, destPos, length, Kind.Float); - } - - @Snippet - public static void arraycopy(long[] src, int srcPos, long[] dest, int destPos, int length) { - longCounter.inc(); - checkedCopy(src, srcPos, dest, destPos, length, Kind.Long); - } - - @Snippet - public static void arraycopy(double[] src, int srcPos, double[] dest, int destPos, int length) { - doubleCounter.inc(); - checkedCopy(src, srcPos, dest, destPos, length, Kind.Double); - } - - @Snippet - public static void arraycopy(Object[] src, int srcPos, Object[] dest, int destPos, int length) { - objectCounter.inc(); - checkedCopy(src, srcPos, dest, destPos, length, Kind.Object); - } - - @Snippet - public static void checkcastArraycopy(Object[] src, int srcPos, Object[] dest, int destPos, int length) { - objectCheckcastCounter.inc(); - Object nonNullSrc = guardingNonNull(src); - Object nonNullDest = guardingNonNull(dest); - checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); - KlassPointer destElemKlass = loadHub(nonNullDest); - checkcastArraycopyHelper(srcPos, destPos, length, nonNullSrc, nonNullDest, destElemKlass); - } - - private static void checkcastArraycopyHelper(int srcPos, int destPos, int length, Object nonNullSrc, Object nonNullDest, KlassPointer destElemKlass) { - Word superCheckOffset = Word.signed(destElemKlass.readInt(superCheckOffsetOffset(), KLASS_SUPER_CHECK_OFFSET_LOCATION)); - int copiedElements = CheckcastArrayCopyCallNode.checkcastArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, superCheckOffset, destElemKlass, false); - if (copiedElements != 0) { - // the checkcast stub doesn't throw the ArrayStoreException, but returns the number of - // copied elements (xor'd with -1). - copiedElements ^= -1; - System.arraycopy(nonNullSrc, srcPos + copiedElements, nonNullDest, destPos + copiedElements, length - copiedElements); - } - } - - @Snippet - public static void arraycopy(Object src, int srcPos, Object dest, int destPos, int length) { - Object nonNullSrc = guardingNonNull(src); - Object nonNullDest = guardingNonNull(dest); - KlassPointer srcHub = loadHub(nonNullSrc); - KlassPointer destHub = loadHub(nonNullDest); - if (probability(FAST_PATH_PROBABILITY, srcHub.equal(destHub)) && probability(FAST_PATH_PROBABILITY, nonNullSrc != nonNullDest)) { - int layoutHelper = checkArrayType(srcHub); - final boolean isObjectArray = ((layoutHelper & layoutHelperElementTypePrimitiveInPlace()) == 0); - checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); - if (probability(FAST_PATH_PROBABILITY, isObjectArray)) { - genericObjectExactCallCounter.inc(); - UnsafeArrayCopyNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, Kind.Object); - } else { - genericPrimitiveCallCounter.inc(); - UnsafeArrayCopyNode.arraycopyPrimitive(nonNullSrc, srcPos, nonNullDest, destPos, length, layoutHelper); - } - } else { - genericObjectCallCounter.inc(); - System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length); - } - } - - @NodeIntrinsic(ForeignCallNode.class) - public static native void callArraycopy(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word src, Word dest, Word len); - - private static void callArraycopyTemplate(SnippetCounter counter, Kind kind, boolean aligned, boolean disjoint, Object src, int srcPos, Object dest, int destPos, int length) { - counter.inc(); - Object nonNullSrc = guardingNonNull(src); - Object nonNullDest = guardingNonNull(dest); - checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); - ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, kind, aligned, disjoint); - } - - @Snippet - public static void objectArraycopyUnchecked(Object[] src, int srcPos, Object[] dest, int destPos, int length) { - callArraycopyTemplate(objectCallCounter, Kind.Object, false, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void byteArraycopy(byte[] src, int srcPos, byte[] dest, int destPos, int length) { - callArraycopyTemplate(byteCallCounter, Kind.Byte, false, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void booleanArraycopy(boolean[] src, int srcPos, boolean[] dest, int destPos, int length) { - callArraycopyTemplate(booleanCallCounter, Kind.Boolean, false, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void charArraycopy(char[] src, int srcPos, char[] dest, int destPos, int length) { - callArraycopyTemplate(charCallCounter, Kind.Char, false, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void shortArraycopy(short[] src, int srcPos, short[] dest, int destPos, int length) { - callArraycopyTemplate(shortCallCounter, Kind.Short, false, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void intArraycopy(int[] src, int srcPos, int[] dest, int destPos, int length) { - callArraycopyTemplate(intCallCounter, Kind.Int, false, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void floatArraycopy(float[] src, int srcPos, float[] dest, int destPos, int length) { - callArraycopyTemplate(floatCallCounter, Kind.Float, false, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void longArraycopy(long[] src, int srcPos, long[] dest, int destPos, int length) { - callArraycopyTemplate(longCallCounter, Kind.Long, false, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void doubleArraycopy(double[] src, int srcPos, double[] dest, int destPos, int length) { - callArraycopyTemplate(doubleCallCounter, Kind.Double, false, false, src, srcPos, dest, destPos, length); - } - - private static final SnippetCounter.Group checkCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy checkInputs") : null; - private static final SnippetCounter checkSuccessCounter = new SnippetCounter(checkCounters, "checkSuccess", "checkSuccess"); - private static final SnippetCounter checkNPECounter = new SnippetCounter(checkCounters, "checkNPE", "checkNPE"); - private static final SnippetCounter checkAIOOBECounter = new SnippetCounter(checkCounters, "checkAIOOBE", "checkAIOOBE"); - - private static final SnippetCounter.Group counters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy") : null; - private static final SnippetCounter byteCounter = new SnippetCounter(counters, "byte[]", "arraycopy for byte[] arrays"); - private static final SnippetCounter charCounter = new SnippetCounter(counters, "char[]", "arraycopy for char[] arrays"); - private static final SnippetCounter shortCounter = new SnippetCounter(counters, "short[]", "arraycopy for short[] arrays"); - private static final SnippetCounter intCounter = new SnippetCounter(counters, "int[]", "arraycopy for int[] arrays"); - private static final SnippetCounter booleanCounter = new SnippetCounter(counters, "boolean[]", "arraycopy for boolean[] arrays"); - private static final SnippetCounter longCounter = new SnippetCounter(counters, "long[]", "arraycopy for long[] arrays"); - private static final SnippetCounter objectCounter = new SnippetCounter(counters, "Object[]", "arraycopy for Object[] arrays"); - private static final SnippetCounter objectCheckcastCounter = new SnippetCounter(counters, "Object[]", "arraycopy for non-exact Object[] arrays"); - private static final SnippetCounter floatCounter = new SnippetCounter(counters, "float[]", "arraycopy for float[] arrays"); - private static final SnippetCounter doubleCounter = new SnippetCounter(counters, "double[]", "arraycopy for double[] arrays"); - - private static final SnippetCounter objectCallCounter = new SnippetCounter(counters, "Object[]", "arraycopy call for Object[] arrays"); - - private static final SnippetCounter booleanCallCounter = new SnippetCounter(counters, "boolean[]", "arraycopy call for boolean[] arrays"); - private static final SnippetCounter byteCallCounter = new SnippetCounter(counters, "byte[]", "arraycopy call for byte[] arrays"); - private static final SnippetCounter charCallCounter = new SnippetCounter(counters, "char[]", "arraycopy call for char[] arrays"); - private static final SnippetCounter doubleCallCounter = new SnippetCounter(counters, "double[]", "arraycopy call for double[] arrays"); - private static final SnippetCounter floatCallCounter = new SnippetCounter(counters, "float[]", "arraycopy call for float[] arrays"); - private static final SnippetCounter intCallCounter = new SnippetCounter(counters, "int[]", "arraycopy call for int[] arrays"); - private static final SnippetCounter longCallCounter = new SnippetCounter(counters, "long[]", "arraycopy call for long[] arrays"); - private static final SnippetCounter shortCallCounter = new SnippetCounter(counters, "short[]", "arraycopy call for short[] arrays"); - - private static final SnippetCounter genericPrimitiveCallCounter = new SnippetCounter(counters, "genericPrimitive", "generic arraycopy snippet for primitive arrays"); - private static final SnippetCounter genericObjectExactCallCounter = new SnippetCounter(counters, "genericObjectExact", "generic arraycopy snippet for special object arrays"); - private static final SnippetCounter genericObjectCallCounter = new SnippetCounter(counters, "genericObject", "call to the generic, native arraycopy method"); -} diff -r be63951fa2bc -r 5afa718edb8b graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/CheckcastArrayCopyCallNode.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/CheckcastArrayCopyCallNode.java Tue Dec 02 20:48:12 2014 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,130 +0,0 @@ -/* - * Copyright (c) 2014, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -//JaCoCo Exclude -package com.oracle.graal.hotspot.replacements; - -import com.oracle.graal.api.meta.*; -import com.oracle.graal.api.runtime.*; -import com.oracle.graal.compiler.common.type.*; -import com.oracle.graal.hotspot.*; -import com.oracle.graal.hotspot.meta.*; -import com.oracle.graal.hotspot.nodes.*; -import com.oracle.graal.nodeinfo.*; -import com.oracle.graal.nodes.*; -import com.oracle.graal.nodes.calc.*; -import com.oracle.graal.nodes.extended.*; -import com.oracle.graal.nodes.spi.*; -import com.oracle.graal.runtime.*; -import com.oracle.graal.word.*; - -@NodeInfo(allowedUsageTypes = {InputType.Memory, InputType.Value}) -public class CheckcastArrayCopyCallNode extends AbstractMemoryCheckpoint implements Lowerable, MemoryCheckpoint.Single { - - @Input ValueNode src; - @Input ValueNode srcPos; - @Input ValueNode dest; - @Input ValueNode destPos; - @Input ValueNode length; - @Input ValueNode destElemKlass; - @Input ValueNode superCheckOffset; - - protected final boolean uninit; - - protected final HotSpotGraalRuntimeProvider runtime; - - public static CheckcastArrayCopyCallNode create(@InjectedNodeParameter HotSpotGraalRuntimeProvider runtime, ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, - ValueNode superCheckOffset, ValueNode destElemKlass, boolean uninit) { - return new CheckcastArrayCopyCallNode(src, srcPos, dest, destPos, length, superCheckOffset, destElemKlass, uninit, runtime); - } - - protected CheckcastArrayCopyCallNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, ValueNode superCheckOffset, ValueNode destElemKlass, boolean uninit, - HotSpotGraalRuntimeProvider runtime) { - super(StampFactory.forKind(Kind.Int)); - this.src = src; - this.srcPos = srcPos; - this.dest = dest; - this.destPos = destPos; - this.length = length; - this.superCheckOffset = superCheckOffset; - this.destElemKlass = destElemKlass; - this.uninit = uninit; - this.runtime = runtime; - } - - public ValueNode getSource() { - return src; - } - - public ValueNode getSourcePosition() { - return srcPos; - } - - public ValueNode getDestination() { - return dest; - } - - public ValueNode getDestinationPosition() { - return destPos; - } - - public ValueNode getLength() { - return length; - } - - public boolean isUninit() { - return uninit; - } - - private ValueNode computeBase(ValueNode base, ValueNode pos) { - FixedWithNextNode basePtr = graph().add(GetObjectAddressNode.create(base)); - graph().addBeforeFixed(this, basePtr); - ValueNode loc = IndexedLocationNode.create(getLocationIdentity(), Kind.Object, runtime.getArrayBaseOffset(Kind.Object), pos, graph(), runtime.getArrayIndexScale(Kind.Object)); - return graph().unique(ComputeAddressNode.create(basePtr, loc, StampFactory.forKind(Kind.Long))); - } - - @Override - public void lower(LoweringTool tool) { - if (graph().getGuardsStage() == StructuredGraph.GuardsStage.AFTER_FSA) { - ForeignCallDescriptor desc = HotSpotHostForeignCallsProvider.lookupCheckcastArraycopyDescriptor(isUninit()); - StructuredGraph graph = graph(); - ValueNode srcAddr = computeBase(getSource(), getSourcePosition()); - ValueNode destAddr = computeBase(getDestination(), getDestinationPosition()); - ValueNode len = getLength(); - if (len.stamp().getStackKind() != Kind.Long) { - len = IntegerConvertNode.convert(len, StampFactory.forKind(Kind.Long), graph()); - } - ForeignCallNode call = graph.add(ForeignCallNode.create(Graal.getRequiredCapability(RuntimeProvider.class).getHostBackend().getForeignCalls(), desc, srcAddr, destAddr, len, - superCheckOffset, destElemKlass)); - call.setStateAfter(stateAfter()); - graph.replaceFixedWithFixed(this, call); - } - } - - @Override - public LocationIdentity getLocationIdentity() { - return NamedLocationIdentity.getArrayLocation(Kind.Object); - } - - @NodeIntrinsic - public static native int checkcastArraycopy(Object src, int srcPos, Object dest, int destPos, int length, Word superCheckOffset, Object destElemKlass, @ConstantNodeParameter boolean uninit); -} diff -r be63951fa2bc -r 5afa718edb8b graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ObjectCloneSnippets.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ObjectCloneSnippets.java Tue Dec 02 20:48:12 2014 +0100 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ObjectCloneSnippets.java Wed Dec 03 10:35:42 2014 +0200 @@ -27,6 +27,7 @@ import com.oracle.graal.api.meta.*; import com.oracle.graal.compiler.common.*; +import com.oracle.graal.hotspot.replacements.arraycopy.*; import com.oracle.graal.nodes.*; import com.oracle.graal.nodes.java.*; import com.oracle.graal.replacements.*; diff -r be63951fa2bc -r 5afa718edb8b graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/SystemSubstitutions.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/SystemSubstitutions.java Tue Dec 02 20:48:12 2014 +0100 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/SystemSubstitutions.java Wed Dec 03 10:35:42 2014 +0200 @@ -29,6 +29,7 @@ import com.oracle.graal.api.replacements.*; import com.oracle.graal.graph.Node.ConstantNodeParameter; import com.oracle.graal.graph.Node.NodeIntrinsic; +import com.oracle.graal.hotspot.replacements.arraycopy.*; import com.oracle.graal.nodes.extended.*; import com.oracle.graal.nodes.spi.*; diff -r be63951fa2bc -r 5afa718edb8b graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/UnsafeArrayCopyNode.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/UnsafeArrayCopyNode.java Tue Dec 02 20:48:12 2014 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,140 +0,0 @@ -/* - * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package com.oracle.graal.hotspot.replacements; - -import static com.oracle.graal.api.meta.LocationIdentity.*; - -import com.oracle.graal.api.meta.*; -import com.oracle.graal.compiler.common.type.*; -import com.oracle.graal.nodeinfo.*; -import com.oracle.graal.nodes.*; -import com.oracle.graal.nodes.extended.*; -import com.oracle.graal.nodes.spi.*; -import com.oracle.graal.replacements.SnippetTemplate.Arguments; - -@NodeInfo(allowedUsageTypes = {InputType.Memory}) -public class UnsafeArrayCopyNode extends ArrayRangeWriteNode implements Lowerable, MemoryCheckpoint.Single { - - @Input ValueNode src; - @Input ValueNode srcPos; - @Input ValueNode dest; - @Input ValueNode destPos; - @Input ValueNode length; - @OptionalInput ValueNode layoutHelper; - - protected Kind elementKind; - - public static UnsafeArrayCopyNode create(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, ValueNode layoutHelper, Kind elementKind) { - return new UnsafeArrayCopyNode(src, srcPos, dest, destPos, length, layoutHelper, elementKind); - } - - protected UnsafeArrayCopyNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, ValueNode layoutHelper, Kind elementKind) { - super(StampFactory.forVoid()); - assert layoutHelper == null || elementKind == null; - this.src = src; - this.srcPos = srcPos; - this.dest = dest; - this.destPos = destPos; - this.length = length; - this.layoutHelper = layoutHelper; - this.elementKind = elementKind; - } - - public static UnsafeArrayCopyNode create(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, Kind elementKind) { - return new UnsafeArrayCopyNode(src, srcPos, dest, destPos, length, elementKind); - } - - protected UnsafeArrayCopyNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, Kind elementKind) { - this(src, srcPos, dest, destPos, length, null, elementKind); - } - - public static UnsafeArrayCopyNode create(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, ValueNode layoutHelper) { - return new UnsafeArrayCopyNode(src, srcPos, dest, destPos, length, layoutHelper); - } - - protected UnsafeArrayCopyNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, ValueNode layoutHelper) { - this(src, srcPos, dest, destPos, length, layoutHelper, null); - } - - @Override - public ValueNode getArray() { - return dest; - } - - @Override - public ValueNode getIndex() { - return destPos; - } - - @Override - public ValueNode getLength() { - return length; - } - - @Override - public boolean isObjectArray() { - return elementKind == Kind.Object; - } - - @Override - public boolean isInitialization() { - return false; - } - - public Kind getElementKind() { - return elementKind; - } - - @Override - public void lower(LoweringTool tool) { - if (graph().getGuardsStage() == StructuredGraph.GuardsStage.AFTER_FSA) { - UnsafeArrayCopySnippets.Templates templates = tool.getReplacements().getSnippetTemplateCache(UnsafeArrayCopySnippets.Templates.class); - templates.lower(this, tool); - } - } - - public void addSnippetArguments(Arguments args) { - args.add("src", src); - args.add("srcPos", srcPos); - args.add("dest", dest); - args.add("destPos", destPos); - args.add("length", length); - if (layoutHelper != null) { - args.add("layoutHelper", layoutHelper); - } - } - - @Override - public LocationIdentity getLocationIdentity() { - if (elementKind != null) { - return NamedLocationIdentity.getArrayLocation(elementKind); - } - return ANY_LOCATION; - } - - @NodeIntrinsic - public static native void arraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter Kind elementKind); - - @NodeIntrinsic - public static native void arraycopyPrimitive(Object src, int srcPos, Object dest, int destPos, int length, int layoutHelper); -} diff -r be63951fa2bc -r 5afa718edb8b graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/UnsafeArrayCopySnippets.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/UnsafeArrayCopySnippets.java Tue Dec 02 20:48:12 2014 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,310 +0,0 @@ -/* - * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package com.oracle.graal.hotspot.replacements; - -import static com.oracle.graal.api.meta.LocationIdentity.*; -import static com.oracle.graal.hotspot.HotSpotGraalRuntime.*; -import static com.oracle.graal.hotspot.replacements.HotSpotReplacementsUtil.*; -import static com.oracle.graal.nodes.extended.BranchProbabilityNode.*; -import static com.oracle.graal.replacements.SnippetTemplate.*; - -import com.oracle.graal.api.code.*; -import com.oracle.graal.api.meta.*; -import com.oracle.graal.api.replacements.*; -import com.oracle.graal.asm.*; -import com.oracle.graal.hotspot.meta.*; -import com.oracle.graal.hotspot.phases.*; -import com.oracle.graal.nodes.extended.*; -import com.oracle.graal.nodes.spi.*; -import com.oracle.graal.replacements.*; -import com.oracle.graal.replacements.SnippetTemplate.AbstractTemplates; -import com.oracle.graal.replacements.SnippetTemplate.Arguments; -import com.oracle.graal.replacements.SnippetTemplate.SnippetInfo; -import com.oracle.graal.replacements.nodes.*; -import com.oracle.graal.word.*; - -/** - * As opposed to {@link ArrayCopySnippets}, these Snippets do not perform store checks. - */ -public class UnsafeArrayCopySnippets implements Snippets { - private static final boolean supportsUnalignedMemoryAccess = runtime().getTarget().arch.supportsUnalignedMemoryAccess(); - - private static final Kind VECTOR_KIND = Kind.Long; - private static final long VECTOR_SIZE = arrayIndexScale(VECTOR_KIND); - - private static void vectorizedCopy(Object src, int srcPos, Object dest, int destPos, int length, Kind baseKind, LocationIdentity locationIdentity) { - int arrayBaseOffset = arrayBaseOffset(baseKind); - int elementSize = arrayIndexScale(baseKind); - long byteLength = (long) length * elementSize; - long srcOffset = (long) srcPos * elementSize; - long destOffset = (long) destPos * elementSize; - - long preLoopBytes; - long mainLoopBytes; - long postLoopBytes; - - // We can easily vectorize the loop if both offsets have the same alignment. - if (byteLength >= VECTOR_SIZE && (srcOffset % VECTOR_SIZE) == (destOffset % VECTOR_SIZE)) { - preLoopBytes = NumUtil.roundUp(arrayBaseOffset + srcOffset, VECTOR_SIZE) - (arrayBaseOffset + srcOffset); - postLoopBytes = (byteLength - preLoopBytes) % VECTOR_SIZE; - mainLoopBytes = byteLength - preLoopBytes - postLoopBytes; - } else { - // Does the architecture support unaligned memory accesses? - if (supportsUnalignedMemoryAccess) { - preLoopBytes = byteLength % VECTOR_SIZE; - mainLoopBytes = byteLength - preLoopBytes; - postLoopBytes = 0; - } else { - // No. Let's do element-wise copying. - preLoopBytes = byteLength; - mainLoopBytes = 0; - postLoopBytes = 0; - } - } - - if (probability(NOT_FREQUENT_PROBABILITY, src == dest) && probability(NOT_FREQUENT_PROBABILITY, srcPos < destPos)) { - // bad aliased case - srcOffset += byteLength; - destOffset += byteLength; - - // Post-loop - for (long i = 0; i < postLoopBytes; i += elementSize) { - srcOffset -= elementSize; - destOffset -= elementSize; - Object a = UnsafeLoadNode.load(src, arrayBaseOffset + srcOffset, baseKind, locationIdentity); - UnsafeStoreNode.store(dest, arrayBaseOffset + destOffset, a, baseKind, locationIdentity); - } - // Main-loop - for (long i = 0; i < mainLoopBytes; i += VECTOR_SIZE) { - srcOffset -= VECTOR_SIZE; - destOffset -= VECTOR_SIZE; - Long a = UnsafeLoadNode.load(src, arrayBaseOffset + srcOffset, VECTOR_KIND, locationIdentity); - UnsafeStoreNode.store(dest, arrayBaseOffset + destOffset, a, VECTOR_KIND, locationIdentity); - } - // Pre-loop - for (long i = 0; i < preLoopBytes; i += elementSize) { - srcOffset -= elementSize; - destOffset -= elementSize; - Object a = UnsafeLoadNode.load(src, arrayBaseOffset + srcOffset, baseKind, locationIdentity); - UnsafeStoreNode.store(dest, arrayBaseOffset + destOffset, a, baseKind, locationIdentity); - } - } else { - // Pre-loop - for (long i = 0; i < preLoopBytes; i += elementSize) { - Object a = UnsafeLoadNode.load(src, arrayBaseOffset + srcOffset, baseKind, locationIdentity); - UnsafeStoreNode.store(dest, arrayBaseOffset + destOffset, a, baseKind, locationIdentity); - srcOffset += elementSize; - destOffset += elementSize; - } - // Main-loop - for (long i = 0; i < mainLoopBytes; i += VECTOR_SIZE) { - Long a = UnsafeLoadNode.load(src, arrayBaseOffset + srcOffset, VECTOR_KIND, locationIdentity); - UnsafeStoreNode.store(dest, arrayBaseOffset + destOffset, a, VECTOR_KIND, locationIdentity); - srcOffset += VECTOR_SIZE; - destOffset += VECTOR_SIZE; - } - // Post-loop - for (long i = 0; i < postLoopBytes; i += elementSize) { - Object a = UnsafeLoadNode.load(src, arrayBaseOffset + srcOffset, baseKind, locationIdentity); - UnsafeStoreNode.store(dest, arrayBaseOffset + destOffset, a, baseKind, locationIdentity); - srcOffset += elementSize; - destOffset += elementSize; - } - } - } - - @Fold - private static LocationIdentity getArrayLocation(Kind kind) { - return NamedLocationIdentity.getArrayLocation(kind); - } - - @Snippet - public static void arraycopyByte(byte[] src, int srcPos, byte[] dest, int destPos, int length) { - Kind kind = Kind.Byte; - vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); - } - - @Snippet - public static void arraycopyBoolean(boolean[] src, int srcPos, boolean[] dest, int destPos, int length) { - Kind kind = Kind.Boolean; - vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); - } - - @Snippet - public static void arraycopyChar(char[] src, int srcPos, char[] dest, int destPos, int length) { - Kind kind = Kind.Char; - vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); - } - - @Snippet - public static void arraycopyShort(short[] src, int srcPos, short[] dest, int destPos, int length) { - Kind kind = Kind.Short; - vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); - } - - @Snippet - public static void arraycopyInt(int[] src, int srcPos, int[] dest, int destPos, int length) { - Kind kind = Kind.Int; - vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); - } - - @Snippet - public static void arraycopyFloat(float[] src, int srcPos, float[] dest, int destPos, int length) { - Kind kind = Kind.Float; - vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); - } - - @Snippet - public static void arraycopyLong(long[] src, int srcPos, long[] dest, int destPos, int length) { - Kind kind = Kind.Long; - vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); - } - - @Snippet - public static void arraycopyDouble(double[] src, int srcPos, double[] dest, int destPos, int length) { - Kind kind = Kind.Double; - /* - * TODO atomicity problem on 32-bit architectures: The JVM spec requires double values to be - * copied atomically, but not long values. For example, on Intel 32-bit this code is not - * atomic as long as the vector kind remains Kind.Long. - */ - vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); - } - - /** - * For this kind, Object, we want to avoid write barriers between writes, but instead have them - * at the end of the snippet. This is done by using {@link DirectObjectStoreNode}, and rely on - * {@link WriteBarrierAdditionPhase} to put write barriers after the {@link UnsafeArrayCopyNode} - * with kind Object. - */ - @Snippet - public static void arraycopyObject(Object[] src, int srcPos, Object[] dest, int destPos, int length) { - Kind kind = Kind.Object; - final int scale = arrayIndexScale(kind); - int arrayBaseOffset = arrayBaseOffset(kind); - LocationIdentity arrayLocation = getArrayLocation(kind); - if (src == dest && srcPos < destPos) { // bad aliased case - long start = (long) (length - 1) * scale; - for (long i = start; i >= 0; i -= scale) { - Object a = UnsafeLoadNode.load(src, arrayBaseOffset + i + (long) srcPos * scale, kind, arrayLocation); - DirectObjectStoreNode.storeObject(dest, arrayBaseOffset, i + (long) destPos * scale, a, getArrayLocation(kind)); - } - } else { - long end = (long) length * scale; - for (long i = 0; i < end; i += scale) { - Object a = UnsafeLoadNode.load(src, arrayBaseOffset + i + (long) srcPos * scale, kind, arrayLocation); - DirectObjectStoreNode.storeObject(dest, arrayBaseOffset, i + (long) destPos * scale, a, getArrayLocation(kind)); - } - } - } - - @Snippet - public static void arraycopyPrimitive(Object src, int srcPos, Object dest, int destPos, int length, int layoutHelper) { - int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift()) & layoutHelperLog2ElementSizeMask(); - int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift()) & layoutHelperHeaderSizeMask(); - - Unsigned vectorSize = Word.unsigned(VECTOR_SIZE); - Unsigned srcOffset = Word.unsigned(srcPos).shiftLeft(log2ElementSize).add(headerSize); - Unsigned destOffset = Word.unsigned(destPos).shiftLeft(log2ElementSize).add(headerSize); - Unsigned destStart = destOffset; - Unsigned destEnd = destOffset.add(Word.unsigned(length).shiftLeft(log2ElementSize)); - - Unsigned destVectorEnd = null; - Unsigned nonVectorBytes = null; - Unsigned sizeInBytes = Word.unsigned(length).shiftLeft(log2ElementSize); - if (supportsUnalignedMemoryAccess) { - nonVectorBytes = sizeInBytes.unsignedRemainder(vectorSize); - destVectorEnd = destEnd; - } else { - boolean inPhase = srcOffset.and((int) VECTOR_SIZE - 1).equal(destOffset.and((int) VECTOR_SIZE - 1)); - boolean hasAtLeastOneVector = sizeInBytes.aboveOrEqual(vectorSize); - // We must have at least one full vector, otherwise we must copy each byte separately - if (hasAtLeastOneVector && inPhase) { // If in phase, we can vectorize - nonVectorBytes = vectorSize.subtract(destStart.unsignedRemainder(vectorSize)); - } else { // fallback is byte-wise - nonVectorBytes = sizeInBytes; - } - destVectorEnd = destEnd.subtract(destEnd.unsignedRemainder(vectorSize)); - } - - Unsigned destNonVectorEnd = destStart.add(nonVectorBytes); - while (destOffset.belowThan(destNonVectorEnd)) { - ObjectAccess.writeByte(dest, destOffset, ObjectAccess.readByte(src, srcOffset, ANY_LOCATION), ANY_LOCATION); - destOffset = destOffset.add(1); - srcOffset = srcOffset.add(1); - } - // Unsigned destVectorEnd = destEnd.subtract(destEnd.unsignedRemainder(8)); - while (destOffset.belowThan(destVectorEnd)) { - ObjectAccess.writeWord(dest, destOffset, ObjectAccess.readWord(src, srcOffset, ANY_LOCATION), ANY_LOCATION); - destOffset = destOffset.add(wordSize()); - srcOffset = srcOffset.add(wordSize()); - } - // Do the last bytes each when it is required to have absolute alignment. - while (!supportsUnalignedMemoryAccess && destOffset.belowThan(destEnd)) { - ObjectAccess.writeByte(dest, destOffset, ObjectAccess.readByte(src, srcOffset, ANY_LOCATION), ANY_LOCATION); - destOffset = destOffset.add(1); - srcOffset = srcOffset.add(1); - } - } - - public static class Templates extends AbstractTemplates { - - private final SnippetInfo[] arraycopySnippets; - private final SnippetInfo genericPrimitiveSnippet; - - public Templates(HotSpotProviders providers, TargetDescription target) { - super(providers, providers.getSnippetReflection(), target); - - arraycopySnippets = new SnippetInfo[Kind.values().length]; - arraycopySnippets[Kind.Boolean.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyBoolean"); - arraycopySnippets[Kind.Byte.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyByte"); - arraycopySnippets[Kind.Short.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyShort"); - arraycopySnippets[Kind.Char.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyChar"); - arraycopySnippets[Kind.Int.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyInt"); - arraycopySnippets[Kind.Long.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyLong"); - arraycopySnippets[Kind.Float.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyFloat"); - arraycopySnippets[Kind.Double.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyDouble"); - arraycopySnippets[Kind.Object.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyObject"); - - genericPrimitiveSnippet = snippet(UnsafeArrayCopySnippets.class, "arraycopyPrimitive"); - } - - public void lower(UnsafeArrayCopyNode node, LoweringTool tool) { - Kind elementKind = node.getElementKind(); - SnippetInfo snippet; - if (elementKind == null) { - // primitive array of unknown kind - snippet = genericPrimitiveSnippet; - } else { - snippet = arraycopySnippets[elementKind.ordinal()]; - assert snippet != null : "arraycopy snippet for " + elementKind.name() + " not found"; - } - - Arguments args = new Arguments(snippet, node.graph().getGuardsStage(), tool.getLoweringStage()); - node.addSnippetArguments(args); - - SnippetTemplate template = template(args); - template.instantiate(providers.getMetaAccess(), node, DEFAULT_REPLACER, args); - } - } -} diff -r be63951fa2bc -r 5afa718edb8b graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/ArrayCopyCallNode.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/ArrayCopyCallNode.java Wed Dec 03 10:35:42 2014 +0200 @@ -0,0 +1,195 @@ +/* + * Copyright (c) 2014, 2014, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +//JaCoCo Exclude +package com.oracle.graal.hotspot.replacements.arraycopy; + +import static com.oracle.graal.api.meta.LocationIdentity.*; + +import com.oracle.graal.api.meta.*; +import com.oracle.graal.api.runtime.*; +import com.oracle.graal.compiler.common.type.*; +import com.oracle.graal.hotspot.*; +import com.oracle.graal.hotspot.meta.*; +import com.oracle.graal.hotspot.nodes.*; +import com.oracle.graal.nodeinfo.*; +import com.oracle.graal.nodes.*; +import com.oracle.graal.nodes.calc.*; +import com.oracle.graal.nodes.extended.*; +import com.oracle.graal.nodes.spi.*; +import com.oracle.graal.runtime.*; + +@NodeInfo(allowedUsageTypes = {InputType.Memory}) +public class ArrayCopyCallNode extends AbstractMemoryCheckpoint implements Lowerable, MemoryCheckpoint.Single { + + @Input ValueNode src; + @Input ValueNode srcPos; + @Input ValueNode dest; + @Input ValueNode destPos; + @Input ValueNode length; + + protected Kind elementKind; + + /** + * Aligned means that the offset of the copy is heap word aligned. + */ + protected boolean aligned; + protected boolean disjoint; + protected boolean uninitialized; + + protected final HotSpotGraalRuntimeProvider runtime; + + public static ArrayCopyCallNode create(@InjectedNodeParameter HotSpotGraalRuntimeProvider runtime, ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, + Kind elementKind, boolean aligned, boolean disjoint, boolean uninitialized) { + return new ArrayCopyCallNode(src, srcPos, dest, destPos, length, elementKind, aligned, disjoint, uninitialized, runtime); + } + + protected ArrayCopyCallNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, Kind elementKind, boolean aligned, boolean disjoint, boolean uninitialized, + HotSpotGraalRuntimeProvider runtime) { + super(StampFactory.forVoid()); + assert elementKind != null; + this.src = src; + this.srcPos = srcPos; + this.dest = dest; + this.destPos = destPos; + this.length = length; + this.elementKind = elementKind; + this.aligned = aligned; + this.disjoint = disjoint; + this.uninitialized = uninitialized; + this.runtime = runtime; + } + + public static ArrayCopyCallNode create(@InjectedNodeParameter HotSpotGraalRuntimeProvider runtime, ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, + Kind elementKind, boolean disjoint) { + return new ArrayCopyCallNode(src, srcPos, dest, destPos, length, elementKind, false, disjoint, false, runtime); + } + + public ValueNode getSource() { + return src; + } + + public ValueNode getSourcePosition() { + return srcPos; + } + + public ValueNode getDestination() { + return dest; + } + + public ValueNode getDestinationPosition() { + return destPos; + } + + public ValueNode getLength() { + return length; + } + + public Kind getElementKind() { + return elementKind; + } + + private ValueNode computeBase(ValueNode base, ValueNode pos) { + FixedWithNextNode basePtr = graph().add(GetObjectAddressNode.create(base)); + graph().addBeforeFixed(this, basePtr); + ValueNode loc = IndexedLocationNode.create(getLocationIdentity(), elementKind, runtime.getArrayBaseOffset(elementKind), pos, graph(), runtime.getArrayIndexScale(elementKind)); + return graph().unique(ComputeAddressNode.create(basePtr, loc, StampFactory.forKind(Kind.Long))); + } + + @Override + public void lower(LoweringTool tool) { + if (graph().getGuardsStage() == StructuredGraph.GuardsStage.AFTER_FSA) { + updateAlignedDisjoint(); + ForeignCallDescriptor desc = HotSpotHostForeignCallsProvider.lookupArraycopyDescriptor(elementKind, isAligned(), isDisjoint(), isUninitialized()); + StructuredGraph graph = graph(); + ValueNode srcAddr = computeBase(getSource(), getSourcePosition()); + ValueNode destAddr = computeBase(getDestination(), getDestinationPosition()); + ValueNode len = getLength(); + if (len.stamp().getStackKind() != Kind.Long) { + len = IntegerConvertNode.convert(len, StampFactory.forKind(Kind.Long), graph()); + } + ForeignCallNode call = graph.add(ForeignCallNode.create(Graal.getRequiredCapability(RuntimeProvider.class).getHostBackend().getForeignCalls(), desc, srcAddr, destAddr, len)); + call.setStateAfter(stateAfter()); + graph.replaceFixedWithFixed(this, call); + + } + } + + @Override + public LocationIdentity getLocationIdentity() { + if (elementKind != null) { + return NamedLocationIdentity.getArrayLocation(elementKind); + } + return ANY_LOCATION; + } + + @NodeIntrinsic + private static native void arraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter Kind elementKind, @ConstantNodeParameter boolean aligned, + @ConstantNodeParameter boolean disjoint, @ConstantNodeParameter boolean uninitialized); + + public static void arraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter Kind elementKind, boolean aligned, boolean disjoint) { + arraycopy(src, srcPos, dest, destPos, length, elementKind, aligned, disjoint, false); + } + + public static void disjointArraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter Kind elementKind) { + arraycopy(src, srcPos, dest, destPos, length, elementKind, false, true, false); + } + + public static void disjointUninitializedArraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter Kind elementKind) { + arraycopy(src, srcPos, dest, destPos, length, elementKind, false, true, true); + } + + public boolean isAligned() { + return aligned; + } + + public boolean isDisjoint() { + return disjoint; + } + + public boolean isUninitialized() { + return uninitialized; + } + + boolean isHeapWordAligned(JavaConstant value, Kind kind) { + return (runtime.getArrayBaseOffset(kind) + (long) value.asInt() * runtime.getArrayIndexScale(kind)) % runtime.getConfig().heapWordSize == 0; + } + + public void updateAlignedDisjoint() { + Kind componentKind = elementKind; + if (srcPos == destPos) { + // Can treat as disjoint + disjoint = true; + } + PrimitiveConstant constantSrc = (PrimitiveConstant) srcPos.stamp().asConstant(); + PrimitiveConstant constantDst = (PrimitiveConstant) destPos.stamp().asConstant(); + if (constantSrc != null && constantDst != null) { + if (!aligned) { + aligned = isHeapWordAligned(constantSrc, componentKind) && isHeapWordAligned(constantDst, componentKind); + } + if (constantSrc.asInt() >= constantDst.asInt()) { + // low to high copy so treat as disjoint + disjoint = true; + } + } + } +} diff -r be63951fa2bc -r 5afa718edb8b graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/ArrayCopyNode.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/ArrayCopyNode.java Wed Dec 03 10:35:42 2014 +0200 @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.graal.hotspot.replacements.arraycopy; + +import static com.oracle.graal.compiler.GraalCompiler.*; + +import com.oracle.graal.api.meta.*; +import com.oracle.graal.compiler.common.*; +import com.oracle.graal.debug.*; +import com.oracle.graal.debug.Debug.Scope; +import com.oracle.graal.loop.phases.*; +import com.oracle.graal.nodeinfo.*; +import com.oracle.graal.nodes.*; +import com.oracle.graal.nodes.spi.*; +import com.oracle.graal.nodes.type.*; +import com.oracle.graal.phases.common.*; +import com.oracle.graal.phases.tiers.*; +import com.oracle.graal.replacements.nodes.*; + +@NodeInfo +public class ArrayCopyNode extends BasicArrayCopyNode implements Virtualizable, Lowerable { + + public static ArrayCopyNode create(Invoke invoke) { + return new ArrayCopyNode(invoke); + } + + protected ArrayCopyNode(Invoke invoke) { + super(invoke); + } + + private StructuredGraph selectSnippet(LoweringTool tool, final Replacements replacements) { + ResolvedJavaType srcType = StampTool.typeOrNull(getSource().stamp()); + ResolvedJavaType destType = StampTool.typeOrNull(getDestination().stamp()); + + if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) { + return null; + } + if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) { + return null; + } + if (!isExact()) { + return null; + } + Kind componentKind = srcType.getComponentType().getKind(); + final ResolvedJavaMethod snippetMethod = tool.getMetaAccess().lookupJavaMethod(ArrayCopySnippets.getSnippetForKind(componentKind, shouldUnroll(), isExact())); + try (Scope s = Debug.scope("ArrayCopySnippet", snippetMethod)) { + return replacements.getSnippet(snippetMethod); + } catch (Throwable e) { + throw Debug.handle(e); + } + } + + private static void unrollFixedLengthLoop(StructuredGraph snippetGraph, int length, LoweringTool tool) { + ParameterNode lengthParam = snippetGraph.getParameter(4); + if (lengthParam != null) { + snippetGraph.replaceFloating(lengthParam, ConstantNode.forInt(length, snippetGraph)); + } + // the canonicalization before loop unrolling is needed to propagate the length into + // additions, etc. + PhaseContext context = new PhaseContext(tool.getMetaAccess(), tool.getConstantReflection(), tool.getLowerer(), tool.getReplacements(), tool.assumptions(), tool.getStampProvider()); + new CanonicalizerPhase(true).apply(snippetGraph, context); + new LoopFullUnrollPhase(new CanonicalizerPhase(true)).apply(snippetGraph, context); + new CanonicalizerPhase(true).apply(snippetGraph, context); + } + + @Override + protected StructuredGraph getLoweredSnippetGraph(final LoweringTool tool) { + if (!shouldIntrinsify(getTargetMethod())) { + return null; + } + + final Replacements replacements = tool.getReplacements(); + StructuredGraph snippetGraph = selectSnippet(tool, replacements); + if (snippetGraph == null) { + ResolvedJavaType srcType = StampTool.typeOrNull(getSource().stamp()); + ResolvedJavaType destType = StampTool.typeOrNull(getDestination().stamp()); + ResolvedJavaType srcComponentType = srcType == null ? null : srcType.getComponentType(); + ResolvedJavaType destComponentType = destType == null ? null : destType.getComponentType(); + ResolvedJavaMethod snippetMethod = null; + if (srcComponentType != null && destComponentType != null && srcComponentType.getKind().equals(Kind.Object) && destComponentType.getKind().equals(Kind.Object)) { + snippetMethod = tool.getMetaAccess().lookupJavaMethod(ArrayCopySnippets.checkcastArraycopySnippet); + } else { + snippetMethod = tool.getMetaAccess().lookupJavaMethod(ArrayCopySnippets.genericArraycopySnippet); + } + snippetGraph = null; + try (Scope s = Debug.scope("ArrayCopySnippet", snippetMethod)) { + snippetGraph = replacements.getSnippet(snippetMethod, getTargetMethod()).copy(); + } catch (Throwable e) { + throw Debug.handle(e); + } + replaceSnippetInvokes(snippetGraph); + } else { + assert snippetGraph != null : "ArrayCopySnippets should be installed"; + snippetGraph = snippetGraph.copy(); + if (shouldUnroll()) { + final StructuredGraph copy = snippetGraph; + try (Scope s = Debug.scope("ArrayCopySnippetSpecialization", snippetGraph.method())) { + unrollFixedLengthLoop(copy, getLength().asJavaConstant().asInt(), tool); + } catch (Throwable e) { + throw Debug.handle(e); + } + } + } + return lowerReplacement(snippetGraph, tool); + } + + private boolean shouldUnroll() { + return getLength().isConstant() && getLength().asJavaConstant().asInt() <= GraalOptions.MaximumEscapeAnalysisArrayLength.getValue(); + } + + /* + * Returns true if this copy doesn't require store checks. Trivially true for primitive arrays. + */ + private boolean isExact() { + ResolvedJavaType srcType = StampTool.typeOrNull(getSource().stamp()); + if (srcType.getComponentType().getKind().isPrimitive() || getSource() == getDestination()) { + return true; + } + + ResolvedJavaType destType = StampTool.typeOrNull(getDestination().stamp()); + if (StampTool.isExactType(getDestination().stamp())) { + if (destType != null && destType.isAssignableFrom(srcType)) { + return true; + } + } + return false; + } +} diff -r be63951fa2bc -r 5afa718edb8b graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/ArrayCopySnippets.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/ArrayCopySnippets.java Wed Dec 03 10:35:42 2014 +0200 @@ -0,0 +1,318 @@ +/* + * Copyright (c) 2011, 2014, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.graal.hotspot.replacements.arraycopy; + +import static com.oracle.graal.compiler.common.GraalOptions.*; +import static com.oracle.graal.hotspot.replacements.HotSpotReplacementsUtil.*; +import static com.oracle.graal.nodes.GuardingPiNode.*; +import static com.oracle.graal.nodes.extended.BranchProbabilityNode.*; + +import java.lang.reflect.*; +import java.util.*; + +import com.oracle.graal.api.meta.*; +import com.oracle.graal.compiler.common.*; +import com.oracle.graal.graph.Node.ConstantNodeParameter; +import com.oracle.graal.graph.Node.NodeIntrinsic; +import com.oracle.graal.hotspot.word.*; +import com.oracle.graal.nodes.*; +import com.oracle.graal.nodes.extended.*; +import com.oracle.graal.nodes.java.*; +import com.oracle.graal.replacements.*; +import com.oracle.graal.word.*; + +public class ArrayCopySnippets implements Snippets { + + private static final EnumMap arraycopyMethods = new EnumMap<>(Kind.class); + private static final EnumMap arraycopyCalls = new EnumMap<>(Kind.class); + + public static final Method checkcastArraycopySnippet; + public static final Method genericArraycopySnippet; + + private static void addArraycopySnippetMethod(Kind kind, Class arrayClass) throws NoSuchMethodException { + arraycopyMethods.put(kind, ArrayCopySnippets.class.getDeclaredMethod("arraycopy", arrayClass, int.class, arrayClass, int.class, int.class)); + if (CallArrayCopy.getValue()) { + if (kind == Kind.Object) { + arraycopyCalls.put(kind, ArrayCopySnippets.class.getDeclaredMethod("objectArraycopyUnchecked", arrayClass, int.class, arrayClass, int.class, int.class)); + } else { + arraycopyCalls.put(kind, ArrayCopySnippets.class.getDeclaredMethod(kind + "Arraycopy", arrayClass, int.class, arrayClass, int.class, int.class)); + } + } + } + + static { + try { + addArraycopySnippetMethod(Kind.Byte, byte[].class); + addArraycopySnippetMethod(Kind.Boolean, boolean[].class); + addArraycopySnippetMethod(Kind.Char, char[].class); + addArraycopySnippetMethod(Kind.Short, short[].class); + addArraycopySnippetMethod(Kind.Int, int[].class); + addArraycopySnippetMethod(Kind.Long, long[].class); + addArraycopySnippetMethod(Kind.Float, float[].class); + addArraycopySnippetMethod(Kind.Double, double[].class); + addArraycopySnippetMethod(Kind.Object, Object[].class); + checkcastArraycopySnippet = ArrayCopySnippets.class.getDeclaredMethod("checkcastArraycopy", Object[].class, int.class, Object[].class, int.class, int.class); + genericArraycopySnippet = ArrayCopySnippets.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class); + } catch (SecurityException | NoSuchMethodException e) { + throw new GraalInternalError(e); + } + } + + public static Method getSnippetForKind(Kind kind, boolean shouldUnroll, boolean exact) { + Method m = null; + if (!shouldUnroll && exact) { + // use hotspot stubs + m = arraycopyCalls.get(kind); + if (m != null) { + return m; + } + } + // use snippets + return arraycopyMethods.get(kind); + } + + private static void checkedCopy(Object src, int srcPos, Object dest, int destPos, int length, Kind baseKind) { + Object nonNullSrc = guardingNonNull(src); + Object nonNullDest = guardingNonNull(dest); + checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); + UnsafeArrayCopyNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, baseKind); + } + + private static int checkArrayType(KlassPointer hub) { + int layoutHelper = readLayoutHelper(hub); + if (probability(SLOW_PATH_PROBABILITY, layoutHelper >= 0)) { + DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); + } + return layoutHelper; + } + + private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length) { + if (probability(SLOW_PATH_PROBABILITY, srcPos < 0)) { + checkAIOOBECounter.inc(); + DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); + } + if (probability(SLOW_PATH_PROBABILITY, destPos < 0)) { + checkAIOOBECounter.inc(); + DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); + } + if (probability(SLOW_PATH_PROBABILITY, length < 0)) { + checkAIOOBECounter.inc(); + DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); + } + if (probability(SLOW_PATH_PROBABILITY, srcPos + length > ArrayLengthNode.arrayLength(src))) { + checkAIOOBECounter.inc(); + DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); + } + if (probability(SLOW_PATH_PROBABILITY, destPos + length > ArrayLengthNode.arrayLength(dest))) { + checkAIOOBECounter.inc(); + DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); + } + checkSuccessCounter.inc(); + } + + @Snippet + public static void arraycopy(byte[] src, int srcPos, byte[] dest, int destPos, int length) { + byteCounter.inc(); + checkedCopy(src, srcPos, dest, destPos, length, Kind.Byte); + } + + @Snippet + public static void arraycopy(boolean[] src, int srcPos, boolean[] dest, int destPos, int length) { + booleanCounter.inc(); + checkedCopy(src, srcPos, dest, destPos, length, Kind.Boolean); + } + + @Snippet + public static void arraycopy(char[] src, int srcPos, char[] dest, int destPos, int length) { + charCounter.inc(); + checkedCopy(src, srcPos, dest, destPos, length, Kind.Char); + } + + @Snippet + public static void arraycopy(short[] src, int srcPos, short[] dest, int destPos, int length) { + shortCounter.inc(); + checkedCopy(src, srcPos, dest, destPos, length, Kind.Short); + } + + @Snippet + public static void arraycopy(int[] src, int srcPos, int[] dest, int destPos, int length) { + intCounter.inc(); + checkedCopy(src, srcPos, dest, destPos, length, Kind.Int); + } + + @Snippet + public static void arraycopy(float[] src, int srcPos, float[] dest, int destPos, int length) { + floatCounter.inc(); + checkedCopy(src, srcPos, dest, destPos, length, Kind.Float); + } + + @Snippet + public static void arraycopy(long[] src, int srcPos, long[] dest, int destPos, int length) { + longCounter.inc(); + checkedCopy(src, srcPos, dest, destPos, length, Kind.Long); + } + + @Snippet + public static void arraycopy(double[] src, int srcPos, double[] dest, int destPos, int length) { + doubleCounter.inc(); + checkedCopy(src, srcPos, dest, destPos, length, Kind.Double); + } + + @Snippet + public static void arraycopy(Object[] src, int srcPos, Object[] dest, int destPos, int length) { + objectCounter.inc(); + checkedCopy(src, srcPos, dest, destPos, length, Kind.Object); + } + + @Snippet + public static void checkcastArraycopy(Object[] src, int srcPos, Object[] dest, int destPos, int length) { + objectCheckcastCounter.inc(); + Object nonNullSrc = guardingNonNull(src); + Object nonNullDest = guardingNonNull(dest); + checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); + KlassPointer destElemKlass = loadHub(nonNullDest); + checkcastArraycopyHelper(srcPos, destPos, length, nonNullSrc, nonNullDest, destElemKlass); + } + + private static void checkcastArraycopyHelper(int srcPos, int destPos, int length, Object nonNullSrc, Object nonNullDest, KlassPointer destElemKlass) { + Word superCheckOffset = Word.signed(destElemKlass.readInt(superCheckOffsetOffset(), KLASS_SUPER_CHECK_OFFSET_LOCATION)); + int copiedElements = CheckcastArrayCopyCallNode.checkcastArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, superCheckOffset, destElemKlass, false); + if (copiedElements != 0) { + // the checkcast stub doesn't throw the ArrayStoreException, but returns the number of + // copied elements (xor'd with -1). + copiedElements ^= -1; + System.arraycopy(nonNullSrc, srcPos + copiedElements, nonNullDest, destPos + copiedElements, length - copiedElements); + } + } + + @Snippet + public static void arraycopy(Object src, int srcPos, Object dest, int destPos, int length) { + Object nonNullSrc = guardingNonNull(src); + Object nonNullDest = guardingNonNull(dest); + KlassPointer srcHub = loadHub(nonNullSrc); + KlassPointer destHub = loadHub(nonNullDest); + if (probability(FAST_PATH_PROBABILITY, srcHub.equal(destHub)) && probability(FAST_PATH_PROBABILITY, nonNullSrc != nonNullDest)) { + int layoutHelper = checkArrayType(srcHub); + final boolean isObjectArray = ((layoutHelper & layoutHelperElementTypePrimitiveInPlace()) == 0); + checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); + if (probability(FAST_PATH_PROBABILITY, isObjectArray)) { + genericObjectExactCallCounter.inc(); + UnsafeArrayCopyNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, Kind.Object); + } else { + genericPrimitiveCallCounter.inc(); + UnsafeArrayCopyNode.arraycopyPrimitive(nonNullSrc, srcPos, nonNullDest, destPos, length, layoutHelper); + } + } else { + genericObjectCallCounter.inc(); + System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length); + } + } + + @NodeIntrinsic(ForeignCallNode.class) + public static native void callArraycopy(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word src, Word dest, Word len); + + private static void callArraycopyTemplate(SnippetCounter counter, Kind kind, boolean aligned, boolean disjoint, Object src, int srcPos, Object dest, int destPos, int length) { + counter.inc(); + Object nonNullSrc = guardingNonNull(src); + Object nonNullDest = guardingNonNull(dest); + checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); + ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, kind, aligned, disjoint); + } + + @Snippet + public static void objectArraycopyUnchecked(Object[] src, int srcPos, Object[] dest, int destPos, int length) { + callArraycopyTemplate(objectCallCounter, Kind.Object, false, false, src, srcPos, dest, destPos, length); + } + + @Snippet + public static void byteArraycopy(byte[] src, int srcPos, byte[] dest, int destPos, int length) { + callArraycopyTemplate(byteCallCounter, Kind.Byte, false, false, src, srcPos, dest, destPos, length); + } + + @Snippet + public static void booleanArraycopy(boolean[] src, int srcPos, boolean[] dest, int destPos, int length) { + callArraycopyTemplate(booleanCallCounter, Kind.Boolean, false, false, src, srcPos, dest, destPos, length); + } + + @Snippet + public static void charArraycopy(char[] src, int srcPos, char[] dest, int destPos, int length) { + callArraycopyTemplate(charCallCounter, Kind.Char, false, false, src, srcPos, dest, destPos, length); + } + + @Snippet + public static void shortArraycopy(short[] src, int srcPos, short[] dest, int destPos, int length) { + callArraycopyTemplate(shortCallCounter, Kind.Short, false, false, src, srcPos, dest, destPos, length); + } + + @Snippet + public static void intArraycopy(int[] src, int srcPos, int[] dest, int destPos, int length) { + callArraycopyTemplate(intCallCounter, Kind.Int, false, false, src, srcPos, dest, destPos, length); + } + + @Snippet + public static void floatArraycopy(float[] src, int srcPos, float[] dest, int destPos, int length) { + callArraycopyTemplate(floatCallCounter, Kind.Float, false, false, src, srcPos, dest, destPos, length); + } + + @Snippet + public static void longArraycopy(long[] src, int srcPos, long[] dest, int destPos, int length) { + callArraycopyTemplate(longCallCounter, Kind.Long, false, false, src, srcPos, dest, destPos, length); + } + + @Snippet + public static void doubleArraycopy(double[] src, int srcPos, double[] dest, int destPos, int length) { + callArraycopyTemplate(doubleCallCounter, Kind.Double, false, false, src, srcPos, dest, destPos, length); + } + + private static final SnippetCounter.Group checkCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy checkInputs") : null; + private static final SnippetCounter checkSuccessCounter = new SnippetCounter(checkCounters, "checkSuccess", "checkSuccess"); + private static final SnippetCounter checkNPECounter = new SnippetCounter(checkCounters, "checkNPE", "checkNPE"); + private static final SnippetCounter checkAIOOBECounter = new SnippetCounter(checkCounters, "checkAIOOBE", "checkAIOOBE"); + + private static final SnippetCounter.Group counters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy") : null; + private static final SnippetCounter byteCounter = new SnippetCounter(counters, "byte[]", "arraycopy for byte[] arrays"); + private static final SnippetCounter charCounter = new SnippetCounter(counters, "char[]", "arraycopy for char[] arrays"); + private static final SnippetCounter shortCounter = new SnippetCounter(counters, "short[]", "arraycopy for short[] arrays"); + private static final SnippetCounter intCounter = new SnippetCounter(counters, "int[]", "arraycopy for int[] arrays"); + private static final SnippetCounter booleanCounter = new SnippetCounter(counters, "boolean[]", "arraycopy for boolean[] arrays"); + private static final SnippetCounter longCounter = new SnippetCounter(counters, "long[]", "arraycopy for long[] arrays"); + private static final SnippetCounter objectCounter = new SnippetCounter(counters, "Object[]", "arraycopy for Object[] arrays"); + private static final SnippetCounter objectCheckcastCounter = new SnippetCounter(counters, "Object[]", "arraycopy for non-exact Object[] arrays"); + private static final SnippetCounter floatCounter = new SnippetCounter(counters, "float[]", "arraycopy for float[] arrays"); + private static final SnippetCounter doubleCounter = new SnippetCounter(counters, "double[]", "arraycopy for double[] arrays"); + + private static final SnippetCounter objectCallCounter = new SnippetCounter(counters, "Object[]", "arraycopy call for Object[] arrays"); + + private static final SnippetCounter booleanCallCounter = new SnippetCounter(counters, "boolean[]", "arraycopy call for boolean[] arrays"); + private static final SnippetCounter byteCallCounter = new SnippetCounter(counters, "byte[]", "arraycopy call for byte[] arrays"); + private static final SnippetCounter charCallCounter = new SnippetCounter(counters, "char[]", "arraycopy call for char[] arrays"); + private static final SnippetCounter doubleCallCounter = new SnippetCounter(counters, "double[]", "arraycopy call for double[] arrays"); + private static final SnippetCounter floatCallCounter = new SnippetCounter(counters, "float[]", "arraycopy call for float[] arrays"); + private static final SnippetCounter intCallCounter = new SnippetCounter(counters, "int[]", "arraycopy call for int[] arrays"); + private static final SnippetCounter longCallCounter = new SnippetCounter(counters, "long[]", "arraycopy call for long[] arrays"); + private static final SnippetCounter shortCallCounter = new SnippetCounter(counters, "short[]", "arraycopy call for short[] arrays"); + + private static final SnippetCounter genericPrimitiveCallCounter = new SnippetCounter(counters, "genericPrimitive", "generic arraycopy snippet for primitive arrays"); + private static final SnippetCounter genericObjectExactCallCounter = new SnippetCounter(counters, "genericObjectExact", "generic arraycopy snippet for special object arrays"); + private static final SnippetCounter genericObjectCallCounter = new SnippetCounter(counters, "genericObject", "call to the generic, native arraycopy method"); +} diff -r be63951fa2bc -r 5afa718edb8b graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/CheckcastArrayCopyCallNode.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/CheckcastArrayCopyCallNode.java Wed Dec 03 10:35:42 2014 +0200 @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2014, 2014, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +//JaCoCo Exclude +package com.oracle.graal.hotspot.replacements.arraycopy; + +import com.oracle.graal.api.meta.*; +import com.oracle.graal.api.runtime.*; +import com.oracle.graal.compiler.common.type.*; +import com.oracle.graal.hotspot.*; +import com.oracle.graal.hotspot.meta.*; +import com.oracle.graal.hotspot.nodes.*; +import com.oracle.graal.nodeinfo.*; +import com.oracle.graal.nodes.*; +import com.oracle.graal.nodes.calc.*; +import com.oracle.graal.nodes.extended.*; +import com.oracle.graal.nodes.spi.*; +import com.oracle.graal.runtime.*; +import com.oracle.graal.word.*; + +@NodeInfo(allowedUsageTypes = {InputType.Memory, InputType.Value}) +public class CheckcastArrayCopyCallNode extends AbstractMemoryCheckpoint implements Lowerable, MemoryCheckpoint.Single { + + @Input ValueNode src; + @Input ValueNode srcPos; + @Input ValueNode dest; + @Input ValueNode destPos; + @Input ValueNode length; + @Input ValueNode destElemKlass; + @Input ValueNode superCheckOffset; + + protected final boolean uninit; + + protected final HotSpotGraalRuntimeProvider runtime; + + public static CheckcastArrayCopyCallNode create(@InjectedNodeParameter HotSpotGraalRuntimeProvider runtime, ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, + ValueNode superCheckOffset, ValueNode destElemKlass, boolean uninit) { + return new CheckcastArrayCopyCallNode(src, srcPos, dest, destPos, length, superCheckOffset, destElemKlass, uninit, runtime); + } + + protected CheckcastArrayCopyCallNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, ValueNode superCheckOffset, ValueNode destElemKlass, boolean uninit, + HotSpotGraalRuntimeProvider runtime) { + super(StampFactory.forKind(Kind.Int)); + this.src = src; + this.srcPos = srcPos; + this.dest = dest; + this.destPos = destPos; + this.length = length; + this.superCheckOffset = superCheckOffset; + this.destElemKlass = destElemKlass; + this.uninit = uninit; + this.runtime = runtime; + } + + public ValueNode getSource() { + return src; + } + + public ValueNode getSourcePosition() { + return srcPos; + } + + public ValueNode getDestination() { + return dest; + } + + public ValueNode getDestinationPosition() { + return destPos; + } + + public ValueNode getLength() { + return length; + } + + public boolean isUninit() { + return uninit; + } + + private ValueNode computeBase(ValueNode base, ValueNode pos) { + FixedWithNextNode basePtr = graph().add(GetObjectAddressNode.create(base)); + graph().addBeforeFixed(this, basePtr); + ValueNode loc = IndexedLocationNode.create(getLocationIdentity(), Kind.Object, runtime.getArrayBaseOffset(Kind.Object), pos, graph(), runtime.getArrayIndexScale(Kind.Object)); + return graph().unique(ComputeAddressNode.create(basePtr, loc, StampFactory.forKind(Kind.Long))); + } + + @Override + public void lower(LoweringTool tool) { + if (graph().getGuardsStage() == StructuredGraph.GuardsStage.AFTER_FSA) { + ForeignCallDescriptor desc = HotSpotHostForeignCallsProvider.lookupCheckcastArraycopyDescriptor(isUninit()); + StructuredGraph graph = graph(); + ValueNode srcAddr = computeBase(getSource(), getSourcePosition()); + ValueNode destAddr = computeBase(getDestination(), getDestinationPosition()); + ValueNode len = getLength(); + if (len.stamp().getStackKind() != Kind.Long) { + len = IntegerConvertNode.convert(len, StampFactory.forKind(Kind.Long), graph()); + } + ForeignCallNode call = graph.add(ForeignCallNode.create(Graal.getRequiredCapability(RuntimeProvider.class).getHostBackend().getForeignCalls(), desc, srcAddr, destAddr, len, + superCheckOffset, destElemKlass)); + call.setStateAfter(stateAfter()); + graph.replaceFixedWithFixed(this, call); + } + } + + @Override + public LocationIdentity getLocationIdentity() { + return NamedLocationIdentity.getArrayLocation(Kind.Object); + } + + @NodeIntrinsic + public static native int checkcastArraycopy(Object src, int srcPos, Object dest, int destPos, int length, Word superCheckOffset, Object destElemKlass, @ConstantNodeParameter boolean uninit); +} diff -r be63951fa2bc -r 5afa718edb8b graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/UnsafeArrayCopyNode.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/UnsafeArrayCopyNode.java Wed Dec 03 10:35:42 2014 +0200 @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.graal.hotspot.replacements.arraycopy; + +import static com.oracle.graal.api.meta.LocationIdentity.*; + +import com.oracle.graal.api.meta.*; +import com.oracle.graal.compiler.common.type.*; +import com.oracle.graal.nodeinfo.*; +import com.oracle.graal.nodes.*; +import com.oracle.graal.nodes.extended.*; +import com.oracle.graal.nodes.spi.*; +import com.oracle.graal.replacements.SnippetTemplate.Arguments; + +@NodeInfo(allowedUsageTypes = {InputType.Memory}) +public class UnsafeArrayCopyNode extends ArrayRangeWriteNode implements Lowerable, MemoryCheckpoint.Single { + + @Input ValueNode src; + @Input ValueNode srcPos; + @Input ValueNode dest; + @Input ValueNode destPos; + @Input ValueNode length; + @OptionalInput ValueNode layoutHelper; + + protected Kind elementKind; + + public static UnsafeArrayCopyNode create(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, ValueNode layoutHelper, Kind elementKind) { + return new UnsafeArrayCopyNode(src, srcPos, dest, destPos, length, layoutHelper, elementKind); + } + + protected UnsafeArrayCopyNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, ValueNode layoutHelper, Kind elementKind) { + super(StampFactory.forVoid()); + assert layoutHelper == null || elementKind == null; + this.src = src; + this.srcPos = srcPos; + this.dest = dest; + this.destPos = destPos; + this.length = length; + this.layoutHelper = layoutHelper; + this.elementKind = elementKind; + } + + public static UnsafeArrayCopyNode create(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, Kind elementKind) { + return new UnsafeArrayCopyNode(src, srcPos, dest, destPos, length, elementKind); + } + + protected UnsafeArrayCopyNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, Kind elementKind) { + this(src, srcPos, dest, destPos, length, null, elementKind); + } + + public static UnsafeArrayCopyNode create(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, ValueNode layoutHelper) { + return new UnsafeArrayCopyNode(src, srcPos, dest, destPos, length, layoutHelper); + } + + protected UnsafeArrayCopyNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, ValueNode layoutHelper) { + this(src, srcPos, dest, destPos, length, layoutHelper, null); + } + + @Override + public ValueNode getArray() { + return dest; + } + + @Override + public ValueNode getIndex() { + return destPos; + } + + @Override + public ValueNode getLength() { + return length; + } + + @Override + public boolean isObjectArray() { + return elementKind == Kind.Object; + } + + @Override + public boolean isInitialization() { + return false; + } + + public Kind getElementKind() { + return elementKind; + } + + @Override + public void lower(LoweringTool tool) { + if (graph().getGuardsStage() == StructuredGraph.GuardsStage.AFTER_FSA) { + UnsafeArrayCopySnippets.Templates templates = tool.getReplacements().getSnippetTemplateCache(UnsafeArrayCopySnippets.Templates.class); + templates.lower(this, tool); + } + } + + public void addSnippetArguments(Arguments args) { + args.add("src", src); + args.add("srcPos", srcPos); + args.add("dest", dest); + args.add("destPos", destPos); + args.add("length", length); + if (layoutHelper != null) { + args.add("layoutHelper", layoutHelper); + } + } + + @Override + public LocationIdentity getLocationIdentity() { + if (elementKind != null) { + return NamedLocationIdentity.getArrayLocation(elementKind); + } + return ANY_LOCATION; + } + + @NodeIntrinsic + public static native void arraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter Kind elementKind); + + @NodeIntrinsic + public static native void arraycopyPrimitive(Object src, int srcPos, Object dest, int destPos, int length, int layoutHelper); +} diff -r be63951fa2bc -r 5afa718edb8b graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/UnsafeArrayCopySnippets.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/UnsafeArrayCopySnippets.java Wed Dec 03 10:35:42 2014 +0200 @@ -0,0 +1,310 @@ +/* + * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.graal.hotspot.replacements.arraycopy; + +import static com.oracle.graal.api.meta.LocationIdentity.*; +import static com.oracle.graal.hotspot.HotSpotGraalRuntime.*; +import static com.oracle.graal.hotspot.replacements.HotSpotReplacementsUtil.*; +import static com.oracle.graal.nodes.extended.BranchProbabilityNode.*; +import static com.oracle.graal.replacements.SnippetTemplate.*; + +import com.oracle.graal.api.code.*; +import com.oracle.graal.api.meta.*; +import com.oracle.graal.api.replacements.*; +import com.oracle.graal.asm.*; +import com.oracle.graal.hotspot.meta.*; +import com.oracle.graal.hotspot.phases.*; +import com.oracle.graal.nodes.extended.*; +import com.oracle.graal.nodes.spi.*; +import com.oracle.graal.replacements.*; +import com.oracle.graal.replacements.SnippetTemplate.AbstractTemplates; +import com.oracle.graal.replacements.SnippetTemplate.Arguments; +import com.oracle.graal.replacements.SnippetTemplate.SnippetInfo; +import com.oracle.graal.replacements.nodes.*; +import com.oracle.graal.word.*; + +/** + * As opposed to {@link ArrayCopySnippets}, these Snippets do not perform store checks. + */ +public class UnsafeArrayCopySnippets implements Snippets { + private static final boolean supportsUnalignedMemoryAccess = runtime().getTarget().arch.supportsUnalignedMemoryAccess(); + + private static final Kind VECTOR_KIND = Kind.Long; + private static final long VECTOR_SIZE = arrayIndexScale(VECTOR_KIND); + + private static void vectorizedCopy(Object src, int srcPos, Object dest, int destPos, int length, Kind baseKind, LocationIdentity locationIdentity) { + int arrayBaseOffset = arrayBaseOffset(baseKind); + int elementSize = arrayIndexScale(baseKind); + long byteLength = (long) length * elementSize; + long srcOffset = (long) srcPos * elementSize; + long destOffset = (long) destPos * elementSize; + + long preLoopBytes; + long mainLoopBytes; + long postLoopBytes; + + // We can easily vectorize the loop if both offsets have the same alignment. + if (byteLength >= VECTOR_SIZE && (srcOffset % VECTOR_SIZE) == (destOffset % VECTOR_SIZE)) { + preLoopBytes = NumUtil.roundUp(arrayBaseOffset + srcOffset, VECTOR_SIZE) - (arrayBaseOffset + srcOffset); + postLoopBytes = (byteLength - preLoopBytes) % VECTOR_SIZE; + mainLoopBytes = byteLength - preLoopBytes - postLoopBytes; + } else { + // Does the architecture support unaligned memory accesses? + if (supportsUnalignedMemoryAccess) { + preLoopBytes = byteLength % VECTOR_SIZE; + mainLoopBytes = byteLength - preLoopBytes; + postLoopBytes = 0; + } else { + // No. Let's do element-wise copying. + preLoopBytes = byteLength; + mainLoopBytes = 0; + postLoopBytes = 0; + } + } + + if (probability(NOT_FREQUENT_PROBABILITY, src == dest) && probability(NOT_FREQUENT_PROBABILITY, srcPos < destPos)) { + // bad aliased case + srcOffset += byteLength; + destOffset += byteLength; + + // Post-loop + for (long i = 0; i < postLoopBytes; i += elementSize) { + srcOffset -= elementSize; + destOffset -= elementSize; + Object a = UnsafeLoadNode.load(src, arrayBaseOffset + srcOffset, baseKind, locationIdentity); + UnsafeStoreNode.store(dest, arrayBaseOffset + destOffset, a, baseKind, locationIdentity); + } + // Main-loop + for (long i = 0; i < mainLoopBytes; i += VECTOR_SIZE) { + srcOffset -= VECTOR_SIZE; + destOffset -= VECTOR_SIZE; + Long a = UnsafeLoadNode.load(src, arrayBaseOffset + srcOffset, VECTOR_KIND, locationIdentity); + UnsafeStoreNode.store(dest, arrayBaseOffset + destOffset, a, VECTOR_KIND, locationIdentity); + } + // Pre-loop + for (long i = 0; i < preLoopBytes; i += elementSize) { + srcOffset -= elementSize; + destOffset -= elementSize; + Object a = UnsafeLoadNode.load(src, arrayBaseOffset + srcOffset, baseKind, locationIdentity); + UnsafeStoreNode.store(dest, arrayBaseOffset + destOffset, a, baseKind, locationIdentity); + } + } else { + // Pre-loop + for (long i = 0; i < preLoopBytes; i += elementSize) { + Object a = UnsafeLoadNode.load(src, arrayBaseOffset + srcOffset, baseKind, locationIdentity); + UnsafeStoreNode.store(dest, arrayBaseOffset + destOffset, a, baseKind, locationIdentity); + srcOffset += elementSize; + destOffset += elementSize; + } + // Main-loop + for (long i = 0; i < mainLoopBytes; i += VECTOR_SIZE) { + Long a = UnsafeLoadNode.load(src, arrayBaseOffset + srcOffset, VECTOR_KIND, locationIdentity); + UnsafeStoreNode.store(dest, arrayBaseOffset + destOffset, a, VECTOR_KIND, locationIdentity); + srcOffset += VECTOR_SIZE; + destOffset += VECTOR_SIZE; + } + // Post-loop + for (long i = 0; i < postLoopBytes; i += elementSize) { + Object a = UnsafeLoadNode.load(src, arrayBaseOffset + srcOffset, baseKind, locationIdentity); + UnsafeStoreNode.store(dest, arrayBaseOffset + destOffset, a, baseKind, locationIdentity); + srcOffset += elementSize; + destOffset += elementSize; + } + } + } + + @Fold + private static LocationIdentity getArrayLocation(Kind kind) { + return NamedLocationIdentity.getArrayLocation(kind); + } + + @Snippet + public static void arraycopyByte(byte[] src, int srcPos, byte[] dest, int destPos, int length) { + Kind kind = Kind.Byte; + vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); + } + + @Snippet + public static void arraycopyBoolean(boolean[] src, int srcPos, boolean[] dest, int destPos, int length) { + Kind kind = Kind.Boolean; + vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); + } + + @Snippet + public static void arraycopyChar(char[] src, int srcPos, char[] dest, int destPos, int length) { + Kind kind = Kind.Char; + vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); + } + + @Snippet + public static void arraycopyShort(short[] src, int srcPos, short[] dest, int destPos, int length) { + Kind kind = Kind.Short; + vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); + } + + @Snippet + public static void arraycopyInt(int[] src, int srcPos, int[] dest, int destPos, int length) { + Kind kind = Kind.Int; + vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); + } + + @Snippet + public static void arraycopyFloat(float[] src, int srcPos, float[] dest, int destPos, int length) { + Kind kind = Kind.Float; + vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); + } + + @Snippet + public static void arraycopyLong(long[] src, int srcPos, long[] dest, int destPos, int length) { + Kind kind = Kind.Long; + vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); + } + + @Snippet + public static void arraycopyDouble(double[] src, int srcPos, double[] dest, int destPos, int length) { + Kind kind = Kind.Double; + /* + * TODO atomicity problem on 32-bit architectures: The JVM spec requires double values to be + * copied atomically, but not long values. For example, on Intel 32-bit this code is not + * atomic as long as the vector kind remains Kind.Long. + */ + vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); + } + + /** + * For this kind, Object, we want to avoid write barriers between writes, but instead have them + * at the end of the snippet. This is done by using {@link DirectObjectStoreNode}, and rely on + * {@link WriteBarrierAdditionPhase} to put write barriers after the {@link UnsafeArrayCopyNode} + * with kind Object. + */ + @Snippet + public static void arraycopyObject(Object[] src, int srcPos, Object[] dest, int destPos, int length) { + Kind kind = Kind.Object; + final int scale = arrayIndexScale(kind); + int arrayBaseOffset = arrayBaseOffset(kind); + LocationIdentity arrayLocation = getArrayLocation(kind); + if (src == dest && srcPos < destPos) { // bad aliased case + long start = (long) (length - 1) * scale; + for (long i = start; i >= 0; i -= scale) { + Object a = UnsafeLoadNode.load(src, arrayBaseOffset + i + (long) srcPos * scale, kind, arrayLocation); + DirectObjectStoreNode.storeObject(dest, arrayBaseOffset, i + (long) destPos * scale, a, getArrayLocation(kind)); + } + } else { + long end = (long) length * scale; + for (long i = 0; i < end; i += scale) { + Object a = UnsafeLoadNode.load(src, arrayBaseOffset + i + (long) srcPos * scale, kind, arrayLocation); + DirectObjectStoreNode.storeObject(dest, arrayBaseOffset, i + (long) destPos * scale, a, getArrayLocation(kind)); + } + } + } + + @Snippet + public static void arraycopyPrimitive(Object src, int srcPos, Object dest, int destPos, int length, int layoutHelper) { + int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift()) & layoutHelperLog2ElementSizeMask(); + int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift()) & layoutHelperHeaderSizeMask(); + + Unsigned vectorSize = Word.unsigned(VECTOR_SIZE); + Unsigned srcOffset = Word.unsigned(srcPos).shiftLeft(log2ElementSize).add(headerSize); + Unsigned destOffset = Word.unsigned(destPos).shiftLeft(log2ElementSize).add(headerSize); + Unsigned destStart = destOffset; + Unsigned destEnd = destOffset.add(Word.unsigned(length).shiftLeft(log2ElementSize)); + + Unsigned destVectorEnd = null; + Unsigned nonVectorBytes = null; + Unsigned sizeInBytes = Word.unsigned(length).shiftLeft(log2ElementSize); + if (supportsUnalignedMemoryAccess) { + nonVectorBytes = sizeInBytes.unsignedRemainder(vectorSize); + destVectorEnd = destEnd; + } else { + boolean inPhase = srcOffset.and((int) VECTOR_SIZE - 1).equal(destOffset.and((int) VECTOR_SIZE - 1)); + boolean hasAtLeastOneVector = sizeInBytes.aboveOrEqual(vectorSize); + // We must have at least one full vector, otherwise we must copy each byte separately + if (hasAtLeastOneVector && inPhase) { // If in phase, we can vectorize + nonVectorBytes = vectorSize.subtract(destStart.unsignedRemainder(vectorSize)); + } else { // fallback is byte-wise + nonVectorBytes = sizeInBytes; + } + destVectorEnd = destEnd.subtract(destEnd.unsignedRemainder(vectorSize)); + } + + Unsigned destNonVectorEnd = destStart.add(nonVectorBytes); + while (destOffset.belowThan(destNonVectorEnd)) { + ObjectAccess.writeByte(dest, destOffset, ObjectAccess.readByte(src, srcOffset, ANY_LOCATION), ANY_LOCATION); + destOffset = destOffset.add(1); + srcOffset = srcOffset.add(1); + } + // Unsigned destVectorEnd = destEnd.subtract(destEnd.unsignedRemainder(8)); + while (destOffset.belowThan(destVectorEnd)) { + ObjectAccess.writeWord(dest, destOffset, ObjectAccess.readWord(src, srcOffset, ANY_LOCATION), ANY_LOCATION); + destOffset = destOffset.add(wordSize()); + srcOffset = srcOffset.add(wordSize()); + } + // Do the last bytes each when it is required to have absolute alignment. + while (!supportsUnalignedMemoryAccess && destOffset.belowThan(destEnd)) { + ObjectAccess.writeByte(dest, destOffset, ObjectAccess.readByte(src, srcOffset, ANY_LOCATION), ANY_LOCATION); + destOffset = destOffset.add(1); + srcOffset = srcOffset.add(1); + } + } + + public static class Templates extends AbstractTemplates { + + private final SnippetInfo[] arraycopySnippets; + private final SnippetInfo genericPrimitiveSnippet; + + public Templates(HotSpotProviders providers, TargetDescription target) { + super(providers, providers.getSnippetReflection(), target); + + arraycopySnippets = new SnippetInfo[Kind.values().length]; + arraycopySnippets[Kind.Boolean.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyBoolean"); + arraycopySnippets[Kind.Byte.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyByte"); + arraycopySnippets[Kind.Short.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyShort"); + arraycopySnippets[Kind.Char.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyChar"); + arraycopySnippets[Kind.Int.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyInt"); + arraycopySnippets[Kind.Long.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyLong"); + arraycopySnippets[Kind.Float.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyFloat"); + arraycopySnippets[Kind.Double.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyDouble"); + arraycopySnippets[Kind.Object.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyObject"); + + genericPrimitiveSnippet = snippet(UnsafeArrayCopySnippets.class, "arraycopyPrimitive"); + } + + public void lower(UnsafeArrayCopyNode node, LoweringTool tool) { + Kind elementKind = node.getElementKind(); + SnippetInfo snippet; + if (elementKind == null) { + // primitive array of unknown kind + snippet = genericPrimitiveSnippet; + } else { + snippet = arraycopySnippets[elementKind.ordinal()]; + assert snippet != null : "arraycopy snippet for " + elementKind.name() + " not found"; + } + + Arguments args = new Arguments(snippet, node.graph().getGuardsStage(), tool.getLoweringStage()); + node.addSnippetArguments(args); + + SnippetTemplate template = template(args); + template.instantiate(providers.getMetaAccess(), node, DEFAULT_REPLACER, args); + } + } +}