Mercurial > hg > truffle
changeset 13977:39076a984c33
lower arraycopy calls later and support unchecked object arraycopy
| author | Tom Rodriguez <tom.rodriguez@oracle.com> |
|---|---|
| date | Wed, 19 Feb 2014 00:39:44 -0800 |
| parents | 272995b3c019 |
| children | 67905c049016 f46cab39a9a2 |
| files | graal/com.oracle.graal.graph/src/com/oracle/graal/graph/NodeMap.java graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/HotSpotHostForeignCallsProvider.java graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ArrayCopyCallNode.java graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ArrayCopyNode.java graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ArrayCopySnippets.java graal/com.oracle.graal.phases.common/src/com/oracle/graal/phases/common/InliningUtil.java graal/com.oracle.graal.replacements/src/com/oracle/graal/replacements/ReplacementsImpl.java |
| diffstat | 7 files changed, 295 insertions(+), 227 deletions(-) [+] |
line wrap: on
line diff
--- a/graal/com.oracle.graal.graph/src/com/oracle/graal/graph/NodeMap.java Wed Feb 19 15:58:45 2014 +0100 +++ b/graal/com.oracle.graal.graph/src/com/oracle/graal/graph/NodeMap.java Wed Feb 19 00:39:44 2014 -0800 @@ -101,7 +101,7 @@ if (autogrow && isNew(node)) { grow(); } - assert node.graph() == graph : "this node is not part of the graph"; + assert node.graph() == graph : String.format("%s is not part of the graph", node); assert !isNew(node) : "this node was added to the graph after creating the node map : " + node; }
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/HotSpotHostForeignCallsProvider.java Wed Feb 19 15:58:45 2014 +0100 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/HotSpotHostForeignCallsProvider.java Wed Feb 19 00:39:44 2014 -0800 @@ -48,11 +48,13 @@ import static com.oracle.graal.replacements.Log.*; import static com.oracle.graal.replacements.MathSubstitutionsX86.*; +import java.util.*; + import com.oracle.graal.api.code.*; import com.oracle.graal.api.meta.*; import com.oracle.graal.hotspot.*; -import com.oracle.graal.hotspot.replacements.*; import com.oracle.graal.hotspot.stubs.*; +import com.oracle.graal.word.*; /** * HotSpot implementation of {@link ForeignCallsProvider}. @@ -67,12 +69,35 @@ stub.getLinkage().setCompiledStub(stub); } + public static ForeignCallDescriptor lookupArraycopyDescriptor(Kind kind, boolean aligned, boolean disjoint) { + return (ForeignCallDescriptor) arraycopyDescriptors[aligned ? 1 : 0][disjoint ? 1 : 0].get(kind); + } + + private static final EnumMap[][] arraycopyDescriptors = new EnumMap[2][2]; + + static { + // Populate the EnumMap instances + for (int i = 0; i < arraycopyDescriptors.length; i++) { + for (int j = 0; j < arraycopyDescriptors[i].length; j++) { + arraycopyDescriptors[i][j] = new EnumMap<Kind, ForeignCallDescriptor>(Kind.class); + } + } + } + + @SuppressWarnings("unchecked") + private static ForeignCallDescriptor registerArraycopyDescriptor(Kind kind, boolean aligned, boolean disjoint) { + String name = kind + (aligned ? "Aligned" : "") + (disjoint ? "Disjoint" : "") + "Arraycopy"; + ForeignCallDescriptor desc = new ForeignCallDescriptor(name, void.class, Word.class, Word.class, Word.class); + arraycopyDescriptors[aligned ? 1 : 0][disjoint ? 1 : 0].put(kind, desc); + return desc; + } + private void registerArrayCopy(Kind kind, long routine, long alignedRoutine, long disjointRoutine, long alignedDisjointRoutine) { LocationIdentity killed = NamedLocationIdentity.getArrayLocation(kind); - registerForeignCall(ArrayCopySnippets.lookupArraycopyDescriptor(kind, false, false), routine, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, NOT_REEXECUTABLE, killed); - registerForeignCall(ArrayCopySnippets.lookupArraycopyDescriptor(kind, true, false), alignedRoutine, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, NOT_REEXECUTABLE, killed); - registerForeignCall(ArrayCopySnippets.lookupArraycopyDescriptor(kind, false, true), disjointRoutine, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, NOT_REEXECUTABLE, killed); - registerForeignCall(ArrayCopySnippets.lookupArraycopyDescriptor(kind, true, true), alignedDisjointRoutine, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, NOT_REEXECUTABLE, killed); + registerForeignCall(registerArraycopyDescriptor(kind, false, false), routine, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, NOT_REEXECUTABLE, killed); + registerForeignCall(registerArraycopyDescriptor(kind, true, false), alignedRoutine, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, NOT_REEXECUTABLE, killed); + registerForeignCall(registerArraycopyDescriptor(kind, false, true), disjointRoutine, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, NOT_REEXECUTABLE, killed); + registerForeignCall(registerArraycopyDescriptor(kind, true, true), alignedDisjointRoutine, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, NOT_REEXECUTABLE, killed); } public void initialize(HotSpotProviders providers, HotSpotVMConfig c) { @@ -128,6 +153,6 @@ registerArrayCopy(Kind.Float, c.jintArraycopy, c.jintAlignedArraycopy, c.jintDisjointArraycopy, c.jintAlignedDisjointArraycopy); registerArrayCopy(Kind.Long, c.jlongArraycopy, c.jlongAlignedArraycopy, c.jlongDisjointArraycopy, c.jlongAlignedDisjointArraycopy); registerArrayCopy(Kind.Double, c.jlongArraycopy, c.jlongAlignedArraycopy, c.jlongDisjointArraycopy, c.jlongAlignedDisjointArraycopy); - + registerArrayCopy(Kind.Object, c.oopArraycopy, c.oopArraycopy, c.oopDisjointArraycopy, c.oopDisjointArraycopy); } }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ArrayCopyCallNode.java Wed Feb 19 00:39:44 2014 -0800 @@ -0,0 +1,194 @@ +/* + * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.graal.hotspot.replacements; + +import static com.oracle.graal.api.meta.LocationIdentity.*; +import static com.oracle.graal.hotspot.replacements.HotSpotReplacementsUtil.*; + +import com.oracle.graal.api.meta.*; +import com.oracle.graal.api.runtime.*; +import com.oracle.graal.hotspot.meta.*; +import com.oracle.graal.hotspot.nodes.*; +import com.oracle.graal.nodes.*; +import com.oracle.graal.nodes.calc.*; +import com.oracle.graal.nodes.extended.*; +import com.oracle.graal.nodes.spi.*; +import com.oracle.graal.nodes.type.*; +import com.oracle.graal.phases.*; +import com.oracle.graal.replacements.SnippetTemplate.Arguments; +import com.oracle.graal.runtime.*; + +public final class ArrayCopyCallNode extends ArrayRangeWriteNode implements Lowerable, MemoryCheckpoint.Single { + + @Input private ValueNode src; + @Input private ValueNode srcPos; + @Input private ValueNode dest; + @Input private ValueNode destPos; + @Input private ValueNode length; + + private Kind elementKind; + private boolean aligned; + private boolean disjoint; + + private ArrayCopyCallNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, Kind elementKind, boolean aligned, boolean disjoint) { + super(StampFactory.forVoid()); + assert elementKind != null; + this.src = src; + this.srcPos = srcPos; + this.dest = dest; + this.destPos = destPos; + this.length = length; + this.elementKind = elementKind; + this.aligned = aligned; + this.disjoint = disjoint; + } + + private ArrayCopyCallNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, Kind elementKind) { + this(src, srcPos, dest, destPos, length, elementKind, false, false); + } + + public ValueNode getSource() { + return src; + } + + public ValueNode getSourcePosition() { + return srcPos; + } + + public ValueNode getDestination() { + return dest; + } + + public ValueNode getDestinationPosition() { + return destPos; + } + + @Override + public ValueNode getArray() { + return dest; + } + + @Override + public ValueNode getIndex() { + return destPos; + } + + @Override + public ValueNode getLength() { + return length; + } + + @Override + public boolean isObjectArray() { + return elementKind == Kind.Object; + } + + @Override + public boolean isInitialization() { + return false; + } + + public void addSnippetArguments(Arguments args) { + args.add("src", src); + args.add("srcPos", srcPos); + args.add("dest", dest); + args.add("destPos", destPos); + args.add("length", length); + } + + public Kind getElementKind() { + return elementKind; + } + + private boolean shouldUnroll() { + return getLength().isConstant() && getLength().asConstant().asInt() <= GraalOptions.MaximumEscapeAnalysisArrayLength.getValue(); + } + + private ValueNode computeBase(ValueNode base, ValueNode pos) { + FixedWithNextNode basePtr = graph().add(new GetObjectAddressNode(base)); + graph().addBeforeFixed(this, basePtr); + ValueNode loc = IndexedLocationNode.create(getLocationIdentity(), elementKind, arrayBaseOffset(elementKind), pos, graph(), arrayIndexScale(elementKind)); + return graph().unique(new ComputeAddressNode(basePtr, loc, StampFactory.forKind(Kind.Long))); + } + + @Override + public void lower(LoweringTool tool) { + if (graph().getGuardsStage() == StructuredGraph.GuardsStage.AFTER_FSA) { + updateAlignedDisjoint(); + ForeignCallDescriptor desc = HotSpotHostForeignCallsProvider.lookupArraycopyDescriptor(elementKind, isAligned(), isDisjoint()); + StructuredGraph graph = graph(); + ValueNode srcAddr = computeBase(getSource(), getSourcePosition()); + ValueNode destAddr = computeBase(getDestination(), getDestinationPosition()); + ValueNode len = getLength(); + if (len.kind() != Kind.Long) { + len = graph().unique(new ConvertNode(len.kind(), Kind.Long, len)); + } + ForeignCallNode call = graph.add(new ForeignCallNode(Graal.getRequiredCapability(RuntimeProvider.class).getHostBackend().getForeignCalls(), desc, srcAddr, destAddr, len)); + call.setStateAfter(stateAfter()); + graph.replaceFixedWithFixed(this, call); + + } + } + + @Override + public LocationIdentity getLocationIdentity() { + if (elementKind != null) { + return NamedLocationIdentity.getArrayLocation(elementKind); + } + return ANY_LOCATION; + } + + @NodeIntrinsic + public static native void arraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter Kind elementKind); + + @NodeIntrinsic + public static native void arraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter Kind elementKind, @ConstantNodeParameter boolean aligned, + @ConstantNodeParameter boolean disjoint); + + public boolean isAligned() { + return aligned; + } + + public boolean isDisjoint() { + return disjoint; + } + + public void updateAlignedDisjoint() { + Kind componentKind = elementKind; + if (srcPos == destPos) { + // Can treat as disjoint + disjoint = true; + } + Constant constantSrc = srcPos.stamp().asConstant(); + Constant constantDst = destPos.stamp().asConstant(); + if (constantSrc != null && constantDst != null) { + if (!aligned) { + aligned = ArrayCopyNode.isHeapWordAligned(constantSrc, componentKind) && ArrayCopyNode.isHeapWordAligned(constantDst, componentKind); + } + if (constantSrc.asInt() >= constantDst.asInt()) { + // low to high copy so treat as disjoint + disjoint = true; + } + } + } +}
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ArrayCopyNode.java Wed Feb 19 15:58:45 2014 +0100 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ArrayCopyNode.java Wed Feb 19 00:39:44 2014 -0800 @@ -64,8 +64,8 @@ return arguments.get(4); } - private static boolean isHeapWordAligned(Constant value, Kind kind) { - return (arrayBaseOffset(kind) + value.asInt() * arrayIndexScale(kind)) % heapWordSize() == 0; + static boolean isHeapWordAligned(Constant value, Kind kind) { + return (arrayBaseOffset(kind) + (long) value.asInt() * arrayIndexScale(kind)) % heapWordSize() == 0; } private StructuredGraph selectSnippet(LoweringTool tool, final Replacements replacements) { @@ -75,27 +75,14 @@ if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) { return null; } - if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType()) || !ObjectStamp.isExactType(getDestination().stamp())) { + if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) { + return null; + } + if (!isExact()) { return null; } Kind componentKind = srcType.getComponentType().getKind(); - boolean disjoint = false; - boolean aligned = false; - if (getSourcePosition() == getDestinationPosition()) { - // Can treat as disjoint - disjoint = true; - } - Constant constantSrc = getSourcePosition().stamp().asConstant(); - Constant constantDst = getDestinationPosition().stamp().asConstant(); - if (constantSrc != null && constantDst != null) { - aligned = isHeapWordAligned(constantSrc, componentKind) && isHeapWordAligned(constantDst, componentKind); - if (constantSrc.asInt() >= constantDst.asInt()) { - // low to high copy so treat as disjoint - disjoint = true; - } - } - - final ResolvedJavaMethod snippetMethod = tool.getMetaAccess().lookupJavaMethod(ArrayCopySnippets.getSnippetForKind(componentKind, aligned, disjoint)); + final ResolvedJavaMethod snippetMethod = tool.getMetaAccess().lookupJavaMethod(ArrayCopySnippets.getSnippetForKind(componentKind, shouldUnroll(), isExact())); try (Scope s = Debug.scope("ArrayCopySnippet", snippetMethod)) { return replacements.getSnippet(snippetMethod); } catch (Throwable e) { @@ -136,7 +123,7 @@ } else { assert snippetGraph != null : "ArrayCopySnippets should be installed"; snippetGraph = snippetGraph.copy(); - if (getLength().isConstant() && getLength().asConstant().asInt() <= GraalOptions.MaximumEscapeAnalysisArrayLength.getValue()) { + if (shouldUnroll()) { final StructuredGraph copy = snippetGraph; try (Scope s = Debug.scope("ArrayCopySnippetSpecialization", snippetGraph.method())) { unrollFixedLengthLoop(copy, getLength().asConstant().asInt(), tool); @@ -148,6 +135,28 @@ return lowerReplacement(snippetGraph, tool); } + private boolean shouldUnroll() { + return getLength().isConstant() && getLength().asConstant().asInt() <= GraalOptions.MaximumEscapeAnalysisArrayLength.getValue(); + } + + /* + * Returns true if this copy doesn't require store checks. Trivially true for primitive arrays. + */ + private boolean isExact() { + ResolvedJavaType srcType = ObjectStamp.typeOrNull(getSource().stamp()); + if (srcType.getComponentType().getKind().isPrimitive() || getSource() == getDestination()) { + return true; + } + + ResolvedJavaType destType = ObjectStamp.typeOrNull(getDestination().stamp()); + if (ObjectStamp.isExactType(getDestination().stamp())) { + if (destType != null && destType.isAssignableFrom(srcType)) { + return true; + } + } + return false; + } + private static boolean checkBounds(int position, int length, VirtualObjectNode virtualObject) { return position >= 0 && position + length <= virtualObject.entryCount(); }
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ArrayCopySnippets.java Wed Feb 19 15:58:45 2014 +0100 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ArrayCopySnippets.java Wed Feb 19 00:39:44 2014 -0800 @@ -22,77 +22,57 @@ */ package com.oracle.graal.hotspot.replacements; -import static com.oracle.graal.api.meta.LocationIdentity.*; import static com.oracle.graal.hotspot.replacements.HotSpotReplacementsUtil.*; import static com.oracle.graal.nodes.GuardingPiNode.*; -import static com.oracle.graal.nodes.calc.IsNullNode.*; import static com.oracle.graal.nodes.extended.BranchProbabilityNode.*; import static com.oracle.graal.phases.GraalOptions.*; +import static com.oracle.graal.replacements.SnippetTemplate.*; import java.lang.reflect.*; import java.util.*; import com.oracle.graal.api.code.*; import com.oracle.graal.api.meta.*; +import com.oracle.graal.debug.*; +import com.oracle.graal.debug.Debug.Scope; import com.oracle.graal.graph.*; -import com.oracle.graal.graph.Node.*; -import com.oracle.graal.hotspot.nodes.*; +import com.oracle.graal.graph.Node.ConstantNodeParameter; +import com.oracle.graal.graph.Node.NodeIntrinsic; +import com.oracle.graal.loop.phases.*; import com.oracle.graal.nodes.*; -import com.oracle.graal.nodes.calc.*; import com.oracle.graal.nodes.extended.*; import com.oracle.graal.nodes.java.*; -import com.oracle.graal.nodes.type.*; -import com.oracle.graal.phases.*; +import com.oracle.graal.nodes.spi.*; +import com.oracle.graal.phases.common.*; +import com.oracle.graal.phases.tiers.*; +import com.oracle.graal.phases.util.*; import com.oracle.graal.replacements.*; import com.oracle.graal.replacements.Snippet.Fold; -import com.oracle.graal.replacements.nodes.*; +import com.oracle.graal.replacements.SnippetTemplate.AbstractTemplates; +import com.oracle.graal.replacements.SnippetTemplate.Arguments; +import com.oracle.graal.replacements.SnippetTemplate.SnippetInfo; import com.oracle.graal.word.*; @SuppressWarnings("unused") public class ArrayCopySnippets implements Snippets { private static final EnumMap<Kind, Method> arraycopyMethods = new EnumMap<>(Kind.class); - private static final EnumMap[][] arraycopyCalls = new EnumMap[2][2]; - private static final EnumMap[][] arraycopyDescriptors = new EnumMap[2][2]; + private static final EnumMap<Kind, Method> arraycopyCalls = new EnumMap<>(Kind.class); public static final Method genericArraycopySnippet; - @SuppressWarnings("unchecked") - private static void findArraycopyCall(Kind kind, Class<?> arrayClass, boolean aligned, boolean disjoint) throws NoSuchMethodException { - String name = kind + (aligned ? "Aligned" : "") + (disjoint ? "Disjoint" : "") + "Arraycopy"; - arraycopyCalls[aligned ? 1 : 0][disjoint ? 1 : 0].put(kind, ArrayCopySnippets.class.getDeclaredMethod(name, arrayClass, int.class, arrayClass, int.class, int.class)); - arraycopyDescriptors[aligned ? 1 : 0][disjoint ? 1 : 0].put(kind, new ForeignCallDescriptor(name, void.class, Word.class, Word.class, Word.class)); - } - - private static Method lookupArraycopyCall(Kind kind, boolean aligned, boolean disjoint) { - return (Method) arraycopyCalls[aligned ? 1 : 0][disjoint ? 1 : 0].get(kind); - } - - @Fold - public static ForeignCallDescriptor lookupArraycopyDescriptor(Kind kind, boolean aligned, boolean disjoint) { - return (ForeignCallDescriptor) arraycopyDescriptors[aligned ? 1 : 0][disjoint ? 1 : 0].get(kind); - } - private static void addArraycopySnippetMethod(Kind kind, Class<?> arrayClass) throws NoSuchMethodException { arraycopyMethods.put(kind, ArrayCopySnippets.class.getDeclaredMethod("arraycopy", arrayClass, int.class, arrayClass, int.class, int.class)); if (CallArrayCopy.getValue()) { - if (kind != Kind.Object) { - // Only primitive types are currently supported - findArraycopyCall(kind, arrayClass, false, false); - findArraycopyCall(kind, arrayClass, false, true); - findArraycopyCall(kind, arrayClass, true, false); - findArraycopyCall(kind, arrayClass, true, true); + if (kind == Kind.Object) { + arraycopyCalls.put(kind, ArrayCopySnippets.class.getDeclaredMethod("objectArraycopyUnchecked", arrayClass, int.class, arrayClass, int.class, int.class)); + } else { + arraycopyCalls.put(kind, ArrayCopySnippets.class.getDeclaredMethod(kind + "Arraycopy", arrayClass, int.class, arrayClass, int.class, int.class)); } } } static { - for (int i = 0; i < arraycopyCalls.length; i++) { - for (int j = 0; j < arraycopyCalls[i].length; j++) { - arraycopyCalls[i][j] = new EnumMap<Kind, Method>(Kind.class); - arraycopyDescriptors[i][j] = new EnumMap<Kind, ForeignCallDescriptor>(Kind.class); - } - } try { addArraycopySnippetMethod(Kind.Byte, byte[].class); addArraycopySnippetMethod(Kind.Boolean, boolean[].class); @@ -109,10 +89,13 @@ } } - public static Method getSnippetForKind(Kind kind, boolean aligned, boolean disjoint) { - Method m = lookupArraycopyCall(kind, aligned, disjoint); - if (m != null) { - return m; + public static Method getSnippetForKind(Kind kind, boolean shouldUnroll, boolean exact) { + Method m = null; + if (!shouldUnroll && exact) { + m = arraycopyCalls.get(kind); + if (m != null) { + return m; + } } return arraycopyMethods.get(kind); } @@ -235,23 +218,19 @@ } @NodeIntrinsic(ForeignCallNode.class) - public static native void callArraycopy(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word src, Word dst, Word len); - - private static Word computeBase(Object object, Kind kind, int pos) { - // In this code pos must be cast to long before being multiplied since shifting the value - // could be outside the range of int. arrayIndexScale should probably either return long or - // Word to force the right thing to happen. - return Word.unsigned(GetObjectAddressNode.get(object) + arrayBaseOffset(kind) + (long) pos * arrayIndexScale(kind)); - } + public static native void callArraycopy(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word src, Word dest, Word len); private static void callArraycopyTemplate(SnippetCounter counter, Kind kind, boolean aligned, boolean disjoint, Object src, int srcPos, Object dest, int destPos, int length) { counter.inc(); Object nonNullSrc = guardingNonNull(src); Object nonNullDest = guardingNonNull(dest); checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); - Word srcAddr = computeBase(src, kind, srcPos); - Word destAddr = computeBase(dest, kind, destPos); - callArraycopy(lookupArraycopyDescriptor(kind, aligned, disjoint), srcAddr, destAddr, Word.signed(length)); + ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, kind, aligned, disjoint); + } + + @Snippet + public static void objectArraycopyUnchecked(Object[] src, int srcPos, Object[] dest, int destPos, int length) { + callArraycopyTemplate(objectCallCounter, Kind.Object, false, false, src, srcPos, dest, destPos, length); } @Snippet @@ -260,160 +239,40 @@ } @Snippet - public static void byteDisjointArraycopy(byte[] src, int srcPos, byte[] dest, int destPos, int length) { - callArraycopyTemplate(byteDisjointCallCounter, Kind.Byte, false, true, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void byteAlignedArraycopy(byte[] src, int srcPos, byte[] dest, int destPos, int length) { - callArraycopyTemplate(byteAlignedCallCounter, Kind.Byte, true, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void byteAlignedDisjointArraycopy(byte[] src, int srcPos, byte[] dest, int destPos, int length) { - callArraycopyTemplate(byteAlignedDisjointCallCounter, Kind.Byte, true, true, src, srcPos, dest, destPos, length); - } - - @Snippet public static void booleanArraycopy(boolean[] src, int srcPos, boolean[] dest, int destPos, int length) { callArraycopyTemplate(booleanCallCounter, Kind.Boolean, false, false, src, srcPos, dest, destPos, length); } @Snippet - public static void booleanDisjointArraycopy(boolean[] src, int srcPos, boolean[] dest, int destPos, int length) { - callArraycopyTemplate(booleanDisjointCallCounter, Kind.Boolean, false, true, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void booleanAlignedArraycopy(boolean[] src, int srcPos, boolean[] dest, int destPos, int length) { - callArraycopyTemplate(booleanAlignedCallCounter, Kind.Boolean, true, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void booleanAlignedDisjointArraycopy(boolean[] src, int srcPos, boolean[] dest, int destPos, int length) { - callArraycopyTemplate(booleanAlignedDisjointCallCounter, Kind.Boolean, true, true, src, srcPos, dest, destPos, length); - } - - @Snippet public static void charArraycopy(char[] src, int srcPos, char[] dest, int destPos, int length) { callArraycopyTemplate(charCallCounter, Kind.Char, false, false, src, srcPos, dest, destPos, length); } @Snippet - public static void charDisjointArraycopy(char[] src, int srcPos, char[] dest, int destPos, int length) { - callArraycopyTemplate(charDisjointCallCounter, Kind.Char, false, true, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void charAlignedArraycopy(char[] src, int srcPos, char[] dest, int destPos, int length) { - callArraycopyTemplate(charAlignedCallCounter, Kind.Char, true, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void charAlignedDisjointArraycopy(char[] src, int srcPos, char[] dest, int destPos, int length) { - callArraycopyTemplate(charAlignedDisjointCallCounter, Kind.Char, true, true, src, srcPos, dest, destPos, length); - } - - @Snippet public static void shortArraycopy(short[] src, int srcPos, short[] dest, int destPos, int length) { callArraycopyTemplate(shortCallCounter, Kind.Short, false, false, src, srcPos, dest, destPos, length); } @Snippet - public static void shortDisjointArraycopy(short[] src, int srcPos, short[] dest, int destPos, int length) { - callArraycopyTemplate(shortDisjointCallCounter, Kind.Short, false, true, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void shortAlignedArraycopy(short[] src, int srcPos, short[] dest, int destPos, int length) { - callArraycopyTemplate(shortAlignedCallCounter, Kind.Short, true, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void shortAlignedDisjointArraycopy(short[] src, int srcPos, short[] dest, int destPos, int length) { - callArraycopyTemplate(shortAlignedDisjointCallCounter, Kind.Short, true, true, src, srcPos, dest, destPos, length); - } - - @Snippet public static void intArraycopy(int[] src, int srcPos, int[] dest, int destPos, int length) { callArraycopyTemplate(intCallCounter, Kind.Int, false, false, src, srcPos, dest, destPos, length); } @Snippet - public static void intDisjointArraycopy(int[] src, int srcPos, int[] dest, int destPos, int length) { - callArraycopyTemplate(intDisjointCallCounter, Kind.Int, false, true, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void intAlignedArraycopy(int[] src, int srcPos, int[] dest, int destPos, int length) { - callArraycopyTemplate(intAlignedCallCounter, Kind.Int, true, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void intAlignedDisjointArraycopy(int[] src, int srcPos, int[] dest, int destPos, int length) { - callArraycopyTemplate(intAlignedDisjointCallCounter, Kind.Int, true, true, src, srcPos, dest, destPos, length); - } - - @Snippet public static void floatArraycopy(float[] src, int srcPos, float[] dest, int destPos, int length) { callArraycopyTemplate(floatCallCounter, Kind.Float, false, false, src, srcPos, dest, destPos, length); } @Snippet - public static void floatDisjointArraycopy(float[] src, int srcPos, float[] dest, int destPos, int length) { - callArraycopyTemplate(floatDisjointCallCounter, Kind.Float, false, true, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void floatAlignedArraycopy(float[] src, int srcPos, float[] dest, int destPos, int length) { - callArraycopyTemplate(floatAlignedCallCounter, Kind.Float, true, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void floatAlignedDisjointArraycopy(float[] src, int srcPos, float[] dest, int destPos, int length) { - callArraycopyTemplate(floatAlignedDisjointCallCounter, Kind.Float, true, true, src, srcPos, dest, destPos, length); - } - - @Snippet public static void longArraycopy(long[] src, int srcPos, long[] dest, int destPos, int length) { callArraycopyTemplate(longCallCounter, Kind.Long, false, false, src, srcPos, dest, destPos, length); } @Snippet - public static void longDisjointArraycopy(long[] src, int srcPos, long[] dest, int destPos, int length) { - callArraycopyTemplate(longDisjointCallCounter, Kind.Long, false, true, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void longAlignedArraycopy(long[] src, int srcPos, long[] dest, int destPos, int length) { - callArraycopyTemplate(longAlignedCallCounter, Kind.Long, true, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void longAlignedDisjointArraycopy(long[] src, int srcPos, long[] dest, int destPos, int length) { - callArraycopyTemplate(longAlignedDisjointCallCounter, Kind.Long, true, true, src, srcPos, dest, destPos, length); - } - - @Snippet public static void doubleArraycopy(double[] src, int srcPos, double[] dest, int destPos, int length) { callArraycopyTemplate(doubleCallCounter, Kind.Double, false, false, src, srcPos, dest, destPos, length); } - @Snippet - public static void doubleDisjointArraycopy(double[] src, int srcPos, double[] dest, int destPos, int length) { - callArraycopyTemplate(doubleDisjointCallCounter, Kind.Double, false, true, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void doubleAlignedArraycopy(double[] src, int srcPos, double[] dest, int destPos, int length) { - callArraycopyTemplate(doubleAlignedCallCounter, Kind.Double, true, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void doubleAlignedDisjointArraycopy(double[] src, int srcPos, double[] dest, int destPos, int length) { - callArraycopyTemplate(doubleAlignedDisjointCallCounter, Kind.Double, true, true, src, srcPos, dest, destPos, length); - } - private static final SnippetCounter.Group checkCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy checkInputs") : null; private static final SnippetCounter checkSuccessCounter = new SnippetCounter(checkCounters, "checkSuccess", "checkSuccess"); private static final SnippetCounter checkNPECounter = new SnippetCounter(checkCounters, "checkNPE", "checkNPE"); @@ -430,41 +289,18 @@ private static final SnippetCounter floatCounter = new SnippetCounter(counters, "float[]", "arraycopy for float[] arrays"); private static final SnippetCounter doubleCounter = new SnippetCounter(counters, "double[]", "arraycopy for double[] arrays"); - private static final SnippetCounter booleanAlignedCallCounter = new SnippetCounter(counters, "boolean[]", "aligned arraycopy call for boolean[] arrays"); - private static final SnippetCounter booleanAlignedDisjointCallCounter = new SnippetCounter(counters, "boolean[]", "aligned disjoint arraycopy call for boolean[] arrays"); + private static final SnippetCounter objectCallCounter = new SnippetCounter(counters, "Object[]", "arraycopy call for Object[] arrays"); + private static final SnippetCounter booleanCallCounter = new SnippetCounter(counters, "boolean[]", "arraycopy call for boolean[] arrays"); - private static final SnippetCounter booleanDisjointCallCounter = new SnippetCounter(counters, "boolean[]", "disjoint arraycopy call for boolean[] arrays"); - private static final SnippetCounter byteAlignedCallCounter = new SnippetCounter(counters, "byte[]", "aligned arraycopy call for byte[] arrays"); - private static final SnippetCounter byteAlignedDisjointCallCounter = new SnippetCounter(counters, "byte[]", "aligned disjoint arraycopy call for byte[] arrays"); private static final SnippetCounter byteCallCounter = new SnippetCounter(counters, "byte[]", "arraycopy call for byte[] arrays"); - private static final SnippetCounter byteDisjointCallCounter = new SnippetCounter(counters, "byte[]", "disjoint arraycopy call for byte[] arrays"); - private static final SnippetCounter charAlignedCallCounter = new SnippetCounter(counters, "char[]", "aligned arraycopy call for char[] arrays"); - private static final SnippetCounter charAlignedDisjointCallCounter = new SnippetCounter(counters, "char[]", "aligned disjoint arraycopy call for char[] arrays"); private static final SnippetCounter charCallCounter = new SnippetCounter(counters, "char[]", "arraycopy call for char[] arrays"); - private static final SnippetCounter charDisjointCallCounter = new SnippetCounter(counters, "char[]", "disjoint arraycopy call for char[] arrays"); - private static final SnippetCounter doubleAlignedCallCounter = new SnippetCounter(counters, "double[]", "aligned arraycopy call for double[] arrays"); - private static final SnippetCounter doubleAlignedDisjointCallCounter = new SnippetCounter(counters, "double[]", "aligned disjoint arraycopy call for double[] arrays"); private static final SnippetCounter doubleCallCounter = new SnippetCounter(counters, "double[]", "arraycopy call for double[] arrays"); - private static final SnippetCounter doubleDisjointCallCounter = new SnippetCounter(counters, "double[]", "disjoint arraycopy call for double[] arrays"); - private static final SnippetCounter floatAlignedCallCounter = new SnippetCounter(counters, "float[]", "aligned arraycopy call for float[] arrays"); - private static final SnippetCounter floatAlignedDisjointCallCounter = new SnippetCounter(counters, "float[]", "aligned disjoint arraycopy call for float[] arrays"); private static final SnippetCounter floatCallCounter = new SnippetCounter(counters, "float[]", "arraycopy call for float[] arrays"); - private static final SnippetCounter floatDisjointCallCounter = new SnippetCounter(counters, "float[]", "disjoint arraycopy call for float[] arrays"); - private static final SnippetCounter intAlignedCallCounter = new SnippetCounter(counters, "int[]", "aligned arraycopy call for int[] arrays"); - private static final SnippetCounter intAlignedDisjointCallCounter = new SnippetCounter(counters, "int[]", "aligned disjoint arraycopy call for int[] arrays"); private static final SnippetCounter intCallCounter = new SnippetCounter(counters, "int[]", "arraycopy call for int[] arrays"); - private static final SnippetCounter intDisjointCallCounter = new SnippetCounter(counters, "int[]", "disjoint arraycopy call for int[] arrays"); - private static final SnippetCounter longAlignedCallCounter = new SnippetCounter(counters, "long[]", "aligned arraycopy call for long[] arrays"); - private static final SnippetCounter longAlignedDisjointCallCounter = new SnippetCounter(counters, "long[]", "aligned disjoint arraycopy call for long[] arrays"); private static final SnippetCounter longCallCounter = new SnippetCounter(counters, "long[]", "arraycopy call for long[] arrays"); - private static final SnippetCounter longDisjointCallCounter = new SnippetCounter(counters, "long[]", "disjoint arraycopy call for long[] arrays"); - private static final SnippetCounter shortAlignedCallCounter = new SnippetCounter(counters, "short[]", "aligned arraycopy call for short[] arrays"); - private static final SnippetCounter shortAlignedDisjointCallCounter = new SnippetCounter(counters, "short[]", "aligned disjoint arraycopy call for short[] arrays"); private static final SnippetCounter shortCallCounter = new SnippetCounter(counters, "short[]", "arraycopy call for short[] arrays"); - private static final SnippetCounter shortDisjointCallCounter = new SnippetCounter(counters, "short[]", "disjoint arraycopy call for short[] arrays"); private static final SnippetCounter genericPrimitiveCallCounter = new SnippetCounter(counters, "genericPrimitive", "generic arraycopy snippet for primitive arrays"); private static final SnippetCounter genericObjectExactCallCounter = new SnippetCounter(counters, "genericObjectExact", "generic arraycopy snippet for special object arrays"); private static final SnippetCounter genericObjectCallCounter = new SnippetCounter(counters, "genericObject", "call to the generic, native arraycopy method"); - }
--- a/graal/com.oracle.graal.phases.common/src/com/oracle/graal/phases/common/InliningUtil.java Wed Feb 19 15:58:45 2014 +0100 +++ b/graal/com.oracle.graal.phases.common/src/com/oracle/graal/phases/common/InliningUtil.java Wed Feb 19 00:39:44 2014 -0800 @@ -1317,6 +1317,8 @@ FixedNode invokeNode = invoke.asNode(); StructuredGraph graph = invokeNode.graph(); assert inlineGraph.getGuardsStage().ordinal() >= graph.getGuardsStage().ordinal(); + assert !invokeNode.graph().isAfterFloatingReadPhase() : "inline isn't handled correctly after floating reads phase"; + Kind returnKind = invokeNode.kind(); FrameState stateAfter = invoke.stateAfter();
--- a/graal/com.oracle.graal.replacements/src/com/oracle/graal/replacements/ReplacementsImpl.java Wed Feb 19 15:58:45 2014 +0100 +++ b/graal/com.oracle.graal.replacements/src/com/oracle/graal/replacements/ReplacementsImpl.java Wed Feb 19 00:39:44 2014 -0800 @@ -200,7 +200,9 @@ } } } - if (macroSubstitution != null) { + // We don't have per method guards for macro substitutions but at least respect the + // defaultGuard if there is one. + if (macroSubstitution != null && (defaultGuard == null || defaultGuard.execute())) { String originalName = originalName(substituteMethod, macroSubstitution.value()); JavaSignature originalSignature = originalSignature(substituteMethod, macroSubstitution.signature(), macroSubstitution.isStatic()); Member originalMethod = originalMethod(classSubstitution, macroSubstitution.optional(), originalName, originalSignature);