# HG changeset patch # User Tom Rodriguez # Date 1436555053 25200 # Node ID 6f83839fcbc850dfeba6b466f4da2925738e3313 # Parent cbb24d44d09bc4a1accacd36ba9924b263735fea Add missing barrier in unrolled arraycopy diff -r cbb24d44d09b -r 6f83839fcbc8 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/DefaultHotSpotLoweringProvider.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/DefaultHotSpotLoweringProvider.java Fri Jul 10 11:41:02 2015 -0700 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/DefaultHotSpotLoweringProvider.java Fri Jul 10 12:04:13 2015 -0700 @@ -159,6 +159,8 @@ arraycopySnippets.lower((ArrayCopyNode) n, tool); } else if (n instanceof ArrayCopySlowPathNode) { arraycopySnippets.lower((ArrayCopySlowPathNode) n, tool); + } else if (n instanceof ArrayCopyUnrollNode) { + arraycopySnippets.lower((ArrayCopyUnrollNode) n, tool); } else if (n instanceof G1PreWriteBarrier) { writeBarrierSnippets.lower((G1PreWriteBarrier) n, registers, tool); } else if (n instanceof G1PostWriteBarrier) { diff -r cbb24d44d09b -r 6f83839fcbc8 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/ArrayCopySnippets.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/ArrayCopySnippets.java Fri Jul 10 11:41:02 2015 -0700 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/ArrayCopySnippets.java Fri Jul 10 12:04:13 2015 -0700 @@ -184,6 +184,23 @@ ArrayCopySlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind, slowPath, slowPathArgument); } + /** + * Snippet for unrolled arraycopy. + */ + @Snippet + public static void arraycopyUnrolledIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter int unrolledLength, @ConstantParameter Kind elementKind) { + Object nonNullSrc = GraalDirectives.guardingNonNull(src); + Object nonNullDest = GraalDirectives.guardingNonNull(dest); + checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); + if (length == 0) { + zeroLengthDynamicCounter.inc(); + } else { + nonZeroLengthDynamicCounter.inc(); + nonZeroLengthDynamicCopiedCounter.add(length); + } + ArrayCopyUnrollNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, unrolledLength, elementKind); + } + @Snippet public static void checkcastArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length) { if (length > 0) { @@ -358,6 +375,7 @@ private final SnippetInfo arraycopyGenericSnippet = snippet("arraycopyGeneric"); private final SnippetInfo arraycopySlowPathIntrinsicSnippet = snippet("arraycopySlowPathIntrinsic"); + private final SnippetInfo arraycopyUnrolledIntrinsicSnippet = snippet("arraycopyUnrolledIntrinsic"); private final SnippetInfo arraycopyExactIntrinsicSnippet = snippet("arraycopyExactIntrinsic"); private final SnippetInfo arraycopyZeroLengthIntrinsicSnippet = snippet("arraycopyZeroLengthIntrinsic"); private final SnippetInfo arraycopyPredictedExactIntrinsicSnippet = snippet("arraycopyPredictedExactIntrinsic"); @@ -422,9 +440,7 @@ } else if (arraycopy.isExact()) { snippetInfo = arraycopyExactIntrinsicSnippet; if (shouldUnroll(arraycopy.getLength())) { - snippetInfo = arraycopySlowPathIntrinsicSnippet; - slowPathSnippetInfo = arraycopyUnrolledWorkSnippet; - slowPathArgument = arraycopy.getLength().asJavaConstant().asInt(); + snippetInfo = arraycopyUnrolledIntrinsicSnippet; } } else { if (componentKind == Kind.Object) { @@ -472,7 +488,10 @@ args.add("dest", arraycopy.getDestination()); args.add("destPos", arraycopy.getDestinationPosition()); args.add("length", arraycopy.getLength()); - if (snippetInfo == arraycopySlowPathIntrinsicSnippet) { + if (snippetInfo == arraycopyUnrolledIntrinsicSnippet) { + args.addConst("unrolledLength", arraycopy.getLength().asJavaConstant().asInt()); + args.addConst("elementKind", componentKind != null ? componentKind : Kind.Illegal); + } else if (snippetInfo == arraycopySlowPathIntrinsicSnippet) { args.addConst("elementKind", componentKind != null ? componentKind : Kind.Illegal); args.addConst("slowPath", slowPathSnippetInfo); assert slowPathArgument != null; @@ -514,6 +533,23 @@ instantiate(args, arraycopy); } + public void lower(ArrayCopyUnrollNode arraycopy, LoweringTool tool) { + StructuredGraph graph = arraycopy.graph(); + if (!graph.getGuardsStage().areFrameStatesAtDeopts()) { + // Can't be lowered yet + return; + } + SnippetInfo snippetInfo = arraycopyUnrolledWorkSnippet; + Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage()); + args.add("nonNullSrc", arraycopy.getSource()); + args.add("srcPos", arraycopy.getSourcePosition()); + args.add("nonNullDest", arraycopy.getDestination()); + args.add("destPos", arraycopy.getDestinationPosition()); + args.addConst("length", arraycopy.getUnrollLength()); + args.addConst("elementKind", arraycopy.getElementKind()); + template(args).instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args); + } + /** * Instantiate the snippet template and fix up the FrameState of any Invokes of * System.arraycopy and propagate the captured bci in the ArrayCopySlowPathNode. diff -r cbb24d44d09b -r 6f83839fcbc8 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/ArrayCopyUnrollNode.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/ArrayCopyUnrollNode.java Fri Jul 10 12:04:13 2015 -0700 @@ -0,0 +1,138 @@ +/* + * Copyright (c) 2015, 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.graal.hotspot.replacements.arraycopy; + +import static jdk.internal.jvmci.meta.LocationIdentity.*; +import jdk.internal.jvmci.meta.*; + +import com.oracle.graal.compiler.common.type.*; +import com.oracle.graal.graph.*; +import com.oracle.graal.nodeinfo.*; +import com.oracle.graal.nodes.*; +import com.oracle.graal.nodes.extended.*; +import com.oracle.graal.nodes.memory.*; +import com.oracle.graal.nodes.spi.*; + +@NodeInfo +public class ArrayCopyUnrollNode extends ArrayRangeWriteNode implements MemoryCheckpoint.Single, Lowerable, MemoryAccess { + + public static final NodeClass TYPE = NodeClass.create(ArrayCopyUnrollNode.class); + + @Input protected ValueNode src; + @Input protected ValueNode srcPos; + @Input protected ValueNode dest; + @Input protected ValueNode destPos; + @Input protected ValueNode length; + + private Kind elementKind; + + private int unrolledLength; + + @OptionalInput(InputType.Memory) private MemoryNode lastLocationAccess; + + public ArrayCopyUnrollNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, int unrolledLength, Kind elementKind) { + super(TYPE, StampFactory.forKind(Kind.Void)); + this.src = src; + this.srcPos = srcPos; + this.dest = dest; + this.destPos = destPos; + this.length = length; + this.unrolledLength = unrolledLength; + assert elementKind != null && elementKind != Kind.Illegal; + this.elementKind = elementKind; + } + + public ValueNode getSource() { + return src; + } + + public ValueNode getSourcePosition() { + return srcPos; + } + + public ValueNode getDestination() { + return dest; + } + + public ValueNode getDestinationPosition() { + return destPos; + } + + @Override + public ValueNode getLength() { + return length; + } + + @Override + public ValueNode getArray() { + return dest; + } + + @Override + public ValueNode getIndex() { + return destPos; + } + + @Override + public boolean isObjectArray() { + return elementKind == Kind.Object; + } + + @Override + public boolean isInitialization() { + return false; + } + + @NodeIntrinsic + public static native void arraycopy(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, @ConstantNodeParameter int unrolledLength, @ConstantNodeParameter Kind elementKind); + + public int getUnrollLength() { + return unrolledLength; + } + + public Kind getElementKind() { + return elementKind; + } + + @Override + public LocationIdentity getLocationIdentity() { + if (elementKind != null) { + return NamedLocationIdentity.getArrayLocation(elementKind); + } + return any(); + } + + @Override + public void lower(LoweringTool tool) { + tool.getLowerer().lower(this, tool); + } + + public MemoryNode getLastLocationAccess() { + return lastLocationAccess; + } + + public void setLastLocationAccess(MemoryNode lla) { + updateUsagesInterface(lastLocationAccess, lla); + lastLocationAccess = lla; + } +}