Mercurial > hg > truffle
changeset 21463:b1072d72fa2e
Lower checkcast arraycopy idiom after frame state assignment and convert arraycopy lowering into snippets
line wrap: on
line diff
--- a/graal/com.oracle.graal.hotspot.test/src/com/oracle/graal/hotspot/test/ArrayCopyIntrinsificationTest.java Wed May 20 11:58:51 2015 -0700 +++ b/graal/com.oracle.graal.hotspot.test/src/com/oracle/graal/hotspot/test/ArrayCopyIntrinsificationTest.java Thu May 21 22:20:41 2015 -0700 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -87,12 +87,9 @@ @Test public void test0() { - mustIntrinsify = false; // a generic call to arraycopy will not be intrinsified // Array store checks test("genericArraycopy", new Object(), 0, new Object[0], 0, 0); test("genericArraycopy", new Object[0], 0, new Object(), 0, 0); - - mustIntrinsify = true; } @Test @@ -156,7 +153,7 @@ } /** - * Tests {@link ArrayCopySnippets#checkcastArraycopySnippet}. + * Tests {@link ArrayCopySnippets#checkcastArraycopyWork(Object, int, Object, int, int)}. */ @Test public void testArrayStoreException() { @@ -267,10 +264,8 @@ */ @Test public void testCopyRows() { - mustIntrinsify = false; Object[][] rows = {{"a1", "a2", "a3", "a4"}, {"b1", "b2", "b3", "b4"}, {"c1", "c2", "c3", "c4"}}; test("copyRows", rows, 4, new Integer(rows.length)); - mustIntrinsify = true; } public static Object[][] copyRows(Object[][] rows, int rowSize, Integer rowCount) {
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/DefaultHotSpotLoweringProvider.java Wed May 20 11:58:51 2015 -0700 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/DefaultHotSpotLoweringProvider.java Thu May 21 22:20:41 2015 -0700 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2014, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -68,6 +68,7 @@ protected LoadExceptionObjectSnippets.Templates exceptionObjectSnippets; protected UnsafeLoadSnippets.Templates unsafeLoadSnippets; protected AssertionSnippets.Templates assertionSnippets; + protected ArrayCopySnippets.Templates arraycopySnippets; public DefaultHotSpotLoweringProvider(HotSpotGraalRuntimeProvider runtime, MetaAccessProvider metaAccess, ForeignCallsProvider foreignCalls, HotSpotRegistersProvider registers, TargetDescription target) { @@ -89,6 +90,7 @@ exceptionObjectSnippets = new LoadExceptionObjectSnippets.Templates(providers, target); unsafeLoadSnippets = new UnsafeLoadSnippets.Templates(providers, target); assertionSnippets = new AssertionSnippets.Templates(providers, target); + arraycopySnippets = new ArrayCopySnippets.Templates(providers, target); providers.getReplacements().registerSnippetTemplateCache(new UnsafeArrayCopySnippets.Templates(providers, target)); } @@ -154,6 +156,12 @@ if (graph.getGuardsStage().areFrameStatesAtDeopts()) { monitorSnippets.lower((MonitorExitNode) n, tool); } + } else if (n instanceof ArrayCopyNode) { + arraycopySnippets.lower((ArrayCopyNode) n, tool); + } else if (n instanceof ArrayCopySlowPathNode) { + ArrayCopySlowPathNode slowpath = (ArrayCopySlowPathNode) n; + // FrameState stateAfter = slowpath.stateAfter(); + arraycopySnippets.lower(slowpath, tool); } else if (n instanceof G1PreWriteBarrier) { writeBarrierSnippets.lower((G1PreWriteBarrier) n, registers, tool); } else if (n instanceof G1PostWriteBarrier) {
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/HotSpotGraphBuilderPlugins.java Wed May 20 11:58:51 2015 -0700 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/HotSpotGraphBuilderPlugins.java Thu May 21 22:20:41 2015 -0700 @@ -190,7 +190,7 @@ }); r.register5("arraycopy", Object.class, int.class, Object.class, int.class, int.class, new InvocationPlugin() { public boolean apply(GraphBuilderContext b, ResolvedJavaMethod targetMethod, Receiver receiver, ValueNode src, ValueNode srcPos, ValueNode dst, ValueNode dstPos, ValueNode length) { - b.add(new ArrayCopyNode(b.getInvokeKind(), targetMethod, b.bci(), b.getInvokeReturnType(), src, srcPos, dst, dstPos, length)); + b.add(new ArrayCopyNode(b.bci(), src, srcPos, dst, dstPos, length)); return true; }
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/HotSpotHostForeignCallsProvider.java Wed May 20 11:58:51 2015 -0700 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/HotSpotHostForeignCallsProvider.java Thu May 21 22:20:41 2015 -0700 @@ -71,11 +71,16 @@ return checkcastArraycopyDescriptors[uninit ? 1 : 0]; } - public static ForeignCallDescriptor lookupArraycopyDescriptor(Kind kind, boolean aligned, boolean disjoint, boolean uninit) { + public static ForeignCallDescriptor lookupArraycopyDescriptor(Kind kind, boolean aligned, boolean disjoint, boolean uninit, boolean killAny) { if (uninit) { assert kind == Kind.Object; + assert !killAny : "unsupported"; return uninitObjectArraycopyDescriptors[aligned ? 1 : 0][disjoint ? 1 : 0]; } + if (killAny) { + assert kind == Kind.Object; + return objectArraycopyDescriptorsKillAny[aligned ? 1 : 0][disjoint ? 1 : 0]; + } return arraycopyDescriptors[aligned ? 1 : 0][disjoint ? 1 : 0].get(kind); } @@ -83,6 +88,7 @@ private static final ForeignCallDescriptor[][] uninitObjectArraycopyDescriptors = new ForeignCallDescriptor[2][2]; private static final ForeignCallDescriptor[] checkcastArraycopyDescriptors = new ForeignCallDescriptor[2]; + private static ForeignCallDescriptor[][] objectArraycopyDescriptorsKillAny = new ForeignCallDescriptor[2][2]; static { // Populate the EnumMap instances @@ -93,13 +99,12 @@ } } - private void registerArraycopyDescriptor(Map<Long, ForeignCallDescriptor> descMap, Kind kind, boolean aligned, boolean disjoint, boolean uninit, long routine) { + private void registerArraycopyDescriptor(Map<Long, ForeignCallDescriptor> descMap, Kind kind, boolean aligned, boolean disjoint, boolean uninit, boolean killAny, long routine) { ForeignCallDescriptor desc = descMap.get(routine); - String name = kind + (aligned ? "Aligned" : "") + (disjoint ? "Disjoint" : "") + (uninit ? "Uninit" : "") + "Arraycopy"; - desc = new ForeignCallDescriptor(name, void.class, Word.class, Word.class, Word.class); - LocationIdentity killed = NamedLocationIdentity.getArrayLocation(kind); - registerForeignCall(desc, routine, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, NOT_REEXECUTABLE, killed); - descMap.put(routine, desc); + if (desc == null) { + desc = buildDescriptor(kind, aligned, disjoint, uninit, killAny, routine); + descMap.put(routine, desc); + } if (uninit) { assert kind == Kind.Object; uninitObjectArraycopyDescriptors[aligned ? 1 : 0][disjoint ? 1 : 0] = desc; @@ -108,6 +113,15 @@ } } + private ForeignCallDescriptor buildDescriptor(Kind kind, boolean aligned, boolean disjoint, boolean uninit, boolean killAny, long routine) { + assert !killAny || kind == Kind.Object; + String name = kind + (aligned ? "Aligned" : "") + (disjoint ? "Disjoint" : "") + (uninit ? "Uninit" : "") + "Arraycopy" + (killAny ? "KillAny" : ""); + ForeignCallDescriptor desc = new ForeignCallDescriptor(name, void.class, Word.class, Word.class, Word.class); + LocationIdentity killed = killAny ? LocationIdentity.any() : NamedLocationIdentity.getArrayLocation(kind); + registerForeignCall(desc, routine, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, NOT_REEXECUTABLE, killed); + return desc; + } + private void registerCheckcastArraycopyDescriptor(boolean uninit, long routine) { String name = "Object" + (uninit ? "Uninit" : "") + "Checkcast"; // Input: @@ -134,10 +148,17 @@ * they kill different memory so they still have to be distinct. */ Map<Long, ForeignCallDescriptor> descMap = new HashMap<>(); - registerArraycopyDescriptor(descMap, kind, false, false, uninit, routine); - registerArraycopyDescriptor(descMap, kind, true, false, uninit, alignedRoutine); - registerArraycopyDescriptor(descMap, kind, false, true, uninit, disjointRoutine); - registerArraycopyDescriptor(descMap, kind, true, true, uninit, alignedDisjointRoutine); + registerArraycopyDescriptor(descMap, kind, false, false, uninit, false, routine); + registerArraycopyDescriptor(descMap, kind, true, false, uninit, false, alignedRoutine); + registerArraycopyDescriptor(descMap, kind, false, true, uninit, false, disjointRoutine); + registerArraycopyDescriptor(descMap, kind, true, true, uninit, false, alignedDisjointRoutine); + + if (kind == Kind.Object && !uninit) { + objectArraycopyDescriptorsKillAny[0][0] = buildDescriptor(kind, false, false, uninit, true, routine); + objectArraycopyDescriptorsKillAny[1][0] = buildDescriptor(kind, true, false, uninit, true, alignedRoutine); + objectArraycopyDescriptorsKillAny[0][1] = buildDescriptor(kind, false, true, uninit, true, disjointRoutine); + objectArraycopyDescriptorsKillAny[1][1] = buildDescriptor(kind, true, true, uninit, true, alignedDisjointRoutine); + } } public void initialize(HotSpotProviders providers, HotSpotVMConfig c) {
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/ArrayCopyCallNode.java Wed May 20 11:58:51 2015 -0700 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/ArrayCopyCallNode.java Thu May 21 22:20:41 2015 -0700 @@ -23,12 +23,11 @@ //JaCoCo Exclude package com.oracle.graal.hotspot.replacements.arraycopy; -import static com.oracle.graal.api.meta.LocationIdentity.*; - import com.oracle.graal.api.meta.*; import com.oracle.graal.api.runtime.*; import com.oracle.graal.compiler.common.type.*; import com.oracle.graal.graph.*; +import com.oracle.graal.graph.spi.*; import com.oracle.graal.hotspot.*; import com.oracle.graal.hotspot.meta.*; import com.oracle.graal.hotspot.nodes.*; @@ -40,16 +39,19 @@ import com.oracle.graal.runtime.*; @NodeInfo(allowedUsageTypes = {InputType.Memory}) -public final class ArrayCopyCallNode extends AbstractMemoryCheckpoint implements Lowerable, MemoryCheckpoint.Single { +public final class ArrayCopyCallNode extends AbstractMemoryCheckpoint implements Lowerable, MemoryCheckpoint.Single, MemoryAccess, Canonicalizable { public static final NodeClass<ArrayCopyCallNode> TYPE = NodeClass.create(ArrayCopyCallNode.class); - @Input ValueNode src; - @Input ValueNode srcPos; - @Input ValueNode dest; - @Input ValueNode destPos; - @Input ValueNode length; + @Input protected ValueNode src; + @Input protected ValueNode srcPos; + @Input protected ValueNode dest; + @Input protected ValueNode destPos; + @Input protected ValueNode length; - protected Kind elementKind; + @OptionalInput(InputType.Memory) MemoryNode lastLocationAccess; + + protected final Kind elementKind; + protected final LocationIdentity locationIdentity; /** * Aligned means that the offset of the copy is heap word aligned. @@ -62,16 +64,16 @@ public ArrayCopyCallNode(@InjectedNodeParameter HotSpotGraalRuntimeProvider runtime, ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, Kind elementKind, boolean aligned, boolean disjoint, boolean uninitialized) { - this(src, srcPos, dest, destPos, length, elementKind, aligned, disjoint, uninitialized, runtime); + this(runtime, src, srcPos, dest, destPos, length, elementKind, null, aligned, disjoint, uninitialized); } public ArrayCopyCallNode(@InjectedNodeParameter HotSpotGraalRuntimeProvider runtime, ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, Kind elementKind, boolean disjoint) { - this(src, srcPos, dest, destPos, length, elementKind, false, disjoint, false, runtime); + this(runtime, src, srcPos, dest, destPos, length, elementKind, null, false, disjoint, false); } - protected ArrayCopyCallNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, Kind elementKind, boolean aligned, boolean disjoint, boolean uninitialized, - HotSpotGraalRuntimeProvider runtime) { + protected ArrayCopyCallNode(@InjectedNodeParameter HotSpotGraalRuntimeProvider runtime, ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, Kind elementKind, + LocationIdentity locationIdentity, boolean aligned, boolean disjoint, boolean uninitialized) { super(TYPE, StampFactory.forVoid()); assert elementKind != null; this.src = src; @@ -80,6 +82,7 @@ this.destPos = destPos; this.length = length; this.elementKind = elementKind; + this.locationIdentity = (locationIdentity != null ? locationIdentity : NamedLocationIdentity.getArrayLocation(elementKind)); this.aligned = aligned; this.disjoint = disjoint; this.uninitialized = uninitialized; @@ -121,7 +124,8 @@ public void lower(LoweringTool tool) { if (graph().getGuardsStage().areFrameStatesAtDeopts()) { updateAlignedDisjoint(); - ForeignCallDescriptor desc = HotSpotHostForeignCallsProvider.lookupArraycopyDescriptor(elementKind, isAligned(), isDisjoint(), isUninitialized()); + ForeignCallDescriptor desc = HotSpotHostForeignCallsProvider.lookupArraycopyDescriptor(elementKind, isAligned(), isDisjoint(), isUninitialized(), + locationIdentity.equals(LocationIdentity.any())); StructuredGraph graph = graph(); ValueNode srcAddr = computeBase(getSource(), getSourcePosition()); ValueNode destAddr = computeBase(getDestination(), getDestinationPosition()); @@ -132,24 +136,38 @@ ForeignCallNode call = graph.add(new ForeignCallNode(Graal.getRequiredCapability(RuntimeProvider.class).getHostBackend().getForeignCalls(), desc, srcAddr, destAddr, len)); call.setStateAfter(stateAfter()); graph.replaceFixedWithFixed(this, call); + } + } - } + public MemoryNode getLastLocationAccess() { + return lastLocationAccess; + } + + public void setLastLocationAccess(MemoryNode lla) { + updateUsagesInterface(lastLocationAccess, lla); + lastLocationAccess = lla; } @Override public LocationIdentity getLocationIdentity() { - if (elementKind != null) { - return NamedLocationIdentity.getArrayLocation(elementKind); - } - return any(); + return locationIdentity; } @NodeIntrinsic private static native void arraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter Kind elementKind, @ConstantNodeParameter boolean aligned, @ConstantNodeParameter boolean disjoint, @ConstantNodeParameter boolean uninitialized); - public static void arraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter Kind elementKind, boolean aligned, boolean disjoint) { - arraycopy(src, srcPos, dest, destPos, length, elementKind, aligned, disjoint, false); + @NodeIntrinsic + private static native void arraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter Kind elementKind, + @ConstantNodeParameter LocationIdentity locationIdentity, @ConstantNodeParameter boolean aligned, @ConstantNodeParameter boolean disjoint, + @ConstantNodeParameter boolean uninitialized); + + public static void arraycopyObjectKillsAny(Object src, int srcPos, Object dest, int destPos, int length) { + arraycopy(src, srcPos, dest, destPos, length, Kind.Object, LocationIdentity.any(), false, false, false); + } + + public static void arraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter Kind elementKind) { + arraycopy(src, srcPos, dest, destPos, length, elementKind, false, false, false); } public static void disjointArraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter Kind elementKind) { @@ -194,4 +212,15 @@ } } } + + @Override + public Node canonical(CanonicalizerTool tool) { + if (getLength().isConstant() && getLength().asConstant().isDefaultForKind()) { + if (lastLocationAccess != null) { + replaceAtUsages(InputType.Memory, lastLocationAccess.asNode()); + } + return null; + } + return this; + } }
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/ArrayCopyNode.java Wed May 20 11:58:51 2015 -0700 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/ArrayCopyNode.java Thu May 21 22:20:41 2015 -0700 @@ -22,19 +22,13 @@ */ package com.oracle.graal.hotspot.replacements.arraycopy; +import static com.oracle.graal.api.meta.LocationIdentity.*; + import com.oracle.graal.api.meta.*; -import com.oracle.graal.compiler.common.*; -import com.oracle.graal.debug.*; -import com.oracle.graal.debug.Debug.Scope; import com.oracle.graal.graph.*; -import com.oracle.graal.loop.phases.*; import com.oracle.graal.nodeinfo.*; -import com.oracle.graal.nodes.CallTargetNode.InvokeKind; import com.oracle.graal.nodes.*; import com.oracle.graal.nodes.spi.*; -import com.oracle.graal.nodes.type.*; -import com.oracle.graal.phases.common.*; -import com.oracle.graal.phases.tiers.*; import com.oracle.graal.replacements.nodes.*; @NodeInfo @@ -42,101 +36,26 @@ public static final NodeClass<ArrayCopyNode> TYPE = NodeClass.create(ArrayCopyNode.class); - public ArrayCopyNode(InvokeKind invokeKind, ResolvedJavaMethod targetMethod, int bci, JavaType returnType, ValueNode src, ValueNode srcPos, ValueNode dst, ValueNode dstPos, ValueNode length) { - super(TYPE, invokeKind, targetMethod, bci, returnType, src, srcPos, dst, dstPos, length); - } - - private StructuredGraph selectSnippet(LoweringTool tool, final Replacements replacements) { - ResolvedJavaType srcType = StampTool.typeOrNull(getSource().stamp()); - ResolvedJavaType destType = StampTool.typeOrNull(getDestination().stamp()); + private Kind elementKind; - if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) { - return null; - } - if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) { - return null; - } - if (!isExact()) { - return null; - } - Kind componentKind = srcType.getComponentType().getKind(); - final ResolvedJavaMethod snippetMethod = tool.getMetaAccess().lookupJavaMethod(ArrayCopySnippets.getSnippetForKind(componentKind, shouldUnroll(), isExact())); - try (Scope s = Debug.scope("ArrayCopySnippet", snippetMethod)) { - return replacements.getSnippet(snippetMethod, null, null); - } catch (Throwable e) { - throw Debug.handle(e); - } - } - - private static void unrollFixedLengthLoop(StructuredGraph snippetGraph, int length, LoweringTool tool) { - ParameterNode lengthParam = snippetGraph.getParameter(4); - if (lengthParam != null) { - snippetGraph.replaceFloating(lengthParam, ConstantNode.forInt(length, snippetGraph)); - } - // the canonicalization before loop unrolling is needed to propagate the length into - // additions, etc. - PhaseContext context = new PhaseContext(tool.getMetaAccess(), tool.getConstantReflection(), tool.getLowerer(), tool.getReplacements(), tool.getStampProvider()); - new CanonicalizerPhase().apply(snippetGraph, context); - new LoopFullUnrollPhase(new CanonicalizerPhase()).apply(snippetGraph, context); - new CanonicalizerPhase().apply(snippetGraph, context); + public ArrayCopyNode(int bci, ValueNode src, ValueNode srcPos, ValueNode dst, ValueNode dstPos, ValueNode length) { + super(TYPE, src, srcPos, dst, dstPos, length, null, bci); + elementKind = ArrayCopySnippets.Templates.selectComponentKind(this); } @Override - protected StructuredGraph getLoweredSnippetGraph(final LoweringTool tool) { - final Replacements replacements = tool.getReplacements(); - StructuredGraph snippetGraph = selectSnippet(tool, replacements); - if (snippetGraph == null) { - ResolvedJavaType srcType = StampTool.typeOrNull(getSource().stamp()); - ResolvedJavaType destType = StampTool.typeOrNull(getDestination().stamp()); - ResolvedJavaType srcComponentType = srcType == null ? null : srcType.getComponentType(); - ResolvedJavaType destComponentType = destType == null ? null : destType.getComponentType(); - ResolvedJavaMethod snippetMethod = null; - if (srcComponentType != null && destComponentType != null && !srcComponentType.isPrimitive() && !destComponentType.isPrimitive()) { - snippetMethod = tool.getMetaAccess().lookupJavaMethod(ArrayCopySnippets.checkcastArraycopySnippet); - } else { - snippetMethod = tool.getMetaAccess().lookupJavaMethod(ArrayCopySnippets.genericArraycopySnippet); - } - snippetGraph = null; - try (Scope s = Debug.scope("ArrayCopySnippet", snippetMethod)) { - snippetGraph = replacements.getSnippet(snippetMethod, getTargetMethod(), null).copy(); - } catch (Throwable e) { - throw Debug.handle(e); - } - replaceSnippetInvokes(snippetGraph); - } else { - assert snippetGraph != null : "ArrayCopySnippets should be installed"; - snippetGraph = snippetGraph.copy(); - if (shouldUnroll()) { - final StructuredGraph copy = snippetGraph; - try (Scope s = Debug.scope("ArrayCopySnippetSpecialization", snippetGraph.method())) { - unrollFixedLengthLoop(copy, getLength().asJavaConstant().asInt(), tool); - } catch (Throwable e) { - throw Debug.handle(e); - } - } + public LocationIdentity getLocationIdentity() { + if (elementKind == null) { + elementKind = ArrayCopySnippets.Templates.selectComponentKind(this); } - return lowerReplacement(snippetGraph, tool); + if (elementKind != null) { + return NamedLocationIdentity.getArrayLocation(elementKind); + } + return any(); } - private boolean shouldUnroll() { - return getLength().isConstant() && getLength().asJavaConstant().asInt() <= GraalOptions.MaximumEscapeAnalysisArrayLength.getValue(); - } - - /* - * Returns true if this copy doesn't require store checks. Trivially true for primitive arrays. - */ - private boolean isExact() { - ResolvedJavaType srcType = StampTool.typeOrNull(getSource().stamp()); - if (srcType.getComponentType().getKind().isPrimitive() || getSource() == getDestination()) { - return true; - } - - ResolvedJavaType destType = StampTool.typeOrNull(getDestination().stamp()); - if (StampTool.isExactType(getDestination().stamp())) { - if (destType != null && destType.isAssignableFrom(srcType)) { - return true; - } - } - return false; + @Override + public void lower(LoweringTool tool) { + tool.getLowerer().lower(this, tool); } }
--- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/ArrayCopySlowPathNode.java Thu May 21 22:20:41 2015 -0700 @@ -0,0 +1,68 @@ +/* + * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.graal.hotspot.replacements.arraycopy; + +import static com.oracle.graal.api.meta.LocationIdentity.*; + +import com.oracle.graal.api.code.*; +import com.oracle.graal.api.meta.*; +import com.oracle.graal.graph.*; +import com.oracle.graal.nodeinfo.*; +import com.oracle.graal.nodes.*; +import com.oracle.graal.nodes.type.*; +import com.oracle.graal.replacements.*; +import com.oracle.graal.replacements.nodes.*; + +@NodeInfo +public final class ArrayCopySlowPathNode extends BasicArrayCopyNode { + + public static final NodeClass<ArrayCopySlowPathNode> TYPE = NodeClass.create(ArrayCopySlowPathNode.class); + + private final SnippetTemplate.SnippetInfo snippet; + + public ArrayCopySlowPathNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, Kind elementKind, SnippetTemplate.SnippetInfo snippet) { + super(TYPE, src, srcPos, dest, destPos, length, elementKind, BytecodeFrame.INVALID_FRAMESTATE_BCI); + assert StampTool.isPointerNonNull(src) && StampTool.isPointerNonNull(dest) : "must have been null checked"; + this.snippet = snippet; + } + + @NodeIntrinsic + public static native void arraycopy(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, @ConstantNodeParameter Kind elementKind, + @ConstantNodeParameter SnippetTemplate.SnippetInfo snippet); + + public SnippetTemplate.SnippetInfo getSnippet() { + return snippet; + } + + @Override + public LocationIdentity getLocationIdentity() { + if (elementKind != null) { + return NamedLocationIdentity.getArrayLocation(elementKind); + } + return any(); + } + + public void setBci(int bci) { + this.bci = bci; + } +}
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/ArrayCopySnippets.java Wed May 20 11:58:51 2015 -0700 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/ArrayCopySnippets.java Thu May 21 22:20:41 2015 -0700 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2011, 2014, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -30,74 +30,33 @@ import java.lang.reflect.*; import java.util.*; +import com.oracle.graal.api.code.*; import com.oracle.graal.api.meta.*; +import com.oracle.graal.api.replacements.*; import com.oracle.graal.compiler.common.*; -import com.oracle.graal.graph.Node.ConstantNodeParameter; -import com.oracle.graal.graph.Node.NodeIntrinsic; +import com.oracle.graal.debug.*; +import com.oracle.graal.debug.Debug.Scope; +import com.oracle.graal.graph.*; +import com.oracle.graal.hotspot.meta.*; +import com.oracle.graal.hotspot.nodes.type.*; import com.oracle.graal.hotspot.word.*; +import com.oracle.graal.loop.phases.*; import com.oracle.graal.nodes.*; import com.oracle.graal.nodes.extended.*; import com.oracle.graal.nodes.java.*; +import com.oracle.graal.nodes.spi.*; +import com.oracle.graal.nodes.type.*; +import com.oracle.graal.phases.common.*; +import com.oracle.graal.phases.tiers.*; import com.oracle.graal.replacements.*; +import com.oracle.graal.replacements.Snippet.ConstantParameter; +import com.oracle.graal.replacements.SnippetTemplate.Arguments; +import com.oracle.graal.replacements.SnippetTemplate.SnippetInfo; +import com.oracle.graal.replacements.nodes.*; import com.oracle.graal.word.*; public class ArrayCopySnippets implements Snippets { - private static final EnumMap<Kind, Method> arraycopyMethods = new EnumMap<>(Kind.class); - private static final EnumMap<Kind, Method> arraycopyCalls = new EnumMap<>(Kind.class); - - public static final Method checkcastArraycopySnippet; - public static final Method genericArraycopySnippet; - - private static void addArraycopySnippetMethod(Kind kind, Class<?> arrayClass) throws NoSuchMethodException { - arraycopyMethods.put(kind, ArrayCopySnippets.class.getDeclaredMethod("arraycopy", arrayClass, int.class, arrayClass, int.class, int.class)); - if (CallArrayCopy.getValue()) { - if (kind == Kind.Object) { - arraycopyCalls.put(kind, ArrayCopySnippets.class.getDeclaredMethod("objectArraycopyUnchecked", arrayClass, int.class, arrayClass, int.class, int.class)); - } else { - arraycopyCalls.put(kind, ArrayCopySnippets.class.getDeclaredMethod(kind + "Arraycopy", arrayClass, int.class, arrayClass, int.class, int.class)); - } - } - } - - static { - try { - addArraycopySnippetMethod(Kind.Byte, byte[].class); - addArraycopySnippetMethod(Kind.Boolean, boolean[].class); - addArraycopySnippetMethod(Kind.Char, char[].class); - addArraycopySnippetMethod(Kind.Short, short[].class); - addArraycopySnippetMethod(Kind.Int, int[].class); - addArraycopySnippetMethod(Kind.Long, long[].class); - addArraycopySnippetMethod(Kind.Float, float[].class); - addArraycopySnippetMethod(Kind.Double, double[].class); - addArraycopySnippetMethod(Kind.Object, Object[].class); - checkcastArraycopySnippet = ArrayCopySnippets.class.getDeclaredMethod("checkcastArraycopy", Object[].class, int.class, Object[].class, int.class, int.class); - genericArraycopySnippet = ArrayCopySnippets.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class); - } catch (SecurityException | NoSuchMethodException e) { - throw new GraalInternalError(e); - } - } - - public static Method getSnippetForKind(Kind kind, boolean shouldUnroll, boolean exact) { - Method m = null; - if (!shouldUnroll && exact) { - // use hotspot stubs - m = arraycopyCalls.get(kind); - if (m != null) { - return m; - } - } - // use snippets - return arraycopyMethods.get(kind); - } - - private static void checkedCopy(Object src, int srcPos, Object dest, int destPos, int length, Kind baseKind) { - Object nonNullSrc = guardingNonNull(src); - Object nonNullDest = guardingNonNull(dest); - checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); - UnsafeArrayCopyNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, baseKind); - } - private static int checkArrayType(KlassPointer hub) { int layoutHelper = readLayoutHelper(hub); if (probability(SLOW_PATH_PROBABILITY, layoutHelper >= 0)) { @@ -131,91 +90,119 @@ } @Snippet - public static void arraycopy(byte[] src, int srcPos, byte[] dest, int destPos, int length) { - byteCounter.inc(); - checkedCopy(src, srcPos, dest, destPos, length, Kind.Byte); - } - - @Snippet - public static void arraycopy(boolean[] src, int srcPos, boolean[] dest, int destPos, int length) { - booleanCounter.inc(); - checkedCopy(src, srcPos, dest, destPos, length, Kind.Boolean); - } - - @Snippet - public static void arraycopy(char[] src, int srcPos, char[] dest, int destPos, int length) { - charCounter.inc(); - checkedCopy(src, srcPos, dest, destPos, length, Kind.Char); - } - - @Snippet - public static void arraycopy(short[] src, int srcPos, short[] dest, int destPos, int length) { - shortCounter.inc(); - checkedCopy(src, srcPos, dest, destPos, length, Kind.Short); - } - - @Snippet - public static void arraycopy(int[] src, int srcPos, int[] dest, int destPos, int length) { - intCounter.inc(); - checkedCopy(src, srcPos, dest, destPos, length, Kind.Int); - } - - @Snippet - public static void arraycopy(float[] src, int srcPos, float[] dest, int destPos, int length) { - floatCounter.inc(); - checkedCopy(src, srcPos, dest, destPos, length, Kind.Float); + public static void arraycopyZeroLengthIntrinsic(Object src, int srcPos, Object dest, int destPos, int length) { + Object nonNullSrc = guardingNonNull(src); + Object nonNullDest = guardingNonNull(dest); + KlassPointer srcHub = loadHub(nonNullSrc); + KlassPointer destHub = loadHub(nonNullDest); + checkArrayType(srcHub); + checkArrayType(destHub); + checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); + zeroLengthStaticCounter.inc(); } @Snippet - public static void arraycopy(long[] src, int srcPos, long[] dest, int destPos, int length) { - longCounter.inc(); - checkedCopy(src, srcPos, dest, destPos, length, Kind.Long); - } - - @Snippet - public static void arraycopy(double[] src, int srcPos, double[] dest, int destPos, int length) { - doubleCounter.inc(); - checkedCopy(src, srcPos, dest, destPos, length, Kind.Double); - } - - @Snippet - public static void arraycopy(Object[] src, int srcPos, Object[] dest, int destPos, int length) { - objectCounter.inc(); - checkedCopy(src, srcPos, dest, destPos, length, Kind.Object); - } - - @Snippet - public static void checkcastArraycopy(Object[] src, int srcPos, Object[] dest, int destPos, int length) { - objectCheckcastCounter.inc(); + public static void arraycopyExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Kind elementKind, @ConstantParameter SnippetCounter counter) { Object nonNullSrc = guardingNonNull(src); Object nonNullDest = guardingNonNull(dest); checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); - if (probability(SLOW_PATH_PROBABILITY, nonNullSrc == nonNullDest)) { - // no storecheck required. - ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, Kind.Object, false, false); + counter.inc(); + ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind); + if (length == 0) { + zeroLengthDynamicCounter.inc(); } else { - KlassPointer destKlass = loadHub(nonNullDest); - KlassPointer destElemKlass = destKlass.readKlassPointer(arrayClassElementOffset(), OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION); - checkcastArraycopyHelper(srcPos, destPos, length, nonNullSrc, nonNullDest, destElemKlass); + nonZeroLengthDynamicCounter.inc(); } } - private static void checkcastArraycopyHelper(int srcPos, int destPos, int length, Object nonNullSrc, Object nonNullDest, KlassPointer destElemKlass) { - Word superCheckOffset = Word.signed(destElemKlass.readInt(superCheckOffsetOffset(), KLASS_SUPER_CHECK_OFFSET_LOCATION)); - int copiedElements = CheckcastArrayCopyCallNode.checkcastArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, superCheckOffset, destElemKlass, false); - if (copiedElements != 0) { - // the checkcast stub doesn't throw the ArrayStoreException, but returns the number of - // copied elements (xor'd with -1). - copiedElements ^= -1; - System.arraycopy(nonNullSrc, srcPos + copiedElements, nonNullDest, destPos + copiedElements, length - copiedElements); + /** + * This intrinsic is useful for the case where we know something statically about one of the + * inputs but not the other. + */ + @Snippet + public static void arraycopyPredictedExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Kind elementKind, @ConstantParameter SnippetCounter counter) { + Object nonNullSrc = guardingNonNull(src); + Object nonNullDest = guardingNonNull(dest); + KlassPointer srcHub = loadHub(nonNullSrc); + KlassPointer destHub = loadHub(nonNullDest); + if (probability(SLOW_PATH_PROBABILITY, srcHub != destHub)) { + DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); + } + checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); + counter.inc(); + ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind); + if (length == 0) { + zeroLengthDynamicCounter.inc(); } else { - // Capture an after state for this path. - ArrayCopyStateNode.captureState(); + nonZeroLengthDynamicCounter.inc(); } } @Snippet - public static void arraycopy(Object src, int srcPos, Object dest, int destPos, int length) { + public static void arraycopyPredictedObjectWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, KlassPointer objectArrayKlass, @ConstantParameter SnippetCounter counter) { + if (length > 0) { + KlassPointer srcHub = loadHub(nonNullSrc); + KlassPointer destHub = loadHub(nonNullDest); + if (probability(FAST_PATH_PROBABILITY, srcHub == destHub || destHub == objectArrayKlass)) { + counter.inc(); + predictedObjectArrayCopyFastPathCounter.inc(); + ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length); + } else { + predictedObjectArrayCopySlowPathCounter.inc(); + System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length); + } + } + } + + /** + * This is the basic template for the full arraycopy checks, including a check that the + * underlying type is really an array type. + */ + @Snippet + public static void arraycopySlowPathIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Kind elementKind, @ConstantParameter SnippetInfo slowPath) { + Object nonNullSrc = guardingNonNull(src); + Object nonNullDest = guardingNonNull(dest); + KlassPointer srcHub = loadHub(nonNullSrc); + KlassPointer destHub = loadHub(nonNullDest); + checkArrayType(srcHub); + checkArrayType(destHub); + checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); + if (length == 0) { + zeroLengthDynamicCounter.inc(); + } else { + nonZeroLengthDynamicCounter.inc(); + } + ArrayCopySlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind, slowPath); + } + + @Snippet + public static void checkcastArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length) { + if (length > 0) { + KlassPointer destKlass = loadHub(nonNullDest); + KlassPointer srcKlass = loadHub(nonNullSrc); + if (probability(SLOW_PATH_PROBABILITY, srcKlass == destKlass)) { + // no storecheck required. + objectCheckcastSameTypeCounter.inc(); + ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length); + } else { + KlassPointer destElemKlass = destKlass.readKlassPointer(arrayClassElementOffset(), OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION); + Word superCheckOffset = Word.signed(destElemKlass.readInt(superCheckOffsetOffset(), KLASS_SUPER_CHECK_OFFSET_LOCATION)); + objectCheckcastCounter.inc(); + int copiedElements = CheckcastArrayCopyCallNode.checkcastArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, superCheckOffset, destElemKlass, false); + if (copiedElements != 0) { + /* + * the checkcast stub doesn't throw the ArrayStoreException, but returns the + * number of copied elements (xor'd with -1). + */ + copiedElements ^= -1; + System.arraycopy(nonNullSrc, srcPos + copiedElements, nonNullDest, destPos + copiedElements, length - copiedElements); + } + } + } + } + + @Snippet + public static void arraycopyGeneric(Object src, int srcPos, Object dest, int destPos, int length) { Object nonNullSrc = guardingNonNull(src); Object nonNullDest = guardingNonNull(dest); KlassPointer srcHub = loadHub(nonNullSrc); @@ -226,102 +213,326 @@ checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); if (probability(FAST_PATH_PROBABILITY, isObjectArray)) { genericObjectExactCallCounter.inc(); - UnsafeArrayCopyNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, Kind.Object); + ArrayCopyCallNode.disjointArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, Kind.Object); } else { genericPrimitiveCallCounter.inc(); UnsafeArrayCopyNode.arraycopyPrimitive(nonNullSrc, srcPos, nonNullDest, destPos, length, layoutHelper); } } else { - genericObjectCallCounter.inc(); + SystemArraycopyCounter.inc(); System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length); } } - @NodeIntrinsic(ForeignCallNode.class) - public static native void callArraycopy(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word src, Word dest, Word len); - - private static void callArraycopyTemplate(SnippetCounter counter, Kind kind, boolean aligned, boolean disjoint, Object src, int srcPos, Object dest, int destPos, int length) { - counter.inc(); - Object nonNullSrc = guardingNonNull(src); - Object nonNullDest = guardingNonNull(dest); - checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); - ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, kind, aligned, disjoint); - } - - @Snippet - public static void objectArraycopyUnchecked(Object[] src, int srcPos, Object[] dest, int destPos, int length) { - callArraycopyTemplate(objectCallCounter, Kind.Object, false, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void byteArraycopy(byte[] src, int srcPos, byte[] dest, int destPos, int length) { - callArraycopyTemplate(byteCallCounter, Kind.Byte, false, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void booleanArraycopy(boolean[] src, int srcPos, boolean[] dest, int destPos, int length) { - callArraycopyTemplate(booleanCallCounter, Kind.Boolean, false, false, src, srcPos, dest, destPos, length); + @Fold + private static LocationIdentity getArrayLocation(Kind kind) { + return NamedLocationIdentity.getArrayLocation(kind); } @Snippet - public static void charArraycopy(char[] src, int srcPos, char[] dest, int destPos, int length) { - callArraycopyTemplate(charCallCounter, Kind.Char, false, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void shortArraycopy(short[] src, int srcPos, short[] dest, int destPos, int length) { - callArraycopyTemplate(shortCallCounter, Kind.Short, false, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void intArraycopy(int[] src, int srcPos, int[] dest, int destPos, int length) { - callArraycopyTemplate(intCallCounter, Kind.Int, false, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void floatArraycopy(float[] src, int srcPos, float[] dest, int destPos, int length) { - callArraycopyTemplate(floatCallCounter, Kind.Float, false, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void longArraycopy(long[] src, int srcPos, long[] dest, int destPos, int length) { - callArraycopyTemplate(longCallCounter, Kind.Long, false, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void doubleArraycopy(double[] src, int srcPos, double[] dest, int destPos, int length) { - callArraycopyTemplate(doubleCallCounter, Kind.Double, false, false, src, srcPos, dest, destPos, length); + public static void arraycopyUnrolledWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, @ConstantParameter int length, @ConstantParameter Kind elementKind) { + final int scale = arrayIndexScale(elementKind); + int arrayBaseOffset = arrayBaseOffset(elementKind); + LocationIdentity arrayLocation = getArrayLocation(elementKind); + if (nonNullSrc == nonNullDest && srcPos < destPos) { // bad aliased case + long start = (long) (length - 1) * scale; + long i = start; + ExplodeLoopNode.explodeLoop(); + for (int iteration = 0; iteration < length; iteration++) { + if (i >= 0) { + Object a = UnsafeLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation); + DirectObjectStoreNode.storeObject(nonNullDest, arrayBaseOffset, i + (long) destPos * scale, a, arrayLocation, elementKind); + i -= scale; + } + } + } else { + long end = (long) length * scale; + long i = 0; + ExplodeLoopNode.explodeLoop(); + for (int iteration = 0; iteration < length; iteration++) { + if (i < end) { + Object a = UnsafeLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation); + DirectObjectStoreNode.storeObject(nonNullDest, arrayBaseOffset, i + (long) destPos * scale, a, arrayLocation, elementKind); + i += scale; + } + } + } } private static final SnippetCounter.Group checkCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy checkInputs") : null; private static final SnippetCounter checkSuccessCounter = new SnippetCounter(checkCounters, "checkSuccess", "checkSuccess"); - private static final SnippetCounter checkNPECounter = new SnippetCounter(checkCounters, "checkNPE", "checkNPE"); private static final SnippetCounter checkAIOOBECounter = new SnippetCounter(checkCounters, "checkAIOOBE", "checkAIOOBE"); private static final SnippetCounter.Group counters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy") : null; - private static final SnippetCounter byteCounter = new SnippetCounter(counters, "byte[]", "arraycopy for byte[] arrays"); - private static final SnippetCounter charCounter = new SnippetCounter(counters, "char[]", "arraycopy for char[] arrays"); - private static final SnippetCounter shortCounter = new SnippetCounter(counters, "short[]", "arraycopy for short[] arrays"); - private static final SnippetCounter intCounter = new SnippetCounter(counters, "int[]", "arraycopy for int[] arrays"); - private static final SnippetCounter booleanCounter = new SnippetCounter(counters, "boolean[]", "arraycopy for boolean[] arrays"); - private static final SnippetCounter longCounter = new SnippetCounter(counters, "long[]", "arraycopy for long[] arrays"); - private static final SnippetCounter objectCounter = new SnippetCounter(counters, "Object[]", "arraycopy for Object[] arrays"); + private static final SnippetCounter objectCheckcastCounter = new SnippetCounter(counters, "Object[]", "arraycopy for non-exact Object[] arrays"); - private static final SnippetCounter floatCounter = new SnippetCounter(counters, "float[]", "arraycopy for float[] arrays"); - private static final SnippetCounter doubleCounter = new SnippetCounter(counters, "double[]", "arraycopy for double[] arrays"); + private static final SnippetCounter objectCheckcastSameTypeCounter = new SnippetCounter(counters, "Object[]", "arraycopy call for src.klass == dest.klass Object[] arrays"); + private static final SnippetCounter predictedObjectArrayCopySlowPathCounter = new SnippetCounter(counters, "Object[]", "used System.arraycopy slow path for predicted Object[] arrays"); + private static final SnippetCounter predictedObjectArrayCopyFastPathCounter = new SnippetCounter(counters, "Object[]", "used oop_arraycopy for predicted Object[] arrays"); + + private static final EnumMap<Kind, SnippetCounter> arraycopyCallCounters = new EnumMap<>(Kind.class); + private static final EnumMap<Kind, SnippetCounter> arraycopyCounters = new EnumMap<>(Kind.class); - private static final SnippetCounter objectCallCounter = new SnippetCounter(counters, "Object[]", "arraycopy call for Object[] arrays"); + static void createArraycopyCounter(Kind kind) { + arraycopyCallCounters.put(kind, new SnippetCounter(counters, kind + "[]", "arraycopy call for " + kind + "[] arrays")); + arraycopyCounters.put(kind, new SnippetCounter(counters, kind + "[]", "inline arraycopy for " + kind + "[] arrays")); + } - private static final SnippetCounter booleanCallCounter = new SnippetCounter(counters, "boolean[]", "arraycopy call for boolean[] arrays"); - private static final SnippetCounter byteCallCounter = new SnippetCounter(counters, "byte[]", "arraycopy call for byte[] arrays"); - private static final SnippetCounter charCallCounter = new SnippetCounter(counters, "char[]", "arraycopy call for char[] arrays"); - private static final SnippetCounter doubleCallCounter = new SnippetCounter(counters, "double[]", "arraycopy call for double[] arrays"); - private static final SnippetCounter floatCallCounter = new SnippetCounter(counters, "float[]", "arraycopy call for float[] arrays"); - private static final SnippetCounter intCallCounter = new SnippetCounter(counters, "int[]", "arraycopy call for int[] arrays"); - private static final SnippetCounter longCallCounter = new SnippetCounter(counters, "long[]", "arraycopy call for long[] arrays"); - private static final SnippetCounter shortCallCounter = new SnippetCounter(counters, "short[]", "arraycopy call for short[] arrays"); + static { + createArraycopyCounter(Kind.Byte); + createArraycopyCounter(Kind.Boolean); + createArraycopyCounter(Kind.Char); + createArraycopyCounter(Kind.Short); + createArraycopyCounter(Kind.Int); + createArraycopyCounter(Kind.Long); + createArraycopyCounter(Kind.Float); + createArraycopyCounter(Kind.Double); + createArraycopyCounter(Kind.Object); + } private static final SnippetCounter genericPrimitiveCallCounter = new SnippetCounter(counters, "genericPrimitive", "generic arraycopy snippet for primitive arrays"); private static final SnippetCounter genericObjectExactCallCounter = new SnippetCounter(counters, "genericObjectExact", "generic arraycopy snippet for special object arrays"); - private static final SnippetCounter genericObjectCallCounter = new SnippetCounter(counters, "genericObject", "call to the generic, native arraycopy method"); + private static final SnippetCounter SystemArraycopyCounter = new SnippetCounter(counters, "genericObject", "call to System.arraycopy"); + + private static final SnippetCounter.Group lengthCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy 0-length checks") : null; + + private static final SnippetCounter zeroLengthStaticCounter = new SnippetCounter(lengthCounters, "0-lengthcopy static", "arraycopy where the length is statically 0"); + private static final SnippetCounter zeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "0-lengthcopy dynamically", "arraycopy where the length is dynamically 0"); + private static final SnippetCounter nonZeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero"); + + public static class Templates extends SnippetTemplate.AbstractTemplates { + + public Templates(HotSpotProviders providers, TargetDescription target) { + super(providers, providers.getSnippetReflection(), target); + } + + private ResolvedJavaMethod originalArraycopy() throws GraalInternalError { + if (originalArraycopy == null) { + Method method; + try { + method = System.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class); + } catch (NoSuchMethodException | SecurityException e) { + throw new GraalInternalError(e); + } + originalArraycopy = providers.getMetaAccess().lookupJavaMethod(method); + } + return originalArraycopy; + } + + private ResolvedJavaMethod originalArraycopy; + + private final SnippetInfo checkcastArraycopyWorkSnippet = snippet("checkcastArraycopyWork"); + private final SnippetInfo arraycopyGenericSnippet = snippet("arraycopyGeneric"); + + private final SnippetInfo arraycopySlowPathIntrinsicSnippet = snippet("arraycopySlowPathIntrinsic"); + private final SnippetInfo arraycopyExactIntrinsicSnippet = snippet("arraycopyExactIntrinsic"); + private final SnippetInfo arraycopyZeroLengthIntrinsicSnippet = snippet("arraycopyZeroLengthIntrinsic"); + private final SnippetInfo arraycopyPredictedExactIntrinsicSnippet = snippet("arraycopyPredictedExactIntrinsic"); + private final SnippetInfo arraycopyPredictedObjectWorkSnippet = snippet("arraycopyPredictedObjectWork"); + + private final SnippetInfo arraycopyUnrolledWorkSnippet = snippet("arraycopyUnrolledWork"); + + protected SnippetInfo snippet(String methodName) { + SnippetInfo info = snippet(ArrayCopySnippets.class, methodName, LocationIdentity.any()); + info.setOriginalMethod(originalArraycopy()); + return info; + } + + public static Kind selectComponentKind(BasicArrayCopyNode arraycopy) { + return selectComponentKind(arraycopy, true); + } + + public static Kind selectComponentKind(BasicArrayCopyNode arraycopy, boolean exact) { + ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp()); + ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp()); + + if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) { + if (!exact) { + Kind component = getComponentKind(srcType); + if (component != null) { + return component; + } + return getComponentKind(destType); + } + return null; + } + if (exact) { + if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) { + return null; + } + if (!arraycopy.isExact()) { + return null; + } + } + return srcType.getComponentType().getKind(); + } + + private static Kind getComponentKind(ResolvedJavaType type) { + if (type != null && type.isArray()) { + return type.getComponentType().getKind(); + } + return null; + } + + private static boolean shouldUnroll(ValueNode length) { + return length.isConstant() && length.asJavaConstant().asInt() <= 8 && length.asJavaConstant().asInt() != 0; + } + + private static void unrollFixedLengthLoop(StructuredGraph snippetGraph, int length, LoweringTool tool) { + ParameterNode lengthParam = snippetGraph.getParameter(4); + if (lengthParam != null) { + snippetGraph.replaceFloating(lengthParam, ConstantNode.forInt(length, snippetGraph)); + } + // the canonicalization before loop unrolling is needed to propagate the length into + // additions, etc. + PhaseContext context = new PhaseContext(tool.getMetaAccess(), tool.getConstantReflection(), tool.getLowerer(), tool.getReplacements(), tool.getStampProvider()); + new CanonicalizerPhase().apply(snippetGraph, context); + new LoopFullUnrollPhase(new CanonicalizerPhase()).apply(snippetGraph, context); + new CanonicalizerPhase().apply(snippetGraph, context); + } + + void unrollSnippet(final LoweringTool tool, StructuredGraph snippetGraph, ArrayCopyNode arraycopy) { + if (shouldUnroll(arraycopy.getLength())) { + final StructuredGraph copy = snippetGraph; + try (Scope s = Debug.scope("ArrayCopySnippetSpecialization", snippetGraph.method())) { + unrollFixedLengthLoop(copy, arraycopy.getLength().asJavaConstant().asInt(), tool); + } catch (Throwable e) { + throw Debug.handle(e); + } + } + } + + public void lower(ArrayCopyNode arraycopy, LoweringTool tool) { + Kind componentKind = selectComponentKind(arraycopy); + SnippetInfo snippetInfo = null; + SnippetInfo slowPathSnippetInfo = null; + + if (arraycopy.getLength().isConstant() && arraycopy.getLength().asJavaConstant().asLong() == 0) { + snippetInfo = arraycopyZeroLengthIntrinsicSnippet; + } else if (arraycopy.isExact()) { + snippetInfo = arraycopyExactIntrinsicSnippet; + if (shouldUnroll(arraycopy.getLength())) { + snippetInfo = arraycopySlowPathIntrinsicSnippet; + slowPathSnippetInfo = arraycopyUnrolledWorkSnippet; + } + } else { + if (componentKind == Kind.Object) { + ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp()); + ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp()); + ResolvedJavaType srcComponentType = srcType == null ? null : srcType.getComponentType(); + ResolvedJavaType destComponentType = destType == null ? null : destType.getComponentType(); + if (srcComponentType != null && destComponentType != null && !srcComponentType.isPrimitive() && !destComponentType.isPrimitive()) { + snippetInfo = arraycopySlowPathIntrinsicSnippet; + slowPathSnippetInfo = checkcastArraycopyWorkSnippet; + /* + * Because this snippet has to use Sysytem.arraycopy as a slow path, we must + * pretend to kill any() so clear the componentKind. + */ + componentKind = null; + } + } + if (componentKind == null && snippetInfo == null) { + Kind predictedKind = selectComponentKind(arraycopy, false); + if (predictedKind != null) { + /* + * At least one array is of a known type requiring no store checks, so + * assume the other is of the same type. Generally this is working around + * deficiencies in our propation of type information. + */ + componentKind = predictedKind; + if (predictedKind == Kind.Object) { + snippetInfo = arraycopySlowPathIntrinsicSnippet; + slowPathSnippetInfo = arraycopyPredictedObjectWorkSnippet; + componentKind = null; + } else { + snippetInfo = arraycopyPredictedExactIntrinsicSnippet; + } + } + } + if (snippetInfo == null) { + snippetInfo = arraycopyGenericSnippet; + } + } + Arguments args = new Arguments(snippetInfo, arraycopy.graph().getGuardsStage(), tool.getLoweringStage()); + args.add("src", arraycopy.getSource()); + args.add("srcPos", arraycopy.getSourcePosition()); + args.add("dest", arraycopy.getDestination()); + args.add("destPos", arraycopy.getDestinationPosition()); + args.add("length", arraycopy.getLength()); + if (snippetInfo == arraycopySlowPathIntrinsicSnippet) { + args.addConst("elementKind", componentKind != null ? componentKind : Kind.Illegal); + args.addConst("slowPath", slowPathSnippetInfo); + } else if (snippetInfo == arraycopyExactIntrinsicSnippet || snippetInfo == arraycopyPredictedExactIntrinsicSnippet) { + assert componentKind != null; + args.addConst("elementKind", componentKind); + args.addConst("counter", arraycopyCallCounters.get(componentKind)); + } + instantiate(args, arraycopy); + } + + public void lower(ArrayCopySlowPathNode arraycopy, LoweringTool tool) { + StructuredGraph graph = arraycopy.graph(); + if (!graph.getGuardsStage().areFrameStatesAtDeopts()) { + // Can't be lowered yet + return; + } + SnippetInfo snippetInfo = arraycopy.getSnippet(); + Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage()); + args.add("nonNullSrc", arraycopy.getSource()); + args.add("srcPos", arraycopy.getSourcePosition()); + args.add("nonNullDest", arraycopy.getDestination()); + args.add("destPos", arraycopy.getDestinationPosition()); + if (snippetInfo == arraycopyUnrolledWorkSnippet) { + args.addConst("length", arraycopy.getLength().asJavaConstant().asInt()); + args.addConst("elementKind", arraycopy.getElementKind()); + } else { + args.add("length", arraycopy.getLength()); + } + if (snippetInfo == arraycopyPredictedObjectWorkSnippet) { + HotSpotResolvedObjectType arrayKlass = (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(Object[].class); + ValueNode objectArrayKlass = ConstantNode.forConstant(KlassPointerStamp.klassNonNull(), arrayKlass.klass(), tool.getMetaAccess(), arraycopy.graph()); + args.add("objectArrayKlass", objectArrayKlass); + args.addConst("counter", arraycopyCallCounters.get(Kind.Object)); + } + instantiate(args, arraycopy); + } + + /** + * Instantiate the snippet template and fix up the FrameState of any Invokes of + * System.arraycopy and propagate the captured bci in the ArrayCopySlowPathNode. + * + * @param args + * @param arraycopy + */ + private void instantiate(Arguments args, BasicArrayCopyNode arraycopy) { + StructuredGraph graph = arraycopy.graph(); + SnippetTemplate template = template(args); + Map<Node, Node> replacements = template.instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args); + for (Node originalNode : replacements.keySet()) { + if (originalNode instanceof Invoke) { + Invoke invoke = (Invoke) replacements.get(originalNode); + assert invoke.asNode().graph() == graph; + CallTargetNode call = invoke.callTarget(); + + if (!call.targetMethod().equals(originalArraycopy)) { + throw new GraalInternalError("unexpected invoke %s in snippet", call.targetMethod()); + } + // Here we need to fix the bci of the invoke + InvokeNode newInvoke = graph.add(new InvokeNode(invoke.callTarget(), arraycopy.getBci())); + if (arraycopy.stateDuring() != null) { + newInvoke.setStateDuring(arraycopy.stateDuring()); + } else { + assert arraycopy.stateAfter() != null; + newInvoke.setStateAfter(arraycopy.stateAfter()); + } + graph.replaceFixedWithFixed((InvokeNode) invoke.asNode(), newInvoke); + } else if (originalNode instanceof ArrayCopySlowPathNode) { + ArrayCopySlowPathNode slowPath = (ArrayCopySlowPathNode) replacements.get(originalNode); + assert arraycopy.stateAfter() != null; + slowPath.setStateAfter(arraycopy.stateAfter()); + slowPath.setBci(arraycopy.getBci()); + } + } + } + } }
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/ArrayCopyStateNode.java Wed May 20 11:58:51 2015 -0700 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,57 +0,0 @@ -/* - * Copyright (c) 2015, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -//JaCoCo Exclude -package com.oracle.graal.hotspot.replacements.arraycopy; - -import com.oracle.graal.api.meta.*; -import com.oracle.graal.compiler.common.type.*; -import com.oracle.graal.graph.*; -import com.oracle.graal.nodeinfo.*; -import com.oracle.graal.nodes.*; -import com.oracle.graal.nodes.spi.*; - -/** - * A dummy node whose only purpose is to capture a final {@link FrameState} when lowering complex - * arraycopy snippets. - */ - -@NodeInfo -public final class ArrayCopyStateNode extends AbstractStateSplit implements Lowerable { - - public static final NodeClass<ArrayCopyStateNode> TYPE = NodeClass.create(ArrayCopyStateNode.class); - - protected ArrayCopyStateNode() { - super(TYPE, StampFactory.forKind(Kind.Void)); - - } - - @Override - public void lower(LoweringTool tool) { - if (graph().getGuardsStage().areFrameStatesAtDeopts()) { - graph().removeFixed(this); - } - } - - @NodeIntrinsic - public static native void captureState(); -}
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/CheckcastArrayCopyCallNode.java Wed May 20 11:58:51 2015 -0700 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/CheckcastArrayCopyCallNode.java Thu May 21 22:20:41 2015 -0700 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2014, 2014, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2014, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -24,7 +24,6 @@ package com.oracle.graal.hotspot.replacements.arraycopy; import com.oracle.graal.api.meta.*; -import com.oracle.graal.api.runtime.*; import com.oracle.graal.compiler.common.type.*; import com.oracle.graal.graph.*; import com.oracle.graal.hotspot.*; @@ -35,7 +34,6 @@ import com.oracle.graal.nodes.calc.*; import com.oracle.graal.nodes.extended.*; import com.oracle.graal.nodes.spi.*; -import com.oracle.graal.runtime.*; import com.oracle.graal.word.*; @NodeInfo(allowedUsageTypes = {InputType.Memory, InputType.Value}) @@ -107,11 +105,10 @@ ValueNode srcAddr = computeBase(getSource(), getSourcePosition()); ValueNode destAddr = computeBase(getDestination(), getDestinationPosition()); ValueNode len = getLength(); - if (len.stamp().getStackKind() != Kind.Long) { - len = IntegerConvertNode.convert(len, StampFactory.forKind(Kind.Long), graph()); + if (len.stamp().getStackKind() != runtime.getTarget().wordKind) { + len = IntegerConvertNode.convert(len, StampFactory.forKind(runtime.getTarget().wordKind), graph()); } - ForeignCallNode call = graph.add(new ForeignCallNode(Graal.getRequiredCapability(RuntimeProvider.class).getHostBackend().getForeignCalls(), desc, srcAddr, destAddr, len, superCheckOffset, - destElemKlass)); + ForeignCallNode call = graph.add(new ForeignCallNode(runtime.getHostBackend().getForeignCalls(), desc, srcAddr, destAddr, len, superCheckOffset, destElemKlass)); call.setStateAfter(stateAfter()); graph.replaceFixedWithFixed(this, call); } @@ -119,7 +116,11 @@ @Override public LocationIdentity getLocationIdentity() { - return NamedLocationIdentity.getArrayLocation(Kind.Object); + /* + * Because of restrictions that the memory graph of snippets matches the original node, + * pretend that we kill any. + */ + return LocationIdentity.any(); } @NodeIntrinsic
--- a/graal/com.oracle.graal.replacements/src/com/oracle/graal/replacements/SnippetTemplate.java Wed May 20 11:58:51 2015 -0700 +++ b/graal/com.oracle.graal.replacements/src/com/oracle/graal/replacements/SnippetTemplate.java Thu May 21 22:20:41 2015 -0700 @@ -195,6 +195,11 @@ } return null; } + + @Override + public String toString() { + return getClass().getSimpleName() + ":" + method.format("%h.%n"); + } } protected static class LazySnippetInfo extends SnippetInfo {
--- a/graal/com.oracle.graal.replacements/src/com/oracle/graal/replacements/nodes/BasicArrayCopyNode.java Wed May 20 11:58:51 2015 -0700 +++ b/graal/com.oracle.graal.replacements/src/com/oracle/graal/replacements/nodes/BasicArrayCopyNode.java Thu May 21 22:20:41 2015 -0700 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2013, 2015, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -22,44 +22,109 @@ */ package com.oracle.graal.replacements.nodes; +import static com.oracle.graal.api.meta.LocationIdentity.*; + import com.oracle.graal.api.meta.*; +import com.oracle.graal.compiler.common.type.*; import com.oracle.graal.debug.*; import com.oracle.graal.graph.*; import com.oracle.graal.nodeinfo.*; -import com.oracle.graal.nodes.CallTargetNode.InvokeKind; import com.oracle.graal.nodes.*; +import com.oracle.graal.nodes.extended.*; import com.oracle.graal.nodes.java.*; import com.oracle.graal.nodes.spi.*; import com.oracle.graal.nodes.type.*; import com.oracle.graal.nodes.virtual.*; @NodeInfo -public abstract class BasicArrayCopyNode extends MacroStateSplitNode implements Virtualizable { +public class BasicArrayCopyNode extends AbstractMemoryCheckpoint implements Virtualizable, MemoryCheckpoint.Single, MemoryAccess, Lowerable, DeoptimizingNode.DeoptDuring { public static final NodeClass<BasicArrayCopyNode> TYPE = NodeClass.create(BasicArrayCopyNode.class); - public BasicArrayCopyNode(NodeClass<? extends MacroNode> c, InvokeKind invokeKind, ResolvedJavaMethod targetMethod, int bci, JavaType returnType, ValueNode... arguments) { - super(c, invokeKind, targetMethod, bci, returnType, arguments); + @Input protected ValueNode src; + @Input protected ValueNode srcPos; + @Input protected ValueNode dest; + @Input protected ValueNode destPos; + @Input protected ValueNode length; + + @OptionalInput(InputType.State) FrameState stateDuring; + + @OptionalInput(InputType.Memory) protected MemoryNode lastLocationAccess; + + protected Kind elementKind; + + protected int bci; + + public BasicArrayCopyNode(NodeClass<? extends AbstractMemoryCheckpoint> type, ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, Kind elementKind, int bci) { + super(type, StampFactory.forKind(Kind.Void)); + this.bci = bci; + this.src = src; + this.srcPos = srcPos; + this.dest = dest; + this.destPos = destPos; + this.length = length; + this.elementKind = elementKind != Kind.Illegal ? elementKind : null; } - protected ValueNode getSource() { - return arguments.get(0); + public BasicArrayCopyNode(NodeClass<? extends AbstractMemoryCheckpoint> type, ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, Kind elementKind) { + super(type, StampFactory.forKind(Kind.Void)); + this.bci = -6; + this.src = src; + this.srcPos = srcPos; + this.dest = dest; + this.destPos = destPos; + this.length = length; + this.elementKind = elementKind != Kind.Illegal ? elementKind : null; + } + + public ValueNode getSource() { + return src; } - protected ValueNode getSourcePosition() { - return arguments.get(1); + public ValueNode getSourcePosition() { + return srcPos; + } + + public ValueNode getDestination() { + return dest; + } + + public ValueNode getDestinationPosition() { + return destPos; + } + + public ValueNode getLength() { + return length; + } + + public int getBci() { + return bci; } - protected ValueNode getDestination() { - return arguments.get(2); + public Kind getElementKind() { + return elementKind; + } + + @Override + public LocationIdentity getLocationIdentity() { + if (elementKind != null) { + return NamedLocationIdentity.getArrayLocation(elementKind); + } + return any(); } - protected ValueNode getDestinationPosition() { - return arguments.get(3); + public MemoryNode getLastLocationAccess() { + return lastLocationAccess; } - protected ValueNode getLength() { - return arguments.get(4); + public void setLastLocationAccess(MemoryNode lla) { + updateUsagesInterface(lastLocationAccess, lla); + lastLocationAccess = lla; + } + + @Override + public void lower(LoweringTool tool) { + tool.getLowerer().lower(this, tool); } private static boolean checkBounds(int position, int length, VirtualObjectNode virtualObject) { @@ -89,16 +154,37 @@ return true; } + /* + * Returns true if this copy doesn't require store checks. Trivially true for primitive arrays. + */ + public boolean isExact() { + ResolvedJavaType srcType = StampTool.typeOrNull(getSource().stamp()); + ResolvedJavaType destType = StampTool.typeOrNull(getDestination().stamp()); + if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) { + return false; + } + if (srcType.getComponentType().getKind().isPrimitive() || getSource() == getDestination()) { + return true; + } + + if (StampTool.isExactType(getDestination().stamp())) { + if (destType != null && destType.isAssignableFrom(srcType)) { + return true; + } + } + return false; + } + @Override public void virtualize(VirtualizerTool tool) { ValueNode sourcePosition = tool.getReplacedValue(getSourcePosition()); ValueNode destinationPosition = tool.getReplacedValue(getDestinationPosition()); - ValueNode length = tool.getReplacedValue(getLength()); + ValueNode replacedLength = tool.getReplacedValue(getLength()); - if (sourcePosition.isConstant() && destinationPosition.isConstant() && length.isConstant()) { - int srcPos = sourcePosition.asJavaConstant().asInt(); - int destPos = destinationPosition.asJavaConstant().asInt(); - int len = length.asJavaConstant().asInt(); + if (sourcePosition.isConstant() && destinationPosition.isConstant() && replacedLength.isConstant()) { + int srcPosInt = sourcePosition.asJavaConstant().asInt(); + int destPosInt = destinationPosition.asJavaConstant().asInt(); + int len = replacedLength.asJavaConstant().asInt(); State destState = tool.getObjectState(getDestination()); if (destState != null && destState.getState() == EscapeState.Virtual) { @@ -106,7 +192,7 @@ if (!(destVirtual instanceof VirtualArrayNode)) { return; } - if (len < 0 || !checkBounds(destPos, len, destVirtual)) { + if (len < 0 || !checkBounds(destPosInt, len, destVirtual)) { return; } State srcState = tool.getObjectState(getSource()); @@ -121,18 +207,18 @@ if (((VirtualArrayNode) srcVirtual).componentType().getKind() != Kind.Object) { return; } - if (!checkBounds(srcPos, len, srcVirtual)) { + if (!checkBounds(srcPosInt, len, srcVirtual)) { return; } - if (!checkEntryTypes(srcPos, len, srcState, destVirtual.type().getComponentType(), tool)) { + if (!checkEntryTypes(srcPosInt, len, srcState, destVirtual.type().getComponentType(), tool)) { return; } for (int i = 0; i < len; i++) { - tool.setVirtualEntry(destState, destPos + i, srcState.getEntry(srcPos + i), false); + tool.setVirtualEntry(destState, destPosInt + i, srcState.getEntry(srcPosInt + i), false); } tool.delete(); if (Debug.isLogEnabled()) { - Debug.log("virtualized arraycopyf(%s, %d, %s, %d, %d)", getSource(), srcPos, getDestination(), destPos, len); + Debug.log("virtualized arraycopyf(%s, %d, %s, %d, %d)", getSource(), srcPosInt, getDestination(), destPosInt, len); } } else { ValueNode source = srcState == null ? tool.getReplacedValue(getSource()) : srcState.getMaterializedValue(); @@ -146,13 +232,31 @@ return; } for (int i = 0; i < len; i++) { - LoadIndexedNode load = new LoadIndexedNode(source, ConstantNode.forInt(i + srcPos, graph()), destComponentType.getKind()); + LoadIndexedNode load = new LoadIndexedNode(source, ConstantNode.forInt(i + srcPosInt, graph()), destComponentType.getKind()); tool.addNode(load); - tool.setVirtualEntry(destState, destPos + i, load, false); + tool.setVirtualEntry(destState, destPosInt + i, load, false); } tool.delete(); } } } } + + public boolean canDeoptimize() { + return true; + } + + public FrameState stateDuring() { + return stateDuring; + } + + public void setStateDuring(FrameState stateDuring) { + updateUsages(this.stateDuring, stateDuring); + this.stateDuring = stateDuring; + } + + public void computeStateDuring(FrameState currentStateAfter) { + FrameState newStateDuring = currentStateAfter.duplicateModifiedDuringCall(getBci(), asNode().getKind()); + setStateDuring(newStateDuring); + } }