Source code

001/*
002 * Copyright (c) 2011, 2015, Oracle and/or its affiliates. All rights reserved.
003 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
004 *
005 * This code is free software; you can redistribute it and/or modify it
006 * under the terms of the GNU General Public License version 2 only, as
007 * published by the Free Software Foundation.
008 *
009 * This code is distributed in the hope that it will be useful, but WITHOUT
010 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
011 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
012 * version 2 for more details (a copy is included in the LICENSE file that
013 * accompanied this code).
014 *
015 * You should have received a copy of the GNU General Public License version
016 * 2 along with this work; if not, write to the Free Software Foundation,
017 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
018 *
019 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
020 * or visit www.oracle.com if you need additional information or have any
021 * questions.
022 */
023package com.oracle.graal.hotspot.replacements.arraycopy;
024
025import static com.oracle.graal.compiler.common.GraalOptions.*;
026import static com.oracle.graal.hotspot.replacements.HotSpotReplacementsUtil.*;
027import static com.oracle.graal.nodes.extended.BranchProbabilityNode.*;
028
029import java.lang.reflect.*;
030import java.util.*;
031
032import jdk.internal.jvmci.code.*;
033import jdk.internal.jvmci.common.*;
034import jdk.internal.jvmci.hotspot.*;
035import jdk.internal.jvmci.meta.*;
036
037import com.oracle.graal.api.directives.*;
038import com.oracle.graal.api.replacements.*;
039import com.oracle.graal.graph.*;
040import com.oracle.graal.hotspot.meta.*;
041import com.oracle.graal.hotspot.word.*;
042import com.oracle.graal.nodes.*;
043import com.oracle.graal.nodes.extended.*;
044import com.oracle.graal.nodes.java.*;
045import com.oracle.graal.nodes.spi.*;
046import com.oracle.graal.nodes.type.*;
047import com.oracle.graal.replacements.*;
048import com.oracle.graal.replacements.Snippet.ConstantParameter;
049import com.oracle.graal.replacements.SnippetTemplate.Arguments;
050import com.oracle.graal.replacements.SnippetTemplate.SnippetInfo;
051import com.oracle.graal.replacements.nodes.*;
052import com.oracle.graal.word.*;
053
054public class ArrayCopySnippets implements Snippets {
055
056    private static int checkArrayType(KlassPointer hub) {
057        int layoutHelper = readLayoutHelper(hub);
058        if (probability(SLOW_PATH_PROBABILITY, layoutHelper >= 0)) {
059            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
060        }
061        return layoutHelper;
062    }
063
064    private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length) {
065        if (probability(SLOW_PATH_PROBABILITY, srcPos < 0)) {
066            checkAIOOBECounter.inc();
067            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
068        }
069        if (probability(SLOW_PATH_PROBABILITY, destPos < 0)) {
070            checkAIOOBECounter.inc();
071            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
072        }
073        if (probability(SLOW_PATH_PROBABILITY, length < 0)) {
074            checkAIOOBECounter.inc();
075            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
076        }
077        if (probability(SLOW_PATH_PROBABILITY, srcPos + length > ArrayLengthNode.arrayLength(src))) {
078            checkAIOOBECounter.inc();
079            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
080        }
081        if (probability(SLOW_PATH_PROBABILITY, destPos + length > ArrayLengthNode.arrayLength(dest))) {
082            checkAIOOBECounter.inc();
083            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
084        }
085        checkSuccessCounter.inc();
086    }
087
088    @Snippet
089    public static void arraycopyZeroLengthIntrinsic(Object src, int srcPos, Object dest, int destPos, int length) {
090        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
091        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
092        KlassPointer srcHub = loadHub(nonNullSrc);
093        KlassPointer destHub = loadHub(nonNullDest);
094        checkArrayType(srcHub);
095        checkArrayType(destHub);
096        checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length);
097        zeroLengthStaticCounter.inc();
098    }
099
100    @Snippet
101    public static void arraycopyExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Kind elementKind, @ConstantParameter SnippetCounter counter,
102                    @ConstantParameter SnippetCounter copiedCounter) {
103        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
104        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
105        checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length);
106        counter.inc();
107        copiedCounter.add(length);
108        ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind);
109        if (length == 0) {
110            zeroLengthDynamicCounter.inc();
111        } else {
112            nonZeroLengthDynamicCounter.inc();
113            nonZeroLengthDynamicCopiedCounter.add(length);
114        }
115    }
116
117    /**
118     * This intrinsic is useful for the case where we know something statically about one of the
119     * inputs but not the other.
120     */
121    @Snippet
122    public static void arraycopyPredictedExactIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Kind elementKind, @ConstantParameter SnippetCounter counter,
123                    @ConstantParameter SnippetCounter copiedCounter) {
124        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
125        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
126        KlassPointer srcHub = loadHub(nonNullSrc);
127        KlassPointer destHub = loadHub(nonNullDest);
128        if (probability(SLOW_PATH_PROBABILITY, srcHub != destHub)) {
129            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
130        }
131        checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length);
132        counter.inc();
133        copiedCounter.add(length);
134        ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind);
135        if (length == 0) {
136            zeroLengthDynamicCounter.inc();
137        } else {
138            nonZeroLengthDynamicCounter.inc();
139            nonZeroLengthDynamicCopiedCounter.add(length);
140        }
141    }
142
143    @Snippet
144    public static void arraycopyPredictedObjectWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length, KlassPointer objectArrayKlass,
145                    @ConstantParameter SnippetCounter counter, @ConstantParameter SnippetCounter copiedCounter) {
146        if (length > 0) {
147            KlassPointer srcHub = loadHub(nonNullSrc);
148            KlassPointer destHub = loadHub(nonNullDest);
149            if (probability(FAST_PATH_PROBABILITY, srcHub == destHub || destHub == objectArrayKlass)) {
150                counter.inc();
151                copiedCounter.add(length);
152                predictedObjectArrayCopyFastPathCounter.inc();
153                predictedObjectArrayCopyFastPathCopiedCounter.add(length);
154                ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length);
155            } else {
156                predictedObjectArrayCopySlowPathCounter.inc();
157                predictedObjectArrayCopySlowPathCopiedCounter.add(length);
158                System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length);
159            }
160        }
161    }
162
163    /**
164     * This is the basic template for the full arraycopy checks, including a check that the
165     * underlying type is really an array type.
166     */
167    @Snippet
168    public static void arraycopySlowPathIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter Kind elementKind, @ConstantParameter SnippetInfo slowPath,
169                    @ConstantParameter Object slowPathArgument) {
170        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
171        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
172        KlassPointer srcHub = loadHub(nonNullSrc);
173        KlassPointer destHub = loadHub(nonNullDest);
174        checkArrayType(srcHub);
175        checkArrayType(destHub);
176        checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length);
177        if (length == 0) {
178            zeroLengthDynamicCounter.inc();
179        } else {
180            nonZeroLengthDynamicCounter.inc();
181            nonZeroLengthDynamicCopiedCounter.add(length);
182        }
183        ArrayCopySlowPathNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, elementKind, slowPath, slowPathArgument);
184    }
185
186    /**
187     * Snippet for unrolled arraycopy.
188     */
189    @Snippet
190    public static void arraycopyUnrolledIntrinsic(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter int unrolledLength, @ConstantParameter Kind elementKind) {
191        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
192        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
193        checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length);
194        if (length == 0) {
195            zeroLengthDynamicCounter.inc();
196        } else {
197            nonZeroLengthDynamicCounter.inc();
198            nonZeroLengthDynamicCopiedCounter.add(length);
199        }
200        ArrayCopyUnrollNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, unrolledLength, elementKind);
201    }
202
203    @Snippet
204    public static void checkcastArraycopyWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, int length) {
205        if (length > 0) {
206            KlassPointer destKlass = loadHub(nonNullDest);
207            KlassPointer srcKlass = loadHub(nonNullSrc);
208            if (probability(SLOW_PATH_PROBABILITY, srcKlass == destKlass)) {
209                // no storecheck required.
210                objectCheckcastSameTypeCounter.inc();
211                objectCheckcastSameTypeCopiedCounter.add(length);
212                ArrayCopyCallNode.arraycopyObjectKillsAny(nonNullSrc, srcPos, nonNullDest, destPos, length);
213            } else {
214                KlassPointer destElemKlass = destKlass.readKlassPointer(arrayClassElementOffset(), OBJ_ARRAY_KLASS_ELEMENT_KLASS_LOCATION);
215                Word superCheckOffset = Word.signed(destElemKlass.readInt(superCheckOffsetOffset(), KLASS_SUPER_CHECK_OFFSET_LOCATION));
216                objectCheckcastCounter.inc();
217                objectCheckcastCopiedCounter.add(length);
218                int copiedElements = CheckcastArrayCopyCallNode.checkcastArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, superCheckOffset, destElemKlass, false);
219                if (copiedElements != 0) {
220                    /*
221                     * the checkcast stub doesn't throw the ArrayStoreException, but returns the
222                     * number of copied elements (xor'd with -1).
223                     */
224                    copiedElements ^= -1;
225                    System.arraycopy(nonNullSrc, srcPos + copiedElements, nonNullDest, destPos + copiedElements, length - copiedElements);
226                }
227            }
228        }
229    }
230
231    @Snippet
232    public static void arraycopyGeneric(Object src, int srcPos, Object dest, int destPos, int length) {
233        Object nonNullSrc = GraalDirectives.guardingNonNull(src);
234        Object nonNullDest = GraalDirectives.guardingNonNull(dest);
235        KlassPointer srcHub = loadHub(nonNullSrc);
236        KlassPointer destHub = loadHub(nonNullDest);
237        if (probability(FAST_PATH_PROBABILITY, srcHub.equal(destHub)) && probability(FAST_PATH_PROBABILITY, nonNullSrc != nonNullDest)) {
238            int layoutHelper = checkArrayType(srcHub);
239            final boolean isObjectArray = ((layoutHelper & layoutHelperElementTypePrimitiveInPlace()) == 0);
240            checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length);
241            if (probability(FAST_PATH_PROBABILITY, isObjectArray)) {
242                genericObjectExactCallCounter.inc();
243                genericObjectExactCallCopiedCounter.add(length);
244                ArrayCopyCallNode.disjointArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, Kind.Object);
245            } else {
246                genericPrimitiveCallCounter.inc();
247                genericPrimitiveCallCopiedCounter.add(length);
248                UnsafeArrayCopyNode.arraycopyPrimitive(nonNullSrc, srcPos, nonNullDest, destPos, length, layoutHelper);
249            }
250        } else {
251            SystemArraycopyCounter.inc();
252            SystemArraycopyCopiedCounter.add(length);
253            System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length);
254        }
255    }
256
257    @Fold
258    private static LocationIdentity getArrayLocation(Kind kind) {
259        return NamedLocationIdentity.getArrayLocation(kind);
260    }
261
262    @Snippet
263    public static void arraycopyUnrolledWork(Object nonNullSrc, int srcPos, Object nonNullDest, int destPos, @ConstantParameter int length, @ConstantParameter Kind elementKind) {
264        final int scale = arrayIndexScale(elementKind);
265        int arrayBaseOffset = arrayBaseOffset(elementKind);
266        LocationIdentity arrayLocation = getArrayLocation(elementKind);
267        if (nonNullSrc == nonNullDest && srcPos < destPos) { // bad aliased case
268            long start = (long) (length - 1) * scale;
269            long i = start;
270            ExplodeLoopNode.explodeLoop();
271            for (int iteration = 0; iteration < length; iteration++) {
272                if (i >= 0) {
273                    Object a = UnsafeLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation);
274                    DirectObjectStoreNode.storeObject(nonNullDest, arrayBaseOffset, i + (long) destPos * scale, a, arrayLocation, elementKind);
275                    i -= scale;
276                }
277            }
278        } else {
279            long end = (long) length * scale;
280            long i = 0;
281            ExplodeLoopNode.explodeLoop();
282            for (int iteration = 0; iteration < length; iteration++) {
283                if (i < end) {
284                    Object a = UnsafeLoadNode.load(nonNullSrc, arrayBaseOffset + i + (long) srcPos * scale, elementKind, arrayLocation);
285                    DirectObjectStoreNode.storeObject(nonNullDest, arrayBaseOffset, i + (long) destPos * scale, a, arrayLocation, elementKind);
286                    i += scale;
287                }
288            }
289        }
290    }
291
292    private static final SnippetCounter.Group checkCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy checkInputs") : null;
293    private static final SnippetCounter checkSuccessCounter = new SnippetCounter(checkCounters, "checkSuccess", "checkSuccess");
294    private static final SnippetCounter checkAIOOBECounter = new SnippetCounter(checkCounters, "checkAIOOBE", "checkAIOOBE");
295
296    private static final SnippetCounter.Group counters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy") : null;
297
298    private static final SnippetCounter objectCheckcastCounter = new SnippetCounter(counters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays");
299    private static final SnippetCounter objectCheckcastSameTypeCounter = new SnippetCounter(counters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays");
300    private static final SnippetCounter predictedObjectArrayCopySlowPathCounter = new SnippetCounter(counters, "Object[]{slow-path}", "used System.arraycopy slow path for predicted Object[] arrays");
301    private static final SnippetCounter predictedObjectArrayCopyFastPathCounter = new SnippetCounter(counters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays");
302
303    private static final EnumMap<Kind, SnippetCounter> arraycopyCallCounters = new EnumMap<>(Kind.class);
304    private static final EnumMap<Kind, SnippetCounter> arraycopyCounters = new EnumMap<>(Kind.class);
305
306    private static final EnumMap<Kind, SnippetCounter> arraycopyCallCopiedCounters = new EnumMap<>(Kind.class);
307    private static final EnumMap<Kind, SnippetCounter> arraycopyCopiedCounters = new EnumMap<>(Kind.class);
308
309    static void createArraycopyCounter(Kind kind) {
310        arraycopyCallCounters.put(kind, new SnippetCounter(counters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays"));
311        arraycopyCounters.put(kind, new SnippetCounter(counters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays"));
312
313        arraycopyCallCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{stub}", "arraycopy call for " + kind + "[] arrays"));
314        arraycopyCopiedCounters.put(kind, new SnippetCounter(copiedCounters, kind + "[]{inline}", "inline arraycopy for " + kind + "[] arrays"));
315    }
316
317    static {
318        createArraycopyCounter(Kind.Byte);
319        createArraycopyCounter(Kind.Boolean);
320        createArraycopyCounter(Kind.Char);
321        createArraycopyCounter(Kind.Short);
322        createArraycopyCounter(Kind.Int);
323        createArraycopyCounter(Kind.Long);
324        createArraycopyCounter(Kind.Float);
325        createArraycopyCounter(Kind.Double);
326        createArraycopyCounter(Kind.Object);
327    }
328
329    private static final SnippetCounter genericPrimitiveCallCounter = new SnippetCounter(counters, "genericPrimitive", "generic arraycopy snippet for primitive arrays");
330    private static final SnippetCounter genericObjectExactCallCounter = new SnippetCounter(counters, "genericObjectExact", "generic arraycopy snippet for special object arrays");
331    private static final SnippetCounter SystemArraycopyCounter = new SnippetCounter(counters, "genericObject", "call to System.arraycopy");
332
333    private static final SnippetCounter.Group lengthCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy 0-length checks") : null;
334
335    private static final SnippetCounter zeroLengthStaticCounter = new SnippetCounter(lengthCounters, "0-lengthcopy static", "arraycopy where the length is statically 0");
336    private static final SnippetCounter zeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "0-lengthcopy dynamically", "arraycopy where the length is dynamically 0");
337    private static final SnippetCounter nonZeroLengthDynamicCounter = new SnippetCounter(lengthCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero");
338
339    private static final SnippetCounter.Group copiedCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy copied elements") : null;
340
341    private static final SnippetCounter nonZeroLengthDynamicCopiedCounter = new SnippetCounter(copiedCounters, "non-0-lengthcopy dynamically", "arraycopy where the length is dynamically not zero");
342    private static final SnippetCounter genericPrimitiveCallCopiedCounter = new SnippetCounter(copiedCounters, "genericPrimitive", "generic arraycopy snippet for primitive arrays");
343    private static final SnippetCounter genericObjectExactCallCopiedCounter = new SnippetCounter(copiedCounters, "genericObjectExact", "generic arraycopy snippet for special object arrays");
344    private static final SnippetCounter SystemArraycopyCopiedCounter = new SnippetCounter(copiedCounters, "genericObject", "call to System.arraycopy");
345
346    private static final SnippetCounter objectCheckcastCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{non-exact}", "arraycopy for non-exact Object[] arrays");
347    private static final SnippetCounter objectCheckcastSameTypeCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{same-type}", "arraycopy call for src.klass == dest.klass Object[] arrays");
348    private static final SnippetCounter predictedObjectArrayCopySlowPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{slow-path}",
349                    "used System.arraycopy slow path for predicted Object[] arrays");
350    private static final SnippetCounter predictedObjectArrayCopyFastPathCopiedCounter = new SnippetCounter(copiedCounters, "Object[]{fast-path}", "used oop_arraycopy for predicted Object[] arrays");
351
352    public static class Templates extends SnippetTemplate.AbstractTemplates {
353
354        public Templates(HotSpotProviders providers, TargetDescription target) {
355            super(providers, providers.getSnippetReflection(), target);
356        }
357
358        private ResolvedJavaMethod originalArraycopy() throws JVMCIError {
359            if (originalArraycopy == null) {
360                Method method;
361                try {
362                    method = System.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class);
363                } catch (NoSuchMethodException | SecurityException e) {
364                    throw new JVMCIError(e);
365                }
366                originalArraycopy = providers.getMetaAccess().lookupJavaMethod(method);
367            }
368            return originalArraycopy;
369        }
370
371        private ResolvedJavaMethod originalArraycopy;
372
373        private final SnippetInfo checkcastArraycopyWorkSnippet = snippet("checkcastArraycopyWork");
374        private final SnippetInfo arraycopyGenericSnippet = snippet("arraycopyGeneric");
375
376        private final SnippetInfo arraycopySlowPathIntrinsicSnippet = snippet("arraycopySlowPathIntrinsic");
377        private final SnippetInfo arraycopyUnrolledIntrinsicSnippet = snippet("arraycopyUnrolledIntrinsic");
378        private final SnippetInfo arraycopyExactIntrinsicSnippet = snippet("arraycopyExactIntrinsic");
379        private final SnippetInfo arraycopyZeroLengthIntrinsicSnippet = snippet("arraycopyZeroLengthIntrinsic");
380        private final SnippetInfo arraycopyPredictedExactIntrinsicSnippet = snippet("arraycopyPredictedExactIntrinsic");
381        private final SnippetInfo arraycopyPredictedObjectWorkSnippet = snippet("arraycopyPredictedObjectWork");
382
383        private final SnippetInfo arraycopyUnrolledWorkSnippet = snippet("arraycopyUnrolledWork");
384
385        protected SnippetInfo snippet(String methodName) {
386            SnippetInfo info = snippet(ArrayCopySnippets.class, methodName, LocationIdentity.any());
387            info.setOriginalMethod(originalArraycopy());
388            return info;
389        }
390
391        public static Kind selectComponentKind(BasicArrayCopyNode arraycopy) {
392            return selectComponentKind(arraycopy, true);
393        }
394
395        public static Kind selectComponentKind(BasicArrayCopyNode arraycopy, boolean exact) {
396            ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp());
397            ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp());
398
399            if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) {
400                if (!exact) {
401                    Kind component = getComponentKind(srcType);
402                    if (component != null) {
403                        return component;
404                    }
405                    return getComponentKind(destType);
406                }
407                return null;
408            }
409            if (exact) {
410                if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) {
411                    return null;
412                }
413                if (!arraycopy.isExact()) {
414                    return null;
415                }
416            }
417            return srcType.getComponentType().getKind();
418        }
419
420        private static Kind getComponentKind(ResolvedJavaType type) {
421            if (type != null && type.isArray()) {
422                return type.getComponentType().getKind();
423            }
424            return null;
425        }
426
427        private static boolean shouldUnroll(ValueNode length) {
428            return length.isConstant() && length.asJavaConstant().asInt() <= 8 && length.asJavaConstant().asInt() != 0;
429        }
430
431        public void lower(ArrayCopyNode arraycopy, LoweringTool tool) {
432            Kind componentKind = selectComponentKind(arraycopy);
433            SnippetInfo snippetInfo = null;
434            SnippetInfo slowPathSnippetInfo = null;
435            Object slowPathArgument = null;
436
437            if (arraycopy.getLength().isConstant() && arraycopy.getLength().asJavaConstant().asLong() == 0) {
438                snippetInfo = arraycopyZeroLengthIntrinsicSnippet;
439            } else if (arraycopy.isExact()) {
440                snippetInfo = arraycopyExactIntrinsicSnippet;
441                if (shouldUnroll(arraycopy.getLength())) {
442                    snippetInfo = arraycopyUnrolledIntrinsicSnippet;
443                }
444            } else {
445                if (componentKind == Kind.Object) {
446                    ResolvedJavaType srcType = StampTool.typeOrNull(arraycopy.getSource().stamp());
447                    ResolvedJavaType destType = StampTool.typeOrNull(arraycopy.getDestination().stamp());
448                    ResolvedJavaType srcComponentType = srcType == null ? null : srcType.getComponentType();
449                    ResolvedJavaType destComponentType = destType == null ? null : destType.getComponentType();
450                    if (srcComponentType != null && destComponentType != null && !srcComponentType.isPrimitive() && !destComponentType.isPrimitive()) {
451                        snippetInfo = arraycopySlowPathIntrinsicSnippet;
452                        slowPathSnippetInfo = checkcastArraycopyWorkSnippet;
453                        slowPathArgument = LocationIdentity.any();
454                        /*
455                         * Because this snippet has to use Sysytem.arraycopy as a slow path, we must
456                         * pretend to kill any() so clear the componentKind.
457                         */
458                        componentKind = null;
459                    }
460                }
461                if (componentKind == null && snippetInfo == null) {
462                    Kind predictedKind = selectComponentKind(arraycopy, false);
463                    if (predictedKind != null) {
464                        /*
465                         * At least one array is of a known type requiring no store checks, so
466                         * assume the other is of the same type. Generally this is working around
467                         * deficiencies in our propagation of type information.
468                         */
469                        componentKind = predictedKind;
470                        if (predictedKind == Kind.Object) {
471                            snippetInfo = arraycopySlowPathIntrinsicSnippet;
472                            slowPathSnippetInfo = arraycopyPredictedObjectWorkSnippet;
473                            slowPathArgument = predictedKind;
474                            componentKind = null;
475                        } else {
476                            snippetInfo = arraycopyPredictedExactIntrinsicSnippet;
477                        }
478                    }
479                }
480                if (snippetInfo == null) {
481                    snippetInfo = arraycopyGenericSnippet;
482                }
483            }
484            Arguments args = new Arguments(snippetInfo, arraycopy.graph().getGuardsStage(), tool.getLoweringStage());
485            args.add("src", arraycopy.getSource());
486            args.add("srcPos", arraycopy.getSourcePosition());
487            args.add("dest", arraycopy.getDestination());
488            args.add("destPos", arraycopy.getDestinationPosition());
489            args.add("length", arraycopy.getLength());
490            if (snippetInfo == arraycopyUnrolledIntrinsicSnippet) {
491                args.addConst("unrolledLength", arraycopy.getLength().asJavaConstant().asInt());
492                args.addConst("elementKind", componentKind != null ? componentKind : Kind.Illegal);
493            } else if (snippetInfo == arraycopySlowPathIntrinsicSnippet) {
494                args.addConst("elementKind", componentKind != null ? componentKind : Kind.Illegal);
495                args.addConst("slowPath", slowPathSnippetInfo);
496                assert slowPathArgument != null;
497                args.addConst("slowPathArgument", slowPathArgument);
498            } else if (snippetInfo == arraycopyExactIntrinsicSnippet || snippetInfo == arraycopyPredictedExactIntrinsicSnippet) {
499                assert componentKind != null;
500                args.addConst("elementKind", componentKind);
501                args.addConst("counter", arraycopyCallCounters.get(componentKind));
502                args.addConst("copiedCounter", arraycopyCallCopiedCounters.get(componentKind));
503            }
504            instantiate(args, arraycopy);
505        }
506
507        public void lower(ArrayCopySlowPathNode arraycopy, LoweringTool tool) {
508            StructuredGraph graph = arraycopy.graph();
509            if (!graph.getGuardsStage().areFrameStatesAtDeopts()) {
510                // Can't be lowered yet
511                return;
512            }
513            SnippetInfo snippetInfo = arraycopy.getSnippet();
514            Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage());
515            args.add("nonNullSrc", arraycopy.getSource());
516            args.add("srcPos", arraycopy.getSourcePosition());
517            args.add("nonNullDest", arraycopy.getDestination());
518            args.add("destPos", arraycopy.getDestinationPosition());
519            if (snippetInfo == arraycopyUnrolledWorkSnippet) {
520                args.addConst("length", ((Integer) arraycopy.getArgument()).intValue());
521                args.addConst("elementKind", arraycopy.getElementKind());
522            } else {
523                args.add("length", arraycopy.getLength());
524            }
525            if (snippetInfo == arraycopyPredictedObjectWorkSnippet) {
526                HotSpotResolvedObjectType arrayKlass = (HotSpotResolvedObjectType) tool.getMetaAccess().lookupJavaType(Object[].class);
527                ValueNode objectArrayKlass = ConstantNode.forConstant(tool.getStampProvider().createHubStamp(true), arrayKlass.klass(), tool.getMetaAccess(), arraycopy.graph());
528                args.add("objectArrayKlass", objectArrayKlass);
529                args.addConst("counter", arraycopyCallCounters.get(Kind.Object));
530                args.addConst("copiedCounter", arraycopyCallCopiedCounters.get(Kind.Object));
531            }
532            instantiate(args, arraycopy);
533        }
534
535        public void lower(ArrayCopyUnrollNode arraycopy, LoweringTool tool) {
536            StructuredGraph graph = arraycopy.graph();
537            if (!graph.getGuardsStage().areFrameStatesAtDeopts()) {
538                // Can't be lowered yet
539                return;
540            }
541            SnippetInfo snippetInfo = arraycopyUnrolledWorkSnippet;
542            Arguments args = new Arguments(snippetInfo, graph.getGuardsStage(), tool.getLoweringStage());
543            args.add("nonNullSrc", arraycopy.getSource());
544            args.add("srcPos", arraycopy.getSourcePosition());
545            args.add("nonNullDest", arraycopy.getDestination());
546            args.add("destPos", arraycopy.getDestinationPosition());
547            args.addConst("length", arraycopy.getUnrollLength());
548            args.addConst("elementKind", arraycopy.getElementKind());
549            template(args).instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args);
550        }
551
552        /**
553         * Instantiate the snippet template and fix up the FrameState of any Invokes of
554         * System.arraycopy and propagate the captured bci in the ArrayCopySlowPathNode.
555         *
556         * @param args
557         * @param arraycopy
558         */
559        private void instantiate(Arguments args, BasicArrayCopyNode arraycopy) {
560            StructuredGraph graph = arraycopy.graph();
561            SnippetTemplate template = template(args);
562            Map<Node, Node> replacements = template.instantiate(providers.getMetaAccess(), arraycopy, SnippetTemplate.DEFAULT_REPLACER, args);
563            for (Node originalNode : replacements.keySet()) {
564                if (originalNode instanceof Invoke) {
565                    Invoke invoke = (Invoke) replacements.get(originalNode);
566                    assert invoke.asNode().graph() == graph;
567                    CallTargetNode call = invoke.callTarget();
568
569                    if (!call.targetMethod().equals(originalArraycopy)) {
570                        throw new JVMCIError("unexpected invoke %s in snippet", call.targetMethod());
571                    }
572                    // Here we need to fix the bci of the invoke
573                    InvokeNode newInvoke = graph.add(new InvokeNode(invoke.callTarget(), arraycopy.getBci()));
574                    if (arraycopy.stateDuring() != null) {
575                        newInvoke.setStateDuring(arraycopy.stateDuring());
576                    } else {
577                        assert arraycopy.stateAfter() != null;
578                        newInvoke.setStateAfter(arraycopy.stateAfter());
579                    }
580                    graph.replaceFixedWithFixed((InvokeNode) invoke.asNode(), newInvoke);
581                } else if (originalNode instanceof ArrayCopySlowPathNode) {
582                    ArrayCopySlowPathNode slowPath = (ArrayCopySlowPathNode) replacements.get(originalNode);
583                    assert arraycopy.stateAfter() != null;
584                    slowPath.setStateAfter(arraycopy.stateAfter());
585                    slowPath.setBci(arraycopy.getBci());
586                }
587            }
588        }
589    }
590}