# HG changeset patch # User Stefan Anzinger # Date 1417732467 -3600 # Node ID 5ec45cb4bf222bdfae78715dc06268fe60afaf4f # Parent 12bad81babffa6f9418cc0b98898e6adf01dd477# Parent 196cf131ed32946f2567b9973af8e848cac0a665 Merge diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.graal.api.code/src/com/oracle/graal/api/code/ReferenceMap.java --- a/graal/com.oracle.graal.api.code/src/com/oracle/graal/api/code/ReferenceMap.java Thu Dec 04 23:33:24 2014 +0100 +++ b/graal/com.oracle.graal.api.code/src/com/oracle/graal/api/code/ReferenceMap.java Thu Dec 04 23:34:27 2014 +0100 @@ -25,12 +25,16 @@ import com.oracle.graal.api.code.CodeUtil.RefMapFormatter; import com.oracle.graal.api.meta.*; -public interface ReferenceMap { +public interface ReferenceMap extends Cloneable { void setRegister(int idx, LIRKind kind); + void clearRegister(int idx, LIRKind kind); + void setStackSlot(int offset, LIRKind kind); + void clearStackSlot(int offset, LIRKind kind); + boolean hasRegisterRefMap(); boolean hasFrameRefMap(); @@ -38,4 +42,11 @@ void appendRegisterMap(StringBuilder sb, RefMapFormatter formatterArg); void appendFrameMap(StringBuilder sb, RefMapFormatter formatterArg); + + ReferenceMap clone(); + + /** + * Updates this map with all references marked in {@code other}. + */ + void updateUnion(ReferenceMap other); } diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.graal.baseline/src/com/oracle/graal/baseline/BaselineBytecodeParser.java --- a/graal/com.oracle.graal.baseline/src/com/oracle/graal/baseline/BaselineBytecodeParser.java Thu Dec 04 23:33:24 2014 +0100 +++ b/graal/com.oracle.graal.baseline/src/com/oracle/graal/baseline/BaselineBytecodeParser.java Thu Dec 04 23:34:27 2014 +0100 @@ -158,13 +158,27 @@ } try (Scope s = Debug.scope("Allocator")) { - if (backend.shouldAllocateRegisters()) { LinearScan.allocate(target, lirGenRes); + } else if (!LocationMarker.Options.UseLocationMarker.getValue()) { + // build frame map for targets that do not allocate registers + lirGenRes.buildFrameMap(); } - } catch (Throwable e) { - throw Debug.handle(e); } + if (LocationMarker.Options.UseLocationMarker.getValue()) { + try (Scope s1 = Debug.scope("BuildFrameMap")) { + // build frame map + lirGenRes.buildFrameMap(); + Debug.dump(lir, "After FrameMap building"); + } + try (Scope s1 = Debug.scope("MarkLocations")) { + if (backend.shouldAllocateRegisters()) { + // currently we mark locations only if we do register allocation + LocationMarker.markLocations(lir, lirGenRes.getFrameMap()); + } + } + } + } catch (Throwable e) { throw Debug.handle(e); } diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/GraalCompiler.java --- a/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/GraalCompiler.java Thu Dec 04 23:33:24 2014 +0100 +++ b/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/GraalCompiler.java Thu Dec 04 23:34:27 2014 +0100 @@ -351,7 +351,7 @@ try (Scope s = Debug.scope("Allocator", nodeLirGen)) { if (backend.shouldAllocateRegisters()) { LinearScan.allocate(target, lirGenRes); - } else { + } else if (!LocationMarker.Options.UseLocationMarker.getValue()) { // build frame map for targets that do not allocate registers lirGenRes.buildFrameMap(); } @@ -359,6 +359,20 @@ throw Debug.handle(e); } + if (LocationMarker.Options.UseLocationMarker.getValue()) { + try (Scope s1 = Debug.scope("BuildFrameMap")) { + // build frame map + lirGenRes.buildFrameMap(); + Debug.dump(lir, "After FrameMap building"); + } + try (Scope s1 = Debug.scope("MarkLocations")) { + if (backend.shouldAllocateRegisters()) { + // currently we mark locations only if we do register allocation + LocationMarker.markLocations(lir, lirGenRes.getFrameMap()); + } + } + } + try (Scope s = Debug.scope("ControlFlowOptimizations")) { EdgeMoveOptimizer.optimize(lir); ControlFlowOptimizer.optimize(lir, codeEmittingOrder); diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/alloc/LinearScan.java --- a/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/alloc/LinearScan.java Thu Dec 04 23:33:24 2014 +0100 +++ b/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/alloc/LinearScan.java Thu Dec 04 23:34:27 2014 +0100 @@ -1713,6 +1713,9 @@ * @see InstructionValueProcedure#doValue(LIRInstruction, Value, OperandMode, EnumSet) */ private Value debugInfoProcedure(LIRInstruction op, Value operand, OperandMode valueMode, EnumSet flags) { + if (isVirtualStackSlot(operand)) { + return operand; + } int tempOpId = op.id(); OperandMode mode = OperandMode.USE; AbstractBlock block = blockForId(tempOpId); @@ -1742,12 +1745,16 @@ } private void computeDebugInfo(IntervalWalker iw, final LIRInstruction op, LIRFrameState info) { - FrameMap frameMap = res.getFrameMap(); - info.initDebugInfo(frameMap, !op.destroysCallerSavedRegisters() || !callKillsRegisters); - markFrameLocations(iw, op, info, frameMap); + if (!LocationMarker.Options.UseLocationMarker.getValue()) { + FrameMap frameMap = res.getFrameMap(); + info.initDebugInfo(frameMap, !op.destroysCallerSavedRegisters() || !callKillsRegisters); + markFrameLocations(iw, op, info, frameMap); + } info.forEachState(op, this::debugInfoProcedure); - info.finish(op, frameMap); + if (!LocationMarker.Options.UseLocationMarker.getValue()) { + info.finish(op, res.getFrameMap()); + } } private void assignLocations(List instructions, final IntervalWalker iw) { @@ -1887,8 +1894,11 @@ // register interval mapper frameMapBuilder.requireMapping(new Mapper()); - // build frame map - res.buildFrameMap(); + + if (!LocationMarker.Options.UseLocationMarker.getValue()) { + // build frame map + res.buildFrameMap(); + } printLir("After FrameMap building", true); diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/alloc/LocationMarker.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/alloc/LocationMarker.java Thu Dec 04 23:34:27 2014 +0100 @@ -0,0 +1,218 @@ +/* + * Copyright (c) 2014, 2014, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.graal.compiler.alloc; + +import static com.oracle.graal.api.code.ValueUtil.*; + +import java.util.*; + +import com.oracle.graal.api.code.*; +import com.oracle.graal.api.meta.*; +import com.oracle.graal.compiler.common.cfg.*; +import com.oracle.graal.debug.*; +import com.oracle.graal.lir.*; +import com.oracle.graal.lir.LIRInstruction.OperandFlag; +import com.oracle.graal.lir.LIRInstruction.OperandMode; +import com.oracle.graal.lir.framemap.*; +import com.oracle.graal.options.*; + +public final class LocationMarker { + + public static class Options { + // @formatter:off + @Option(help = "Use decoupled pass for location marking (instead of using LSRA marking)") + public static final OptionValue UseLocationMarker = new OptionValue<>(true); + // @formatter:on + } + + /** + * Mark all live references for a frame state. The frame state use this information to build the + * OOP maps. + */ + public static void markLocations(LIR lir, FrameMap frameMap) { + new LocationMarker(lir, frameMap).build(); + } + + private final LIR lir; + private final FrameMap frameMap; + private final RegisterAttributes[] registerAttributes; + private final BlockMap liveInMap; + private final BlockMap liveOutMap; + + private LocationMarker(LIR lir, FrameMap frameMap) { + this.lir = lir; + this.frameMap = frameMap; + this.registerAttributes = frameMap.getRegisterConfig().getAttributesMap(); + liveInMap = new BlockMap<>(lir.getControlFlowGraph()); + liveOutMap = new BlockMap<>(lir.getControlFlowGraph()); + } + + private void build() { + Deque> worklist = new ArrayDeque<>(); + for (int i = lir.getControlFlowGraph().getBlocks().size() - 1; i >= 0; i--) { + worklist.add(lir.getControlFlowGraph().getBlocks().get(i)); + } + for (AbstractBlock block : lir.getControlFlowGraph().getBlocks()) { + liveInMap.put(block, frameMap.initReferenceMap(true)); + } + while (!worklist.isEmpty()) { + AbstractBlock block = worklist.poll(); + processBlock(block, worklist); + } + // finish states + for (AbstractBlock block : lir.getControlFlowGraph().getBlocks()) { + List instructions = lir.getLIRforBlock(block); + for (int i = instructions.size() - 1; i >= 0; i--) { + LIRInstruction inst = instructions.get(i); + inst.forEachState((op, info) -> info.finish(op, frameMap)); + } + + } + } + + /** + * Merge outSet with in-set of successors. + */ + private boolean updateOutBlock(AbstractBlock block) { + ReferenceMap union = frameMap.initReferenceMap(true); + block.getSuccessors().forEach(succ -> union.updateUnion(liveInMap.get(succ))); + ReferenceMap outSet = liveOutMap.get(block); + // check if changed + if (outSet == null || !union.equals(outSet)) { + liveOutMap.put(block, union); + return true; + } + return false; + } + + private void processBlock(AbstractBlock block, Deque> worklist) { + if (updateOutBlock(block)) { + try (Indent indent = Debug.logAndIndent("handle block %s", block)) { + BlockClosure closure = new BlockClosure(liveOutMap.get(block).clone()); + List instructions = lir.getLIRforBlock(block); + for (int i = instructions.size() - 1; i >= 0; i--) { + LIRInstruction inst = instructions.get(i); + closure.processInstructionBottomUp(inst); + } + liveInMap.put(block, closure.getCurrentSet()); + worklist.addAll(block.getPredecessors()); + } + } + } + + private static final EnumSet REGISTER_FLAG_SET = EnumSet.of(OperandFlag.REG); + private static final LIRKind REFERENCE_KIND = LIRKind.reference(Kind.Object); + + private void forEachDestroyedCallerSavedRegister(LIRInstruction op, ValueConsumer consumer) { + if (op.destroysCallerSavedRegisters()) { + for (Register reg : frameMap.getRegisterConfig().getCallerSaveRegisters()) { + consumer.visitValue(reg.asValue(REFERENCE_KIND), OperandMode.TEMP, REGISTER_FLAG_SET); + } + } + } + + private final class BlockClosure { + private final ReferenceMap currentSet; + + private BlockClosure(ReferenceMap set) { + currentSet = set; + } + + private ReferenceMap getCurrentSet() { + return currentSet; + } + + /** + * Process all values of an instruction bottom-up, i.e. definitions before usages. Values + * that start or end at the current operation are not included. + */ + private void processInstructionBottomUp(LIRInstruction op) { + try (Indent indent = Debug.logAndIndent("handle op %d, %s", op.id(), op)) { + // kills + op.visitEachTemp(this::defConsumer); + op.visitEachOutput(this::defConsumer); + forEachDestroyedCallerSavedRegister(op, this::defConsumer); + + // gen - values that are considered alive for this state + op.visitEachAlive(this::useConsumer); + op.visitEachState(this::useConsumer); + // mark locations + op.forEachState((inst, info) -> markLocation(inst, info, this.getCurrentSet())); + // gen + op.visitEachInput(this::useConsumer); + } + } + + /** + * @see InstructionValueConsumer + * @param operand + * @param mode + * @param flags + */ + private void useConsumer(Value operand, OperandMode mode, EnumSet flags) { + LIRKind kind = operand.getLIRKind(); + if (shouldProcessValue(operand) && !kind.isValue() && !kind.isDerivedReference()) { + // no need to insert values and derived reference + Debug.log("set operand: %s", operand); + frameMap.setReference(operand, currentSet); + } + } + + /** + * @see InstructionValueConsumer + * @param operand + * @param mode + * @param flags + */ + private void defConsumer(Value operand, OperandMode mode, EnumSet flags) { + if (shouldProcessValue(operand)) { + Debug.log("clear operand: %s", operand); + frameMap.clearReference(operand, currentSet); + } + } + + protected boolean shouldProcessValue(Value operand) { + return (isRegister(operand) && attributes(asRegister(operand)).isAllocatable() || isStackSlot(operand)) && operand.getKind() != Kind.Illegal; + } + } + + /** + * This method does the actual marking. + */ + private void markLocation(LIRInstruction op, LIRFrameState info, ReferenceMap refMap) { + if (!info.hasDebugInfo()) { + info.initDebugInfo(frameMap, !op.destroysCallerSavedRegisters() || !frameMap.getRegisterConfig().areAllAllocatableRegistersCallerSaved()); + } + info.updateUnion(refMap); + } + + /** + * Gets an object describing the attributes of a given register according to this register + * configuration. + * + * @see LinearScan#attributes + */ + private RegisterAttributes attributes(Register reg) { + return registerAttributes[reg.number]; + } +} diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.graal.hotspot.amd64/src/com/oracle/graal/hotspot/amd64/AMD64HotSpotRegisterConfig.java --- a/graal/com.oracle.graal.hotspot.amd64/src/com/oracle/graal/hotspot/amd64/AMD64HotSpotRegisterConfig.java Thu Dec 04 23:33:24 2014 +0100 +++ b/graal/com.oracle.graal.hotspot.amd64/src/com/oracle/graal/hotspot/amd64/AMD64HotSpotRegisterConfig.java Thu Dec 04 23:34:27 2014 +0100 @@ -91,6 +91,12 @@ private final Register[] nativeGeneralParameterRegisters; private final Register[] xmmParameterRegisters = {xmm0, xmm1, xmm2, xmm3, xmm4, xmm5, xmm6, xmm7}; + /* + * Some ABIs (e.g. Windows) require a so-called "home space", that is a save area on the stack + * to store the argument registers + */ + private final boolean needsNativeStackHomeSpace; + private final CalleeSaveLayout csl; private static Register findRegister(String name, Register[] all) { @@ -144,9 +150,11 @@ if (config.windowsOs) { javaGeneralParameterRegisters = new Register[]{rdx, r8, r9, rdi, rsi, rcx}; nativeGeneralParameterRegisters = new Register[]{rcx, rdx, r8, r9}; + this.needsNativeStackHomeSpace = true; } else { javaGeneralParameterRegisters = new Register[]{rsi, rdx, rcx, r8, r9, rdi}; nativeGeneralParameterRegisters = new Register[]{rdi, rsi, rdx, rcx, r8, r9}; + this.needsNativeStackHomeSpace = false; } csl = null; @@ -200,7 +208,7 @@ int currentGeneral = 0; int currentXMM = 0; - int currentStackOffset = 0; + int currentStackOffset = type == Type.NativeCall && needsNativeStackHomeSpace ? generalParameterRegisters.length * target.wordSize : 0; for (int i = 0; i < parameterTypes.length; i++) { final Kind kind = parameterTypes[i].getKind(); diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.graal.hotspot.hsail/src/com/oracle/graal/hotspot/hsail/HSAILHotSpotReplacementsImpl.java --- a/graal/com.oracle.graal.hotspot.hsail/src/com/oracle/graal/hotspot/hsail/HSAILHotSpotReplacementsImpl.java Thu Dec 04 23:33:24 2014 +0100 +++ b/graal/com.oracle.graal.hotspot.hsail/src/com/oracle/graal/hotspot/hsail/HSAILHotSpotReplacementsImpl.java Thu Dec 04 23:34:27 2014 +0100 @@ -28,7 +28,7 @@ import com.oracle.graal.api.code.*; import com.oracle.graal.api.meta.*; import com.oracle.graal.api.replacements.*; -import com.oracle.graal.hotspot.replacements.*; +import com.oracle.graal.hotspot.replacements.arraycopy.*; import com.oracle.graal.nodes.*; import com.oracle.graal.nodes.spi.*; import com.oracle.graal.phases.util.*; diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.graal.hotspot.loader/src/com/oracle/graal/hotspot/loader/Factory.java --- a/graal/com.oracle.graal.hotspot.loader/src/com/oracle/graal/hotspot/loader/Factory.java Thu Dec 04 23:33:24 2014 +0100 +++ b/graal/com.oracle.graal.hotspot.loader/src/com/oracle/graal/hotspot/loader/Factory.java Thu Dec 04 23:34:27 2014 +0100 @@ -26,19 +26,31 @@ import java.net.*; /** - * Utility to create a separate class loader for loading classes in {@code graal.jar} and - * {@code graal-truffle.jar}. + * Utility to create and register a separate class loader for loading Graal classes (i.e., those in + * {@code graal.jar} and {@code graal-truffle.jar}). */ public class Factory { /** + * Copy of the {@code UseGraalClassLoader} VM option. Set by the VM before the static + * initializer is called. + */ + private static boolean useGraalClassLoader; + + /** + * Registers the Graal class loader in the VM. + */ + private static native void init(ClassLoader loader); + + static { + init(useGraalClassLoader ? newClassLoader() : null); + } + + /** * Creates a new class loader for loading classes in {@code graal.jar} and - * {@code graal-truffle.jar} - * - * Called from the VM. + * {@code graal-truffle.jar}. */ - @SuppressWarnings("unused") - private static ClassLoader newClassLoader() throws MalformedURLException { + private static ClassLoader newClassLoader() { URL[] urls = {getGraalJarUrl("graal"), getGraalJarUrl("graal-truffle")}; ClassLoader parent = null; return URLClassLoader.newInstance(urls, parent); @@ -47,7 +59,7 @@ /** * Gets the URL for {@code base.jar}. */ - private static URL getGraalJarUrl(String base) throws MalformedURLException { + private static URL getGraalJarUrl(String base) { File file = new File(System.getProperty("java.home")); for (String name : new String[]{"lib", base + ".jar"}) { file = new File(file, name); @@ -57,6 +69,10 @@ throw new InternalError(file + " does not exist"); } - return file.toURI().toURL(); + try { + return file.toURI().toURL(); + } catch (MalformedURLException e) { + throw new InternalError(e); + } } } diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.graal.hotspot.test/src/com/oracle/graal/hotspot/test/ArrayCopyIntrinsificationTest.java --- a/graal/com.oracle.graal.hotspot.test/src/com/oracle/graal/hotspot/test/ArrayCopyIntrinsificationTest.java Thu Dec 04 23:33:24 2014 +0100 +++ b/graal/com.oracle.graal.hotspot.test/src/com/oracle/graal/hotspot/test/ArrayCopyIntrinsificationTest.java Thu Dec 04 23:34:27 2014 +0100 @@ -156,6 +156,19 @@ } @Test + public void testDisjointObject() { + mustIntrinsify = false; // a generic call to arraycopy will not be intrinsified + + Integer[] src1 = {1, 2, 3, 4}; + test("objectArraycopy", src1, 0, src1, 1, src1.length - 1); + + Integer[] src2 = {1, 2, 3, 4}; + test("objectArraycopy", src2, 1, src2, 0, src2.length - 1); + + mustIntrinsify = true; + } + + @Test public void testObjectExact() { Integer[] src = {1, 2, 3, 4}; testHelper("objectArraycopyExact", src); @@ -179,6 +192,10 @@ test(name, src, srcLength - length, newArray(src, length), 0, length); test(name, src, 0, newArray(src, srcLength), 0, length); } + + if (srcLength > 1) { + test(name, src, 0, src, 1, srcLength - 1); + } } public static Object genericArraycopy(Object src, int srcPos, Object dst, int dstPos, int length) { diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.graal.hotspot.test/src/com/oracle/graal/hotspot/test/WriteBarrierVerificationTest.java --- a/graal/com.oracle.graal.hotspot.test/src/com/oracle/graal/hotspot/test/WriteBarrierVerificationTest.java Thu Dec 04 23:33:24 2014 +0100 +++ b/graal/com.oracle.graal.hotspot.test/src/com/oracle/graal/hotspot/test/WriteBarrierVerificationTest.java Thu Dec 04 23:34:27 2014 +0100 @@ -35,6 +35,7 @@ import com.oracle.graal.hotspot.*; import com.oracle.graal.hotspot.nodes.*; import com.oracle.graal.hotspot.phases.*; +import com.oracle.graal.hotspot.replacements.arraycopy.*; import com.oracle.graal.nodes.*; import com.oracle.graal.nodes.extended.*; import com.oracle.graal.nodes.spi.*; @@ -610,10 +611,20 @@ @Test public void test61() { - test("test13Snippet", 1, new int[]{}); + GraphPredicate checkForUnsafeArrayCopy = graph -> graph.getNodes().filter(UnsafeArrayCopyNode.class).count() > 0 ? 1 : 0; + testPredicate("test13Snippet", checkForUnsafeArrayCopy, new int[]{}); + } + + private interface GraphPredicate { + int apply(StructuredGraph graph); } private void test(final String snippet, final int expectedBarriers, final int... removedBarrierIndices) { + GraphPredicate noCheck = noArg -> expectedBarriers; + testPredicate(snippet, noCheck, removedBarrierIndices); + } + + private void testPredicate(final String snippet, final GraphPredicate expectedBarriers, final int... removedBarrierIndices) { try (Scope d = Debug.scope("WriteBarrierVerificationTest", new DebugDumpScope(snippet))) { final StructuredGraph graph = parseEager(snippet); HighTierContext highTierContext = new HighTierContext(getProviders(), new Assumptions(false), null, getDefaultGraphBuilderSuite(), OptimisticOptimizations.ALL); @@ -633,10 +644,10 @@ if (config.useG1GC) { barriers = graph.getNodes().filter(G1PreWriteBarrier.class).count() + graph.getNodes().filter(G1PostWriteBarrier.class).count() + graph.getNodes().filter(G1ArrayRangePreWriteBarrier.class).count() + graph.getNodes().filter(G1ArrayRangePostWriteBarrier.class).count(); - Assert.assertTrue(expectedBarriers * 2 == barriers); + Assert.assertTrue(expectedBarriers.apply(graph) * 2 == barriers); } else { barriers = graph.getNodes().filter(SerialWriteBarrier.class).count() + graph.getNodes().filter(SerialArrayRangeWriteBarrier.class).count(); - Assert.assertTrue(expectedBarriers == barriers); + Assert.assertTrue(expectedBarriers.apply(graph) == barriers); } // Iterate over all write nodes and remove barriers according to input indices. NodeIteratorClosure closure = new NodeIteratorClosure() { diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/HotSpotReferenceMap.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/HotSpotReferenceMap.java Thu Dec 04 23:33:24 2014 +0100 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/HotSpotReferenceMap.java Thu Dec 04 23:34:27 2014 +0100 @@ -79,6 +79,19 @@ this.target = target; } + private HotSpotReferenceMap(HotSpotReferenceMap other) { + this.registerRefMap = (BitSet) other.registerRefMap.clone(); + this.frameRefMap = (BitSet) other.frameRefMap.clone(); + this.target = other.target; + } + + @Override + public ReferenceMap clone() { + return new HotSpotReferenceMap(this); + } + + // setters + private static void setOop(BitSet map, int startIdx, LIRKind kind) { int length = kind.getPlatformKind().getVectorLength(); map.clear(BITS_PER_WORD * startIdx, BITS_PER_WORD * (startIdx + length) - 1); @@ -154,6 +167,194 @@ } } + // clear + + private static void clearOop(BitSet map, int startIdx, LIRKind kind) { + int length = kind.getPlatformKind().getVectorLength(); + map.clear(BITS_PER_WORD * startIdx, BITS_PER_WORD * (startIdx + length) - 1); + } + + private static void clearNarrowOop(BitSet map, int idx, LIRKind kind) { + int length = kind.getPlatformKind().getVectorLength(); + int nextIdx = idx + (length + 1) / 2; + map.clear(BITS_PER_WORD * idx, BITS_PER_WORD * nextIdx - 1); + } + + public void clearRegister(int idx, LIRKind kind) { + + PlatformKind platformKind = kind.getPlatformKind(); + int bytesPerElement = target.getSizeInBytes(platformKind) / platformKind.getVectorLength(); + + if (bytesPerElement == target.wordSize) { + clearOop(registerRefMap, idx, kind); + } else if (bytesPerElement == target.wordSize / 2) { + clearNarrowOop(registerRefMap, idx, kind); + } else { + assert kind.isValue() : "unsupported reference kind " + kind; + } + } + + public void clearStackSlot(int offset, LIRKind kind) { + + PlatformKind platformKind = kind.getPlatformKind(); + int bytesPerElement = target.getSizeInBytes(platformKind) / platformKind.getVectorLength(); + assert offset % bytesPerElement == 0 : "unaligned value in ReferenceMap"; + + if (bytesPerElement == target.wordSize) { + clearOop(frameRefMap, offset / target.wordSize, kind); + } else if (bytesPerElement == target.wordSize / 2) { + if (platformKind.getVectorLength() > 1) { + clearNarrowOop(frameRefMap, offset / target.wordSize, kind); + } else { + // in this case, offset / target.wordSize may not divide evenly + // so setNarrowOop won't work correctly + int idx = offset / target.wordSize; + if (kind.isReference(0)) { + if (offset % target.wordSize == 0) { + frameRefMap.clear(BITS_PER_WORD * idx + 1); + if (!frameRefMap.get(BITS_PER_WORD * idx + 2)) { + // only reset the first bit if there is no other narrow oop + frameRefMap.clear(BITS_PER_WORD * idx); + } + } else { + frameRefMap.clear(BITS_PER_WORD * idx + 2); + if (!frameRefMap.get(BITS_PER_WORD * idx + 1)) { + // only reset the first bit if there is no other narrow oop + frameRefMap.clear(BITS_PER_WORD * idx); + } + } + } + } + } else { + assert kind.isValue() : "unknown reference kind " + kind; + } + } + + public void updateUnion(ReferenceMap otherArg) { + HotSpotReferenceMap other = (HotSpotReferenceMap) otherArg; + if (registerRefMap != null) { + assert other.registerRefMap != null; + updateUnionBitSetRaw(registerRefMap, other.registerRefMap); + } else { + assert other.registerRefMap == null || other.registerRefMap.cardinality() == 0 : "Target register reference map is empty but the source is not: " + other.registerRefMap; + } + updateUnionBitSetRaw(frameRefMap, other.frameRefMap); + } + + /** + * Update {@code src} with the union of {@code src} and {@code dst}. + * + * @see HotSpotReferenceMap#registerRefMap + * @see HotSpotReferenceMap#frameRefMap + */ + private static void updateUnionBitSetRaw(BitSet dst, BitSet src) { + assert dst.size() == src.size(); + assert UpdateUnionVerifier.verifyUpate(dst, src); + dst.or(src); + } + + private enum UpdateUnionVerifier { + NoReference, + WideOop, + NarrowOopLowerHalf, + NarrowOopUpperHalf, + TwoNarrowOops, + Illegal; + + /** + * Create enum values from BitSet. + *

+ * These bits can have the following values (LSB first): + * + * 

+         * 000 - contains no references
+         * 100 - contains a wide oop
+         * 110 - contains a narrow oop in the lower half
+         * 101 - contains a narrow oop in the upper half
+         * 111 - contains two narrow oops
+         *
+ * + * @see HotSpotReferenceMap#registerRefMap + * @see HotSpotReferenceMap#frameRefMap + */ + static UpdateUnionVerifier getFromBits(int idx, BitSet set) { + int n = (set.get(idx) ? 1 : 0) << 0 | (set.get(idx + 1) ? 1 : 0) << 1 | (set.get(idx + 2) ? 1 : 0) << 2; + switch (n) { + case 0: + return NoReference; + case 1: + return WideOop; + case 3: + return NarrowOopLowerHalf; + case 5: + return NarrowOopUpperHalf; + case 7: + return TwoNarrowOops; + default: + return Illegal; + } + } + + String toBitString() { + int bits = toBit(this); + if (bits == -1) { + return "---"; + } + return String.format("%3s", Integer.toBinaryString(bits)).replace(' ', '0'); + } + + static int toBit(UpdateUnionVerifier type) { + switch (type) { + case NoReference: + return 0; + case WideOop: + return 1; + case NarrowOopLowerHalf: + return 3; + case NarrowOopUpperHalf: + return 5; + case TwoNarrowOops: + return 7; + default: + return -1; + } + } + + private static boolean verifyUpate(BitSet dst, BitSet src) { + for (int idx = 0; idx < dst.size(); idx += BITS_PER_WORD) { + if (!verifyUpdateEntry(idx, dst, src)) { + return false; + } + } + return true; + } + + private static boolean verifyUpdateEntry(int idx, BitSet dst, BitSet src) { + UpdateUnionVerifier dstType = UpdateUnionVerifier.getFromBits(idx, dst); + UpdateUnionVerifier srcType = UpdateUnionVerifier.getFromBits(idx, src); + + if (dstType == UpdateUnionVerifier.Illegal || srcType == UpdateUnionVerifier.Illegal) { + assert false : String.format("Illegal RefMap bit pattern: %s (0b%s), %s (0b%s)", dstType, dstType.toBitString(), srcType, srcType.toBitString()); + return false; + } + switch (dstType) { + case NoReference: + return true; + case WideOop: + switch (srcType) { + case NoReference: + case WideOop: + return true; + default: + assert false : String.format("Illegal RefMap combination: %s (0b%s), %s (0b%s)", dstType, dstType.toBitString(), srcType, srcType.toBitString()); + return false; + } + default: + return true; + } + } + } + @Override public int hashCode() { throw new UnsupportedOperationException(); diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/DefaultHotSpotLoweringProvider.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/DefaultHotSpotLoweringProvider.java Thu Dec 04 23:33:24 2014 +0100 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/DefaultHotSpotLoweringProvider.java Thu Dec 04 23:34:27 2014 +0100 @@ -40,6 +40,7 @@ import com.oracle.graal.hotspot.nodes.*; import com.oracle.graal.hotspot.nodes.type.*; import com.oracle.graal.hotspot.replacements.*; +import com.oracle.graal.hotspot.replacements.arraycopy.*; import com.oracle.graal.java.*; import com.oracle.graal.nodes.*; import com.oracle.graal.nodes.CallTargetNode.InvokeKind; diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/HotSpotHostForeignCallsProvider.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/HotSpotHostForeignCallsProvider.java Thu Dec 04 23:33:24 2014 +0100 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/HotSpotHostForeignCallsProvider.java Thu Dec 04 23:34:27 2014 +0100 @@ -66,6 +66,10 @@ stub.getLinkage().setCompiledStub(stub); } + public static ForeignCallDescriptor lookupCheckcastArraycopyDescriptor(boolean uninit) { + return checkcastArraycopyDescriptors[uninit ? 1 : 0]; + } + public static ForeignCallDescriptor lookupArraycopyDescriptor(Kind kind, boolean aligned, boolean disjoint, boolean uninit) { if (uninit) { assert kind == Kind.Object; @@ -77,6 +81,7 @@ @SuppressWarnings("unchecked") private static final EnumMap[][] arraycopyDescriptors = new EnumMap[2][2]; private static final ForeignCallDescriptor[][] uninitObjectArraycopyDescriptors = new ForeignCallDescriptor[2][2]; + private static final ForeignCallDescriptor[] checkcastArraycopyDescriptors = new ForeignCallDescriptor[2]; static { // Populate the EnumMap instances @@ -104,6 +109,21 @@ } } + private void registerCheckcastArraycopyDescriptor(boolean uninit, long routine) { + String name = "Object" + (uninit ? "Uninit" : "") + "Checkcast"; + // Input: + // c_rarg0 - source array address + // c_rarg1 - destination array address + // c_rarg2 - element count, treated as ssize_t, can be zero + // c_rarg3 - size_t ckoff (super_check_offset) + // c_rarg4 - oop ckval (super_klass) + // return: 0 = success, n = number of copied elements xor'd with -1. + ForeignCallDescriptor desc = new ForeignCallDescriptor(name, int.class, Word.class, Word.class, Word.class, Word.class, Word.class); + LocationIdentity killed = NamedLocationIdentity.getArrayLocation(Kind.Object); + registerForeignCall(desc, routine, NativeCall, DESTROYS_REGISTERS, LEAF_NOFP, NOT_REEXECUTABLE, killed); + checkcastArraycopyDescriptors[uninit ? 1 : 0] = desc; + } + private void registerArrayCopy(Map descMap, Kind kind, long routine, long alignedRoutine, long disjointRoutine, long alignedDisjointRoutine) { registerArrayCopy(descMap, kind, routine, alignedRoutine, disjointRoutine, alignedDisjointRoutine, false); } @@ -189,6 +209,9 @@ registerArrayCopy(descMap, Kind.Object, c.oopArraycopy, c.oopAlignedArraycopy, c.oopDisjointArraycopy, c.oopAlignedDisjointArraycopy); registerArrayCopy(descMap, Kind.Object, c.oopArraycopyUninit, c.oopAlignedArraycopyUninit, c.oopDisjointArraycopyUninit, c.oopAlignedDisjointArraycopyUninit, true); + registerCheckcastArraycopyDescriptor(true, c.checkcastArraycopyUninit); + registerCheckcastArraycopyDescriptor(false, c.checkcastArraycopy); + if (c.useAESIntrinsics) { /* * When the java.ext.dirs property is modified then the crypto classes might not be diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ArrayCopyCallNode.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ArrayCopyCallNode.java Thu Dec 04 23:33:24 2014 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,209 +0,0 @@ -/* - * Copyright (c) 2014, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -//JaCoCo Exclude -package com.oracle.graal.hotspot.replacements; - -import static com.oracle.graal.api.meta.LocationIdentity.*; - -import com.oracle.graal.api.meta.*; -import com.oracle.graal.api.runtime.*; -import com.oracle.graal.compiler.common.*; -import com.oracle.graal.compiler.common.type.*; -import com.oracle.graal.hotspot.*; -import com.oracle.graal.hotspot.meta.*; -import com.oracle.graal.hotspot.nodes.*; -import com.oracle.graal.nodeinfo.*; -import com.oracle.graal.nodes.*; -import com.oracle.graal.nodes.calc.*; -import com.oracle.graal.nodes.extended.*; -import com.oracle.graal.nodes.spi.*; -import com.oracle.graal.replacements.SnippetTemplate.Arguments; -import com.oracle.graal.runtime.*; - -@NodeInfo(allowedUsageTypes = {InputType.Memory}) -public class ArrayCopyCallNode extends AbstractMemoryCheckpoint implements Lowerable, MemoryCheckpoint.Single { - - @Input ValueNode src; - @Input ValueNode srcPos; - @Input ValueNode dest; - @Input ValueNode destPos; - @Input ValueNode length; - - protected Kind elementKind; - - /** - * Aligned means that the offset of the copy is heap word aligned. - */ - protected boolean aligned; - protected boolean disjoint; - protected boolean uninitialized; - - protected final HotSpotGraalRuntimeProvider runtime; - - public static ArrayCopyCallNode create(@InjectedNodeParameter HotSpotGraalRuntimeProvider runtime, ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, - Kind elementKind, boolean aligned, boolean disjoint, boolean uninitialized) { - return new ArrayCopyCallNode(src, srcPos, dest, destPos, length, elementKind, aligned, disjoint, uninitialized, runtime); - } - - protected ArrayCopyCallNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, Kind elementKind, boolean aligned, boolean disjoint, boolean uninitialized, - HotSpotGraalRuntimeProvider runtime) { - super(StampFactory.forVoid()); - assert elementKind != null; - this.src = src; - this.srcPos = srcPos; - this.dest = dest; - this.destPos = destPos; - this.length = length; - this.elementKind = elementKind; - this.aligned = aligned; - this.disjoint = disjoint; - this.uninitialized = uninitialized; - this.runtime = runtime; - } - - public static ArrayCopyCallNode create(@InjectedNodeParameter HotSpotGraalRuntimeProvider runtime, ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, - Kind elementKind, boolean disjoint) { - return new ArrayCopyCallNode(src, srcPos, dest, destPos, length, elementKind, false, disjoint, false, runtime); - } - - public ValueNode getSource() { - return src; - } - - public ValueNode getSourcePosition() { - return srcPos; - } - - public ValueNode getDestination() { - return dest; - } - - public ValueNode getDestinationPosition() { - return destPos; - } - - public ValueNode getLength() { - return length; - } - - public void addSnippetArguments(Arguments args) { - args.add("src", src); - args.add("srcPos", srcPos); - args.add("dest", dest); - args.add("destPos", destPos); - args.add("length", length); - } - - public Kind getElementKind() { - return elementKind; - } - - private boolean shouldUnroll() { - return getLength().isConstant() && getLength().asJavaConstant().asInt() <= GraalOptions.MaximumEscapeAnalysisArrayLength.getValue(); - } - - private ValueNode computeBase(ValueNode base, ValueNode pos) { - FixedWithNextNode basePtr = graph().add(GetObjectAddressNode.create(base)); - graph().addBeforeFixed(this, basePtr); - ValueNode loc = IndexedLocationNode.create(getLocationIdentity(), elementKind, runtime.getArrayBaseOffset(elementKind), pos, graph(), runtime.getArrayIndexScale(elementKind)); - return graph().unique(ComputeAddressNode.create(basePtr, loc, StampFactory.forKind(Kind.Long))); - } - - @Override - public void lower(LoweringTool tool) { - if (graph().getGuardsStage() == StructuredGraph.GuardsStage.AFTER_FSA) { - updateAlignedDisjoint(); - ForeignCallDescriptor desc = HotSpotHostForeignCallsProvider.lookupArraycopyDescriptor(elementKind, isAligned(), isDisjoint(), isUninitialized()); - StructuredGraph graph = graph(); - ValueNode srcAddr = computeBase(getSource(), getSourcePosition()); - ValueNode destAddr = computeBase(getDestination(), getDestinationPosition()); - ValueNode len = getLength(); - if (len.stamp().getStackKind() != Kind.Long) { - len = IntegerConvertNode.convert(len, StampFactory.forKind(Kind.Long), graph()); - } - ForeignCallNode call = graph.add(ForeignCallNode.create(Graal.getRequiredCapability(RuntimeProvider.class).getHostBackend().getForeignCalls(), desc, srcAddr, destAddr, len)); - call.setStateAfter(stateAfter()); - graph.replaceFixedWithFixed(this, call); - - } - } - - @Override - public LocationIdentity getLocationIdentity() { - if (elementKind != null) { - return NamedLocationIdentity.getArrayLocation(elementKind); - } - return ANY_LOCATION; - } - - @NodeIntrinsic - private static native void arraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter Kind elementKind, @ConstantNodeParameter boolean aligned, - @ConstantNodeParameter boolean disjoint, @ConstantNodeParameter boolean uninitialized); - - public static void arraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter Kind elementKind, boolean aligned, boolean disjoint) { - arraycopy(src, srcPos, dest, destPos, length, elementKind, aligned, disjoint, false); - } - - public static void disjointArraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter Kind elementKind) { - arraycopy(src, srcPos, dest, destPos, length, elementKind, false, true, false); - } - - public static void disjointUninitializedArraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter Kind elementKind) { - arraycopy(src, srcPos, dest, destPos, length, elementKind, false, true, true); - } - - public boolean isAligned() { - return aligned; - } - - public boolean isDisjoint() { - return disjoint; - } - - public boolean isUninitialized() { - return uninitialized; - } - - boolean isHeapWordAligned(JavaConstant value, Kind kind) { - return (runtime.getArrayBaseOffset(kind) + (long) value.asInt() * runtime.getArrayIndexScale(kind)) % runtime.getConfig().heapWordSize == 0; - } - - public void updateAlignedDisjoint() { - Kind componentKind = elementKind; - if (srcPos == destPos) { - // Can treat as disjoint - disjoint = true; - } - PrimitiveConstant constantSrc = (PrimitiveConstant) srcPos.stamp().asConstant(); - PrimitiveConstant constantDst = (PrimitiveConstant) destPos.stamp().asConstant(); - if (constantSrc != null && constantDst != null) { - if (!aligned) { - aligned = isHeapWordAligned(constantSrc, componentKind) && isHeapWordAligned(constantDst, componentKind); - } - if (constantSrc.asInt() >= constantDst.asInt()) { - // low to high copy so treat as disjoint - disjoint = true; - } - } - } -} diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ArrayCopyNode.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ArrayCopyNode.java Thu Dec 04 23:33:24 2014 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,139 +0,0 @@ -/* - * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package com.oracle.graal.hotspot.replacements; - -import static com.oracle.graal.compiler.GraalCompiler.*; - -import com.oracle.graal.api.meta.*; -import com.oracle.graal.compiler.common.*; -import com.oracle.graal.debug.*; -import com.oracle.graal.debug.Debug.Scope; -import com.oracle.graal.loop.phases.*; -import com.oracle.graal.nodeinfo.*; -import com.oracle.graal.nodes.*; -import com.oracle.graal.nodes.spi.*; -import com.oracle.graal.nodes.type.*; -import com.oracle.graal.phases.common.*; -import com.oracle.graal.phases.tiers.*; -import com.oracle.graal.replacements.nodes.*; - -@NodeInfo -public class ArrayCopyNode extends BasicArrayCopyNode implements Virtualizable, Lowerable { - - public static ArrayCopyNode create(Invoke invoke) { - return new ArrayCopyNode(invoke); - } - - protected ArrayCopyNode(Invoke invoke) { - super(invoke); - } - - private StructuredGraph selectSnippet(LoweringTool tool, final Replacements replacements) { - ResolvedJavaType srcType = StampTool.typeOrNull(getSource().stamp()); - ResolvedJavaType destType = StampTool.typeOrNull(getDestination().stamp()); - - if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) { - return null; - } - if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) { - return null; - } - if (!isExact()) { - return null; - } - Kind componentKind = srcType.getComponentType().getKind(); - final ResolvedJavaMethod snippetMethod = tool.getMetaAccess().lookupJavaMethod(ArrayCopySnippets.getSnippetForKind(componentKind, shouldUnroll(), isExact())); - try (Scope s = Debug.scope("ArrayCopySnippet", snippetMethod)) { - return replacements.getSnippet(snippetMethod); - } catch (Throwable e) { - throw Debug.handle(e); - } - } - - private static void unrollFixedLengthLoop(StructuredGraph snippetGraph, int length, LoweringTool tool) { - ParameterNode lengthParam = snippetGraph.getParameter(4); - if (lengthParam != null) { - snippetGraph.replaceFloating(lengthParam, ConstantNode.forInt(length, snippetGraph)); - } - // the canonicalization before loop unrolling is needed to propagate the length into - // additions, etc. - PhaseContext context = new PhaseContext(tool.getMetaAccess(), tool.getConstantReflection(), tool.getLowerer(), tool.getReplacements(), tool.assumptions(), tool.getStampProvider()); - new CanonicalizerPhase(true).apply(snippetGraph, context); - new LoopFullUnrollPhase(new CanonicalizerPhase(true)).apply(snippetGraph, context); - new CanonicalizerPhase(true).apply(snippetGraph, context); - } - - @Override - protected StructuredGraph getLoweredSnippetGraph(final LoweringTool tool) { - if (!shouldIntrinsify(getTargetMethod())) { - return null; - } - - final Replacements replacements = tool.getReplacements(); - StructuredGraph snippetGraph = selectSnippet(tool, replacements); - if (snippetGraph == null) { - final ResolvedJavaMethod snippetMethod = tool.getMetaAccess().lookupJavaMethod(ArrayCopySnippets.genericArraycopySnippet); - snippetGraph = null; - try (Scope s = Debug.scope("ArrayCopySnippet", snippetMethod)) { - snippetGraph = replacements.getSnippet(snippetMethod, getTargetMethod()).copy(); - } catch (Throwable e) { - throw Debug.handle(e); - } - replaceSnippetInvokes(snippetGraph); - } else { - assert snippetGraph != null : "ArrayCopySnippets should be installed"; - snippetGraph = snippetGraph.copy(); - if (shouldUnroll()) { - final StructuredGraph copy = snippetGraph; - try (Scope s = Debug.scope("ArrayCopySnippetSpecialization", snippetGraph.method())) { - unrollFixedLengthLoop(copy, getLength().asJavaConstant().asInt(), tool); - } catch (Throwable e) { - throw Debug.handle(e); - } - } - } - return lowerReplacement(snippetGraph, tool); - } - - private boolean shouldUnroll() { - return getLength().isConstant() && getLength().asJavaConstant().asInt() <= GraalOptions.MaximumEscapeAnalysisArrayLength.getValue(); - } - - /* - * Returns true if this copy doesn't require store checks. Trivially true for primitive arrays. - */ - private boolean isExact() { - ResolvedJavaType srcType = StampTool.typeOrNull(getSource().stamp()); - if (srcType.getComponentType().getKind().isPrimitive() || getSource() == getDestination()) { - return true; - } - - ResolvedJavaType destType = StampTool.typeOrNull(getDestination().stamp()); - if (StampTool.isExactType(getDestination().stamp())) { - if (destType != null && destType.isAssignableFrom(srcType)) { - return true; - } - } - return false; - } -} diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ArrayCopySnippets.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ArrayCopySnippets.java Thu Dec 04 23:33:24 2014 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,293 +0,0 @@ -/* - * Copyright (c) 2011, 2014, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package com.oracle.graal.hotspot.replacements; - -import static com.oracle.graal.compiler.common.GraalOptions.*; -import static com.oracle.graal.hotspot.replacements.HotSpotReplacementsUtil.*; -import static com.oracle.graal.nodes.GuardingPiNode.*; -import static com.oracle.graal.nodes.extended.BranchProbabilityNode.*; - -import java.lang.reflect.*; -import java.util.*; - -import com.oracle.graal.api.meta.*; -import com.oracle.graal.compiler.common.*; -import com.oracle.graal.graph.Node.ConstantNodeParameter; -import com.oracle.graal.graph.Node.NodeIntrinsic; -import com.oracle.graal.hotspot.word.*; -import com.oracle.graal.nodes.*; -import com.oracle.graal.nodes.extended.*; -import com.oracle.graal.nodes.java.*; -import com.oracle.graal.replacements.*; -import com.oracle.graal.word.*; - -public class ArrayCopySnippets implements Snippets { - - private static final EnumMap arraycopyMethods = new EnumMap<>(Kind.class); - private static final EnumMap arraycopyCalls = new EnumMap<>(Kind.class); - - public static final Method genericArraycopySnippet; - - private static void addArraycopySnippetMethod(Kind kind, Class arrayClass) throws NoSuchMethodException { - arraycopyMethods.put(kind, ArrayCopySnippets.class.getDeclaredMethod("arraycopy", arrayClass, int.class, arrayClass, int.class, int.class)); - if (CallArrayCopy.getValue()) { - if (kind == Kind.Object) { - arraycopyCalls.put(kind, ArrayCopySnippets.class.getDeclaredMethod("objectArraycopyUnchecked", arrayClass, int.class, arrayClass, int.class, int.class)); - } else { - arraycopyCalls.put(kind, ArrayCopySnippets.class.getDeclaredMethod(kind + "Arraycopy", arrayClass, int.class, arrayClass, int.class, int.class)); - } - } - } - - static { - try { - addArraycopySnippetMethod(Kind.Byte, byte[].class); - addArraycopySnippetMethod(Kind.Boolean, boolean[].class); - addArraycopySnippetMethod(Kind.Char, char[].class); - addArraycopySnippetMethod(Kind.Short, short[].class); - addArraycopySnippetMethod(Kind.Int, int[].class); - addArraycopySnippetMethod(Kind.Long, long[].class); - addArraycopySnippetMethod(Kind.Float, float[].class); - addArraycopySnippetMethod(Kind.Double, double[].class); - addArraycopySnippetMethod(Kind.Object, Object[].class); - genericArraycopySnippet = ArrayCopySnippets.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class); - } catch (SecurityException | NoSuchMethodException e) { - throw new GraalInternalError(e); - } - } - - public static Method getSnippetForKind(Kind kind, boolean shouldUnroll, boolean exact) { - Method m = null; - if (!shouldUnroll && exact) { - m = arraycopyCalls.get(kind); - if (m != null) { - return m; - } - } - return arraycopyMethods.get(kind); - } - - private static void checkedCopy(Object src, int srcPos, Object dest, int destPos, int length, Kind baseKind) { - Object nonNullSrc = guardingNonNull(src); - Object nonNullDest = guardingNonNull(dest); - checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); - UnsafeArrayCopyNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, baseKind); - } - - private static int checkArrayType(KlassPointer hub) { - int layoutHelper = readLayoutHelper(hub); - if (probability(SLOW_PATH_PROBABILITY, layoutHelper >= 0)) { - DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); - } - return layoutHelper; - } - - private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length) { - if (probability(SLOW_PATH_PROBABILITY, srcPos < 0)) { - checkAIOOBECounter.inc(); - DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); - } - if (probability(SLOW_PATH_PROBABILITY, destPos < 0)) { - checkAIOOBECounter.inc(); - DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); - } - if (probability(SLOW_PATH_PROBABILITY, length < 0)) { - checkAIOOBECounter.inc(); - DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); - } - if (probability(SLOW_PATH_PROBABILITY, srcPos + length > ArrayLengthNode.arrayLength(src))) { - checkAIOOBECounter.inc(); - DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); - } - if (probability(SLOW_PATH_PROBABILITY, destPos + length > ArrayLengthNode.arrayLength(dest))) { - checkAIOOBECounter.inc(); - DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); - } - checkSuccessCounter.inc(); - } - - @Snippet - public static void arraycopy(byte[] src, int srcPos, byte[] dest, int destPos, int length) { - byteCounter.inc(); - checkedCopy(src, srcPos, dest, destPos, length, Kind.Byte); - } - - @Snippet - public static void arraycopy(boolean[] src, int srcPos, boolean[] dest, int destPos, int length) { - booleanCounter.inc(); - checkedCopy(src, srcPos, dest, destPos, length, Kind.Boolean); - } - - @Snippet - public static void arraycopy(char[] src, int srcPos, char[] dest, int destPos, int length) { - charCounter.inc(); - checkedCopy(src, srcPos, dest, destPos, length, Kind.Char); - } - - @Snippet - public static void arraycopy(short[] src, int srcPos, short[] dest, int destPos, int length) { - shortCounter.inc(); - checkedCopy(src, srcPos, dest, destPos, length, Kind.Short); - } - - @Snippet - public static void arraycopy(int[] src, int srcPos, int[] dest, int destPos, int length) { - intCounter.inc(); - checkedCopy(src, srcPos, dest, destPos, length, Kind.Int); - } - - @Snippet - public static void arraycopy(float[] src, int srcPos, float[] dest, int destPos, int length) { - floatCounter.inc(); - checkedCopy(src, srcPos, dest, destPos, length, Kind.Float); - } - - @Snippet - public static void arraycopy(long[] src, int srcPos, long[] dest, int destPos, int length) { - longCounter.inc(); - checkedCopy(src, srcPos, dest, destPos, length, Kind.Long); - } - - @Snippet - public static void arraycopy(double[] src, int srcPos, double[] dest, int destPos, int length) { - doubleCounter.inc(); - checkedCopy(src, srcPos, dest, destPos, length, Kind.Double); - } - - @Snippet - public static void arraycopy(Object[] src, int srcPos, Object[] dest, int destPos, int length) { - objectCounter.inc(); - checkedCopy(src, srcPos, dest, destPos, length, Kind.Object); - } - - @Snippet - public static void arraycopy(Object src, int srcPos, Object dest, int destPos, int length) { - Object nonNullSrc = guardingNonNull(src); - Object nonNullDest = guardingNonNull(dest); - KlassPointer srcHub = loadHub(nonNullSrc); - KlassPointer destHub = loadHub(nonNullDest); - if (probability(FAST_PATH_PROBABILITY, srcHub.equal(destHub)) && probability(FAST_PATH_PROBABILITY, nonNullSrc != nonNullDest)) { - int layoutHelper = checkArrayType(srcHub); - final boolean isObjectArray = ((layoutHelper & layoutHelperElementTypePrimitiveInPlace()) == 0); - - checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); - if (probability(FAST_PATH_PROBABILITY, isObjectArray)) { - genericObjectExactCallCounter.inc(); - UnsafeArrayCopyNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, Kind.Object); - } else { - genericPrimitiveCallCounter.inc(); - UnsafeArrayCopyNode.arraycopyPrimitive(nonNullSrc, srcPos, nonNullDest, destPos, length, layoutHelper); - } - } else { - genericObjectCallCounter.inc(); - System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length); - } - } - - @NodeIntrinsic(ForeignCallNode.class) - public static native void callArraycopy(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word src, Word dest, Word len); - - private static void callArraycopyTemplate(SnippetCounter counter, Kind kind, boolean aligned, boolean disjoint, Object src, int srcPos, Object dest, int destPos, int length) { - counter.inc(); - Object nonNullSrc = guardingNonNull(src); - Object nonNullDest = guardingNonNull(dest); - checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); - ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, kind, aligned, disjoint); - } - - @Snippet - public static void objectArraycopyUnchecked(Object[] src, int srcPos, Object[] dest, int destPos, int length) { - callArraycopyTemplate(objectCallCounter, Kind.Object, false, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void byteArraycopy(byte[] src, int srcPos, byte[] dest, int destPos, int length) { - callArraycopyTemplate(byteCallCounter, Kind.Byte, false, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void booleanArraycopy(boolean[] src, int srcPos, boolean[] dest, int destPos, int length) { - callArraycopyTemplate(booleanCallCounter, Kind.Boolean, false, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void charArraycopy(char[] src, int srcPos, char[] dest, int destPos, int length) { - callArraycopyTemplate(charCallCounter, Kind.Char, false, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void shortArraycopy(short[] src, int srcPos, short[] dest, int destPos, int length) { - callArraycopyTemplate(shortCallCounter, Kind.Short, false, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void intArraycopy(int[] src, int srcPos, int[] dest, int destPos, int length) { - callArraycopyTemplate(intCallCounter, Kind.Int, false, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void floatArraycopy(float[] src, int srcPos, float[] dest, int destPos, int length) { - callArraycopyTemplate(floatCallCounter, Kind.Float, false, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void longArraycopy(long[] src, int srcPos, long[] dest, int destPos, int length) { - callArraycopyTemplate(longCallCounter, Kind.Long, false, false, src, srcPos, dest, destPos, length); - } - - @Snippet - public static void doubleArraycopy(double[] src, int srcPos, double[] dest, int destPos, int length) { - callArraycopyTemplate(doubleCallCounter, Kind.Double, false, false, src, srcPos, dest, destPos, length); - } - - private static final SnippetCounter.Group checkCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy checkInputs") : null; - private static final SnippetCounter checkSuccessCounter = new SnippetCounter(checkCounters, "checkSuccess", "checkSuccess"); - private static final SnippetCounter checkNPECounter = new SnippetCounter(checkCounters, "checkNPE", "checkNPE"); - private static final SnippetCounter checkAIOOBECounter = new SnippetCounter(checkCounters, "checkAIOOBE", "checkAIOOBE"); - - private static final SnippetCounter.Group counters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy") : null; - private static final SnippetCounter byteCounter = new SnippetCounter(counters, "byte[]", "arraycopy for byte[] arrays"); - private static final SnippetCounter charCounter = new SnippetCounter(counters, "char[]", "arraycopy for char[] arrays"); - private static final SnippetCounter shortCounter = new SnippetCounter(counters, "short[]", "arraycopy for short[] arrays"); - private static final SnippetCounter intCounter = new SnippetCounter(counters, "int[]", "arraycopy for int[] arrays"); - private static final SnippetCounter booleanCounter = new SnippetCounter(counters, "boolean[]", "arraycopy for boolean[] arrays"); - private static final SnippetCounter longCounter = new SnippetCounter(counters, "long[]", "arraycopy for long[] arrays"); - private static final SnippetCounter objectCounter = new SnippetCounter(counters, "Object[]", "arraycopy for Object[] arrays"); - private static final SnippetCounter floatCounter = new SnippetCounter(counters, "float[]", "arraycopy for float[] arrays"); - private static final SnippetCounter doubleCounter = new SnippetCounter(counters, "double[]", "arraycopy for double[] arrays"); - - private static final SnippetCounter objectCallCounter = new SnippetCounter(counters, "Object[]", "arraycopy call for Object[] arrays"); - - private static final SnippetCounter booleanCallCounter = new SnippetCounter(counters, "boolean[]", "arraycopy call for boolean[] arrays"); - private static final SnippetCounter byteCallCounter = new SnippetCounter(counters, "byte[]", "arraycopy call for byte[] arrays"); - private static final SnippetCounter charCallCounter = new SnippetCounter(counters, "char[]", "arraycopy call for char[] arrays"); - private static final SnippetCounter doubleCallCounter = new SnippetCounter(counters, "double[]", "arraycopy call for double[] arrays"); - private static final SnippetCounter floatCallCounter = new SnippetCounter(counters, "float[]", "arraycopy call for float[] arrays"); - private static final SnippetCounter intCallCounter = new SnippetCounter(counters, "int[]", "arraycopy call for int[] arrays"); - private static final SnippetCounter longCallCounter = new SnippetCounter(counters, "long[]", "arraycopy call for long[] arrays"); - private static final SnippetCounter shortCallCounter = new SnippetCounter(counters, "short[]", "arraycopy call for short[] arrays"); - - private static final SnippetCounter genericPrimitiveCallCounter = new SnippetCounter(counters, "genericPrimitive", "generic arraycopy snippet for primitive arrays"); - private static final SnippetCounter genericObjectExactCallCounter = new SnippetCounter(counters, "genericObjectExact", "generic arraycopy snippet for special object arrays"); - private static final SnippetCounter genericObjectCallCounter = new SnippetCounter(counters, "genericObject", "call to the generic, native arraycopy method"); -} diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ObjectCloneSnippets.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ObjectCloneSnippets.java Thu Dec 04 23:33:24 2014 +0100 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ObjectCloneSnippets.java Thu Dec 04 23:34:27 2014 +0100 @@ -27,6 +27,7 @@ import com.oracle.graal.api.meta.*; import com.oracle.graal.compiler.common.*; +import com.oracle.graal.hotspot.replacements.arraycopy.*; import com.oracle.graal.nodes.*; import com.oracle.graal.nodes.java.*; import com.oracle.graal.replacements.*; diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/SystemSubstitutions.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/SystemSubstitutions.java Thu Dec 04 23:33:24 2014 +0100 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/SystemSubstitutions.java Thu Dec 04 23:34:27 2014 +0100 @@ -29,6 +29,7 @@ import com.oracle.graal.api.replacements.*; import com.oracle.graal.graph.Node.ConstantNodeParameter; import com.oracle.graal.graph.Node.NodeIntrinsic; +import com.oracle.graal.hotspot.replacements.arraycopy.*; import com.oracle.graal.nodes.extended.*; import com.oracle.graal.nodes.spi.*; diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/UnsafeArrayCopyNode.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/UnsafeArrayCopyNode.java Thu Dec 04 23:33:24 2014 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,140 +0,0 @@ -/* - * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package com.oracle.graal.hotspot.replacements; - -import static com.oracle.graal.api.meta.LocationIdentity.*; - -import com.oracle.graal.api.meta.*; -import com.oracle.graal.compiler.common.type.*; -import com.oracle.graal.nodeinfo.*; -import com.oracle.graal.nodes.*; -import com.oracle.graal.nodes.extended.*; -import com.oracle.graal.nodes.spi.*; -import com.oracle.graal.replacements.SnippetTemplate.Arguments; - -@NodeInfo(allowedUsageTypes = {InputType.Memory}) -public class UnsafeArrayCopyNode extends ArrayRangeWriteNode implements Lowerable, MemoryCheckpoint.Single { - - @Input ValueNode src; - @Input ValueNode srcPos; - @Input ValueNode dest; - @Input ValueNode destPos; - @Input ValueNode length; - @OptionalInput ValueNode layoutHelper; - - protected Kind elementKind; - - public static UnsafeArrayCopyNode create(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, ValueNode layoutHelper, Kind elementKind) { - return new UnsafeArrayCopyNode(src, srcPos, dest, destPos, length, layoutHelper, elementKind); - } - - protected UnsafeArrayCopyNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, ValueNode layoutHelper, Kind elementKind) { - super(StampFactory.forVoid()); - assert layoutHelper == null || elementKind == null; - this.src = src; - this.srcPos = srcPos; - this.dest = dest; - this.destPos = destPos; - this.length = length; - this.layoutHelper = layoutHelper; - this.elementKind = elementKind; - } - - public static UnsafeArrayCopyNode create(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, Kind elementKind) { - return new UnsafeArrayCopyNode(src, srcPos, dest, destPos, length, elementKind); - } - - protected UnsafeArrayCopyNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, Kind elementKind) { - this(src, srcPos, dest, destPos, length, null, elementKind); - } - - public static UnsafeArrayCopyNode create(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, ValueNode layoutHelper) { - return new UnsafeArrayCopyNode(src, srcPos, dest, destPos, length, layoutHelper); - } - - protected UnsafeArrayCopyNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, ValueNode layoutHelper) { - this(src, srcPos, dest, destPos, length, layoutHelper, null); - } - - @Override - public ValueNode getArray() { - return dest; - } - - @Override - public ValueNode getIndex() { - return destPos; - } - - @Override - public ValueNode getLength() { - return length; - } - - @Override - public boolean isObjectArray() { - return elementKind == Kind.Object; - } - - @Override - public boolean isInitialization() { - return false; - } - - public Kind getElementKind() { - return elementKind; - } - - @Override - public void lower(LoweringTool tool) { - if (graph().getGuardsStage() == StructuredGraph.GuardsStage.AFTER_FSA) { - UnsafeArrayCopySnippets.Templates templates = tool.getReplacements().getSnippetTemplateCache(UnsafeArrayCopySnippets.Templates.class); - templates.lower(this, tool); - } - } - - public void addSnippetArguments(Arguments args) { - args.add("src", src); - args.add("srcPos", srcPos); - args.add("dest", dest); - args.add("destPos", destPos); - args.add("length", length); - if (layoutHelper != null) { - args.add("layoutHelper", layoutHelper); - } - } - - @Override - public LocationIdentity getLocationIdentity() { - if (elementKind != null) { - return NamedLocationIdentity.getArrayLocation(elementKind); - } - return ANY_LOCATION; - } - - @NodeIntrinsic - public static native void arraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter Kind elementKind); - - @NodeIntrinsic - public static native void arraycopyPrimitive(Object src, int srcPos, Object dest, int destPos, int length, int layoutHelper); -} diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/UnsafeArrayCopySnippets.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/UnsafeArrayCopySnippets.java Thu Dec 04 23:33:24 2014 +0100 +++ /dev/null Thu Jan 01 00:00:00 1970 +0000 @@ -1,310 +0,0 @@ -/* - * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved. - * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. - * - * This code is free software; you can redistribute it and/or modify it - * under the terms of the GNU General Public License version 2 only, as - * published by the Free Software Foundation. - * - * This code is distributed in the hope that it will be useful, but WITHOUT - * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or - * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License - * version 2 for more details (a copy is included in the LICENSE file that - * accompanied this code). - * - * You should have received a copy of the GNU General Public License version - * 2 along with this work; if not, write to the Free Software Foundation, - * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. - * - * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA - * or visit www.oracle.com if you need additional information or have any - * questions. - */ -package com.oracle.graal.hotspot.replacements; - -import static com.oracle.graal.api.meta.LocationIdentity.*; -import static com.oracle.graal.hotspot.HotSpotGraalRuntime.*; -import static com.oracle.graal.hotspot.replacements.HotSpotReplacementsUtil.*; -import static com.oracle.graal.nodes.extended.BranchProbabilityNode.*; -import static com.oracle.graal.replacements.SnippetTemplate.*; - -import com.oracle.graal.api.code.*; -import com.oracle.graal.api.meta.*; -import com.oracle.graal.api.replacements.*; -import com.oracle.graal.asm.*; -import com.oracle.graal.hotspot.meta.*; -import com.oracle.graal.hotspot.phases.*; -import com.oracle.graal.nodes.extended.*; -import com.oracle.graal.nodes.spi.*; -import com.oracle.graal.replacements.*; -import com.oracle.graal.replacements.SnippetTemplate.AbstractTemplates; -import com.oracle.graal.replacements.SnippetTemplate.Arguments; -import com.oracle.graal.replacements.SnippetTemplate.SnippetInfo; -import com.oracle.graal.replacements.nodes.*; -import com.oracle.graal.word.*; - -/** - * As opposed to {@link ArrayCopySnippets}, these Snippets do not perform store checks. - */ -public class UnsafeArrayCopySnippets implements Snippets { - private static final boolean supportsUnalignedMemoryAccess = runtime().getTarget().arch.supportsUnalignedMemoryAccess(); - - private static final Kind VECTOR_KIND = Kind.Long; - private static final long VECTOR_SIZE = arrayIndexScale(VECTOR_KIND); - - private static void vectorizedCopy(Object src, int srcPos, Object dest, int destPos, int length, Kind baseKind, LocationIdentity locationIdentity) { - int arrayBaseOffset = arrayBaseOffset(baseKind); - int elementSize = arrayIndexScale(baseKind); - long byteLength = (long) length * elementSize; - long srcOffset = (long) srcPos * elementSize; - long destOffset = (long) destPos * elementSize; - - long preLoopBytes; - long mainLoopBytes; - long postLoopBytes; - - // We can easily vectorize the loop if both offsets have the same alignment. - if (byteLength >= VECTOR_SIZE && (srcOffset % VECTOR_SIZE) == (destOffset % VECTOR_SIZE)) { - preLoopBytes = NumUtil.roundUp(arrayBaseOffset + srcOffset, VECTOR_SIZE) - (arrayBaseOffset + srcOffset); - postLoopBytes = (byteLength - preLoopBytes) % VECTOR_SIZE; - mainLoopBytes = byteLength - preLoopBytes - postLoopBytes; - } else { - // Does the architecture support unaligned memory accesses? - if (supportsUnalignedMemoryAccess) { - preLoopBytes = byteLength % VECTOR_SIZE; - mainLoopBytes = byteLength - preLoopBytes; - postLoopBytes = 0; - } else { - // No. Let's do element-wise copying. - preLoopBytes = byteLength; - mainLoopBytes = 0; - postLoopBytes = 0; - } - } - - if (probability(NOT_FREQUENT_PROBABILITY, src == dest) && probability(NOT_FREQUENT_PROBABILITY, srcPos < destPos)) { - // bad aliased case - srcOffset += byteLength; - destOffset += byteLength; - - // Post-loop - for (long i = 0; i < postLoopBytes; i += elementSize) { - srcOffset -= elementSize; - destOffset -= elementSize; - Object a = UnsafeLoadNode.load(src, arrayBaseOffset + srcOffset, baseKind, locationIdentity); - UnsafeStoreNode.store(dest, arrayBaseOffset + destOffset, a, baseKind, locationIdentity); - } - // Main-loop - for (long i = 0; i < mainLoopBytes; i += VECTOR_SIZE) { - srcOffset -= VECTOR_SIZE; - destOffset -= VECTOR_SIZE; - Long a = UnsafeLoadNode.load(src, arrayBaseOffset + srcOffset, VECTOR_KIND, locationIdentity); - UnsafeStoreNode.store(dest, arrayBaseOffset + destOffset, a, VECTOR_KIND, locationIdentity); - } - // Pre-loop - for (long i = 0; i < preLoopBytes; i += elementSize) { - srcOffset -= elementSize; - destOffset -= elementSize; - Object a = UnsafeLoadNode.load(src, arrayBaseOffset + srcOffset, baseKind, locationIdentity); - UnsafeStoreNode.store(dest, arrayBaseOffset + destOffset, a, baseKind, locationIdentity); - } - } else { - // Pre-loop - for (long i = 0; i < preLoopBytes; i += elementSize) { - Object a = UnsafeLoadNode.load(src, arrayBaseOffset + srcOffset, baseKind, locationIdentity); - UnsafeStoreNode.store(dest, arrayBaseOffset + destOffset, a, baseKind, locationIdentity); - srcOffset += elementSize; - destOffset += elementSize; - } - // Main-loop - for (long i = 0; i < mainLoopBytes; i += VECTOR_SIZE) { - Long a = UnsafeLoadNode.load(src, arrayBaseOffset + srcOffset, VECTOR_KIND, locationIdentity); - UnsafeStoreNode.store(dest, arrayBaseOffset + destOffset, a, VECTOR_KIND, locationIdentity); - srcOffset += VECTOR_SIZE; - destOffset += VECTOR_SIZE; - } - // Post-loop - for (long i = 0; i < postLoopBytes; i += elementSize) { - Object a = UnsafeLoadNode.load(src, arrayBaseOffset + srcOffset, baseKind, locationIdentity); - UnsafeStoreNode.store(dest, arrayBaseOffset + destOffset, a, baseKind, locationIdentity); - srcOffset += elementSize; - destOffset += elementSize; - } - } - } - - @Fold - private static LocationIdentity getArrayLocation(Kind kind) { - return NamedLocationIdentity.getArrayLocation(kind); - } - - @Snippet - public static void arraycopyByte(byte[] src, int srcPos, byte[] dest, int destPos, int length) { - Kind kind = Kind.Byte; - vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); - } - - @Snippet - public static void arraycopyBoolean(boolean[] src, int srcPos, boolean[] dest, int destPos, int length) { - Kind kind = Kind.Boolean; - vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); - } - - @Snippet - public static void arraycopyChar(char[] src, int srcPos, char[] dest, int destPos, int length) { - Kind kind = Kind.Char; - vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); - } - - @Snippet - public static void arraycopyShort(short[] src, int srcPos, short[] dest, int destPos, int length) { - Kind kind = Kind.Short; - vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); - } - - @Snippet - public static void arraycopyInt(int[] src, int srcPos, int[] dest, int destPos, int length) { - Kind kind = Kind.Int; - vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); - } - - @Snippet - public static void arraycopyFloat(float[] src, int srcPos, float[] dest, int destPos, int length) { - Kind kind = Kind.Float; - vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); - } - - @Snippet - public static void arraycopyLong(long[] src, int srcPos, long[] dest, int destPos, int length) { - Kind kind = Kind.Long; - vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); - } - - @Snippet - public static void arraycopyDouble(double[] src, int srcPos, double[] dest, int destPos, int length) { - Kind kind = Kind.Double; - /* - * TODO atomicity problem on 32-bit architectures: The JVM spec requires double values to be - * copied atomically, but not long values. For example, on Intel 32-bit this code is not - * atomic as long as the vector kind remains Kind.Long. - */ - vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); - } - - /** - * For this kind, Object, we want to avoid write barriers between writes, but instead have them - * at the end of the snippet. This is done by using {@link DirectObjectStoreNode}, and rely on - * {@link WriteBarrierAdditionPhase} to put write barriers after the {@link UnsafeArrayCopyNode} - * with kind Object. - */ - @Snippet - public static void arraycopyObject(Object[] src, int srcPos, Object[] dest, int destPos, int length) { - Kind kind = Kind.Object; - final int scale = arrayIndexScale(kind); - int arrayBaseOffset = arrayBaseOffset(kind); - LocationIdentity arrayLocation = getArrayLocation(kind); - if (src == dest && srcPos < destPos) { // bad aliased case - long start = (long) (length - 1) * scale; - for (long i = start; i >= 0; i -= scale) { - Object a = UnsafeLoadNode.load(src, arrayBaseOffset + i + (long) srcPos * scale, kind, arrayLocation); - DirectObjectStoreNode.storeObject(dest, arrayBaseOffset, i + (long) destPos * scale, a, getArrayLocation(kind)); - } - } else { - long end = (long) length * scale; - for (long i = 0; i < end; i += scale) { - Object a = UnsafeLoadNode.load(src, arrayBaseOffset + i + (long) srcPos * scale, kind, arrayLocation); - DirectObjectStoreNode.storeObject(dest, arrayBaseOffset, i + (long) destPos * scale, a, getArrayLocation(kind)); - } - } - } - - @Snippet - public static void arraycopyPrimitive(Object src, int srcPos, Object dest, int destPos, int length, int layoutHelper) { - int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift()) & layoutHelperLog2ElementSizeMask(); - int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift()) & layoutHelperHeaderSizeMask(); - - Unsigned vectorSize = Word.unsigned(VECTOR_SIZE); - Unsigned srcOffset = Word.unsigned(srcPos).shiftLeft(log2ElementSize).add(headerSize); - Unsigned destOffset = Word.unsigned(destPos).shiftLeft(log2ElementSize).add(headerSize); - Unsigned destStart = destOffset; - Unsigned destEnd = destOffset.add(Word.unsigned(length).shiftLeft(log2ElementSize)); - - Unsigned destVectorEnd = null; - Unsigned nonVectorBytes = null; - Unsigned sizeInBytes = Word.unsigned(length).shiftLeft(log2ElementSize); - if (supportsUnalignedMemoryAccess) { - nonVectorBytes = sizeInBytes.unsignedRemainder(vectorSize); - destVectorEnd = destEnd; - } else { - boolean inPhase = srcOffset.and((int) VECTOR_SIZE - 1).equal(destOffset.and((int) VECTOR_SIZE - 1)); - boolean hasAtLeastOneVector = sizeInBytes.aboveOrEqual(vectorSize); - // We must have at least one full vector, otherwise we must copy each byte separately - if (hasAtLeastOneVector && inPhase) { // If in phase, we can vectorize - nonVectorBytes = vectorSize.subtract(destStart.unsignedRemainder(vectorSize)); - } else { // fallback is byte-wise - nonVectorBytes = sizeInBytes; - } - destVectorEnd = destEnd.subtract(destEnd.unsignedRemainder(vectorSize)); - } - - Unsigned destNonVectorEnd = destStart.add(nonVectorBytes); - while (destOffset.belowThan(destNonVectorEnd)) { - ObjectAccess.writeByte(dest, destOffset, ObjectAccess.readByte(src, srcOffset, ANY_LOCATION), ANY_LOCATION); - destOffset = destOffset.add(1); - srcOffset = srcOffset.add(1); - } - // Unsigned destVectorEnd = destEnd.subtract(destEnd.unsignedRemainder(8)); - while (destOffset.belowThan(destVectorEnd)) { - ObjectAccess.writeWord(dest, destOffset, ObjectAccess.readWord(src, srcOffset, ANY_LOCATION), ANY_LOCATION); - destOffset = destOffset.add(wordSize()); - srcOffset = srcOffset.add(wordSize()); - } - // Do the last bytes each when it is required to have absolute alignment. - while (!supportsUnalignedMemoryAccess && destOffset.belowThan(destEnd)) { - ObjectAccess.writeByte(dest, destOffset, ObjectAccess.readByte(src, srcOffset, ANY_LOCATION), ANY_LOCATION); - destOffset = destOffset.add(1); - srcOffset = srcOffset.add(1); - } - } - - public static class Templates extends AbstractTemplates { - - private final SnippetInfo[] arraycopySnippets; - private final SnippetInfo genericPrimitiveSnippet; - - public Templates(HotSpotProviders providers, TargetDescription target) { - super(providers, providers.getSnippetReflection(), target); - - arraycopySnippets = new SnippetInfo[Kind.values().length]; - arraycopySnippets[Kind.Boolean.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyBoolean"); - arraycopySnippets[Kind.Byte.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyByte"); - arraycopySnippets[Kind.Short.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyShort"); - arraycopySnippets[Kind.Char.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyChar"); - arraycopySnippets[Kind.Int.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyInt"); - arraycopySnippets[Kind.Long.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyLong"); - arraycopySnippets[Kind.Float.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyFloat"); - arraycopySnippets[Kind.Double.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyDouble"); - arraycopySnippets[Kind.Object.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyObject"); - - genericPrimitiveSnippet = snippet(UnsafeArrayCopySnippets.class, "arraycopyPrimitive"); - } - - public void lower(UnsafeArrayCopyNode node, LoweringTool tool) { - Kind elementKind = node.getElementKind(); - SnippetInfo snippet; - if (elementKind == null) { - // primitive array of unknown kind - snippet = genericPrimitiveSnippet; - } else { - snippet = arraycopySnippets[elementKind.ordinal()]; - assert snippet != null : "arraycopy snippet for " + elementKind.name() + " not found"; - } - - Arguments args = new Arguments(snippet, node.graph().getGuardsStage(), tool.getLoweringStage()); - node.addSnippetArguments(args); - - SnippetTemplate template = template(args); - template.instantiate(providers.getMetaAccess(), node, DEFAULT_REPLACER, args); - } - } -} diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/ArrayCopyCallNode.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/ArrayCopyCallNode.java Thu Dec 04 23:34:27 2014 +0100 @@ -0,0 +1,195 @@ +/* + * Copyright (c) 2014, 2014, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +//JaCoCo Exclude +package com.oracle.graal.hotspot.replacements.arraycopy; + +import static com.oracle.graal.api.meta.LocationIdentity.*; + +import com.oracle.graal.api.meta.*; +import com.oracle.graal.api.runtime.*; +import com.oracle.graal.compiler.common.type.*; +import com.oracle.graal.hotspot.*; +import com.oracle.graal.hotspot.meta.*; +import com.oracle.graal.hotspot.nodes.*; +import com.oracle.graal.nodeinfo.*; +import com.oracle.graal.nodes.*; +import com.oracle.graal.nodes.calc.*; +import com.oracle.graal.nodes.extended.*; +import com.oracle.graal.nodes.spi.*; +import com.oracle.graal.runtime.*; + +@NodeInfo(allowedUsageTypes = {InputType.Memory}) +public class ArrayCopyCallNode extends AbstractMemoryCheckpoint implements Lowerable, MemoryCheckpoint.Single { + + @Input ValueNode src; + @Input ValueNode srcPos; + @Input ValueNode dest; + @Input ValueNode destPos; + @Input ValueNode length; + + protected Kind elementKind; + + /** + * Aligned means that the offset of the copy is heap word aligned. + */ + protected boolean aligned; + protected boolean disjoint; + protected boolean uninitialized; + + protected final HotSpotGraalRuntimeProvider runtime; + + public static ArrayCopyCallNode create(@InjectedNodeParameter HotSpotGraalRuntimeProvider runtime, ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, + Kind elementKind, boolean aligned, boolean disjoint, boolean uninitialized) { + return new ArrayCopyCallNode(src, srcPos, dest, destPos, length, elementKind, aligned, disjoint, uninitialized, runtime); + } + + protected ArrayCopyCallNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, Kind elementKind, boolean aligned, boolean disjoint, boolean uninitialized, + HotSpotGraalRuntimeProvider runtime) { + super(StampFactory.forVoid()); + assert elementKind != null; + this.src = src; + this.srcPos = srcPos; + this.dest = dest; + this.destPos = destPos; + this.length = length; + this.elementKind = elementKind; + this.aligned = aligned; + this.disjoint = disjoint; + this.uninitialized = uninitialized; + this.runtime = runtime; + } + + public static ArrayCopyCallNode create(@InjectedNodeParameter HotSpotGraalRuntimeProvider runtime, ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, + Kind elementKind, boolean disjoint) { + return new ArrayCopyCallNode(src, srcPos, dest, destPos, length, elementKind, false, disjoint, false, runtime); + } + + public ValueNode getSource() { + return src; + } + + public ValueNode getSourcePosition() { + return srcPos; + } + + public ValueNode getDestination() { + return dest; + } + + public ValueNode getDestinationPosition() { + return destPos; + } + + public ValueNode getLength() { + return length; + } + + public Kind getElementKind() { + return elementKind; + } + + private ValueNode computeBase(ValueNode base, ValueNode pos) { + FixedWithNextNode basePtr = graph().add(GetObjectAddressNode.create(base)); + graph().addBeforeFixed(this, basePtr); + ValueNode loc = IndexedLocationNode.create(getLocationIdentity(), elementKind, runtime.getArrayBaseOffset(elementKind), pos, graph(), runtime.getArrayIndexScale(elementKind)); + return graph().unique(ComputeAddressNode.create(basePtr, loc, StampFactory.forKind(Kind.Long))); + } + + @Override + public void lower(LoweringTool tool) { + if (graph().getGuardsStage() == StructuredGraph.GuardsStage.AFTER_FSA) { + updateAlignedDisjoint(); + ForeignCallDescriptor desc = HotSpotHostForeignCallsProvider.lookupArraycopyDescriptor(elementKind, isAligned(), isDisjoint(), isUninitialized()); + StructuredGraph graph = graph(); + ValueNode srcAddr = computeBase(getSource(), getSourcePosition()); + ValueNode destAddr = computeBase(getDestination(), getDestinationPosition()); + ValueNode len = getLength(); + if (len.stamp().getStackKind() != Kind.Long) { + len = IntegerConvertNode.convert(len, StampFactory.forKind(Kind.Long), graph()); + } + ForeignCallNode call = graph.add(ForeignCallNode.create(Graal.getRequiredCapability(RuntimeProvider.class).getHostBackend().getForeignCalls(), desc, srcAddr, destAddr, len)); + call.setStateAfter(stateAfter()); + graph.replaceFixedWithFixed(this, call); + + } + } + + @Override + public LocationIdentity getLocationIdentity() { + if (elementKind != null) { + return NamedLocationIdentity.getArrayLocation(elementKind); + } + return ANY_LOCATION; + } + + @NodeIntrinsic + private static native void arraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter Kind elementKind, @ConstantNodeParameter boolean aligned, + @ConstantNodeParameter boolean disjoint, @ConstantNodeParameter boolean uninitialized); + + public static void arraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter Kind elementKind, boolean aligned, boolean disjoint) { + arraycopy(src, srcPos, dest, destPos, length, elementKind, aligned, disjoint, false); + } + + public static void disjointArraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter Kind elementKind) { + arraycopy(src, srcPos, dest, destPos, length, elementKind, false, true, false); + } + + public static void disjointUninitializedArraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter Kind elementKind) { + arraycopy(src, srcPos, dest, destPos, length, elementKind, false, true, true); + } + + public boolean isAligned() { + return aligned; + } + + public boolean isDisjoint() { + return disjoint; + } + + public boolean isUninitialized() { + return uninitialized; + } + + boolean isHeapWordAligned(JavaConstant value, Kind kind) { + return (runtime.getArrayBaseOffset(kind) + (long) value.asInt() * runtime.getArrayIndexScale(kind)) % runtime.getConfig().heapWordSize == 0; + } + + public void updateAlignedDisjoint() { + Kind componentKind = elementKind; + if (srcPos == destPos) { + // Can treat as disjoint + disjoint = true; + } + PrimitiveConstant constantSrc = (PrimitiveConstant) srcPos.stamp().asConstant(); + PrimitiveConstant constantDst = (PrimitiveConstant) destPos.stamp().asConstant(); + if (constantSrc != null && constantDst != null) { + if (!aligned) { + aligned = isHeapWordAligned(constantSrc, componentKind) && isHeapWordAligned(constantDst, componentKind); + } + if (constantSrc.asInt() >= constantDst.asInt()) { + // low to high copy so treat as disjoint + disjoint = true; + } + } + } +} diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/ArrayCopyNode.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/ArrayCopyNode.java Thu Dec 04 23:34:27 2014 +0100 @@ -0,0 +1,148 @@ +/* + * Copyright (c) 2013, 2014, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.graal.hotspot.replacements.arraycopy; + +import static com.oracle.graal.compiler.GraalCompiler.*; + +import com.oracle.graal.api.meta.*; +import com.oracle.graal.compiler.common.*; +import com.oracle.graal.debug.*; +import com.oracle.graal.debug.Debug.Scope; +import com.oracle.graal.loop.phases.*; +import com.oracle.graal.nodeinfo.*; +import com.oracle.graal.nodes.*; +import com.oracle.graal.nodes.spi.*; +import com.oracle.graal.nodes.type.*; +import com.oracle.graal.phases.common.*; +import com.oracle.graal.phases.tiers.*; +import com.oracle.graal.replacements.nodes.*; + +@NodeInfo +public class ArrayCopyNode extends BasicArrayCopyNode implements Virtualizable, Lowerable { + + public static ArrayCopyNode create(Invoke invoke) { + return new ArrayCopyNode(invoke); + } + + protected ArrayCopyNode(Invoke invoke) { + super(invoke); + } + + private StructuredGraph selectSnippet(LoweringTool tool, final Replacements replacements) { + ResolvedJavaType srcType = StampTool.typeOrNull(getSource().stamp()); + ResolvedJavaType destType = StampTool.typeOrNull(getDestination().stamp()); + + if (srcType == null || !srcType.isArray() || destType == null || !destType.isArray()) { + return null; + } + if (!destType.getComponentType().isAssignableFrom(srcType.getComponentType())) { + return null; + } + if (!isExact()) { + return null; + } + Kind componentKind = srcType.getComponentType().getKind(); + final ResolvedJavaMethod snippetMethod = tool.getMetaAccess().lookupJavaMethod(ArrayCopySnippets.getSnippetForKind(componentKind, shouldUnroll(), isExact())); + try (Scope s = Debug.scope("ArrayCopySnippet", snippetMethod)) { + return replacements.getSnippet(snippetMethod); + } catch (Throwable e) { + throw Debug.handle(e); + } + } + + private static void unrollFixedLengthLoop(StructuredGraph snippetGraph, int length, LoweringTool tool) { + ParameterNode lengthParam = snippetGraph.getParameter(4); + if (lengthParam != null) { + snippetGraph.replaceFloating(lengthParam, ConstantNode.forInt(length, snippetGraph)); + } + // the canonicalization before loop unrolling is needed to propagate the length into + // additions, etc. + PhaseContext context = new PhaseContext(tool.getMetaAccess(), tool.getConstantReflection(), tool.getLowerer(), tool.getReplacements(), tool.assumptions(), tool.getStampProvider()); + new CanonicalizerPhase(true).apply(snippetGraph, context); + new LoopFullUnrollPhase(new CanonicalizerPhase(true)).apply(snippetGraph, context); + new CanonicalizerPhase(true).apply(snippetGraph, context); + } + + @Override + protected StructuredGraph getLoweredSnippetGraph(final LoweringTool tool) { + if (!shouldIntrinsify(getTargetMethod())) { + return null; + } + + final Replacements replacements = tool.getReplacements(); + StructuredGraph snippetGraph = selectSnippet(tool, replacements); + if (snippetGraph == null) { + ResolvedJavaType srcType = StampTool.typeOrNull(getSource().stamp()); + ResolvedJavaType destType = StampTool.typeOrNull(getDestination().stamp()); + ResolvedJavaType srcComponentType = srcType == null ? null : srcType.getComponentType(); + ResolvedJavaType destComponentType = destType == null ? null : destType.getComponentType(); + ResolvedJavaMethod snippetMethod = null; + if (srcComponentType != null && destComponentType != null && !srcComponentType.isPrimitive() && !destComponentType.isPrimitive()) { + snippetMethod = tool.getMetaAccess().lookupJavaMethod(ArrayCopySnippets.checkcastArraycopySnippet); + } else { + snippetMethod = tool.getMetaAccess().lookupJavaMethod(ArrayCopySnippets.genericArraycopySnippet); + } + snippetGraph = null; + try (Scope s = Debug.scope("ArrayCopySnippet", snippetMethod)) { + snippetGraph = replacements.getSnippet(snippetMethod, getTargetMethod()).copy(); + } catch (Throwable e) { + throw Debug.handle(e); + } + replaceSnippetInvokes(snippetGraph); + } else { + assert snippetGraph != null : "ArrayCopySnippets should be installed"; + snippetGraph = snippetGraph.copy(); + if (shouldUnroll()) { + final StructuredGraph copy = snippetGraph; + try (Scope s = Debug.scope("ArrayCopySnippetSpecialization", snippetGraph.method())) { + unrollFixedLengthLoop(copy, getLength().asJavaConstant().asInt(), tool); + } catch (Throwable e) { + throw Debug.handle(e); + } + } + } + return lowerReplacement(snippetGraph, tool); + } + + private boolean shouldUnroll() { + return getLength().isConstant() && getLength().asJavaConstant().asInt() <= GraalOptions.MaximumEscapeAnalysisArrayLength.getValue(); + } + + /* + * Returns true if this copy doesn't require store checks. Trivially true for primitive arrays. + */ + private boolean isExact() { + ResolvedJavaType srcType = StampTool.typeOrNull(getSource().stamp()); + if (srcType.getComponentType().getKind().isPrimitive() || getSource() == getDestination()) { + return true; + } + + ResolvedJavaType destType = StampTool.typeOrNull(getDestination().stamp()); + if (StampTool.isExactType(getDestination().stamp())) { + if (destType != null && destType.isAssignableFrom(srcType)) { + return true; + } + } + return false; + } +} diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/ArrayCopySnippets.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/ArrayCopySnippets.java Thu Dec 04 23:34:27 2014 +0100 @@ -0,0 +1,323 @@ +/* + * Copyright (c) 2011, 2014, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.graal.hotspot.replacements.arraycopy; + +import static com.oracle.graal.compiler.common.GraalOptions.*; +import static com.oracle.graal.hotspot.replacements.HotSpotReplacementsUtil.*; +import static com.oracle.graal.nodes.GuardingPiNode.*; +import static com.oracle.graal.nodes.extended.BranchProbabilityNode.*; + +import java.lang.reflect.*; +import java.util.*; + +import com.oracle.graal.api.meta.*; +import com.oracle.graal.compiler.common.*; +import com.oracle.graal.graph.Node.ConstantNodeParameter; +import com.oracle.graal.graph.Node.NodeIntrinsic; +import com.oracle.graal.hotspot.word.*; +import com.oracle.graal.nodes.*; +import com.oracle.graal.nodes.extended.*; +import com.oracle.graal.nodes.java.*; +import com.oracle.graal.replacements.*; +import com.oracle.graal.word.*; + +public class ArrayCopySnippets implements Snippets { + + private static final EnumMap arraycopyMethods = new EnumMap<>(Kind.class); + private static final EnumMap arraycopyCalls = new EnumMap<>(Kind.class); + + public static final Method checkcastArraycopySnippet; + public static final Method genericArraycopySnippet; + + private static void addArraycopySnippetMethod(Kind kind, Class arrayClass) throws NoSuchMethodException { + arraycopyMethods.put(kind, ArrayCopySnippets.class.getDeclaredMethod("arraycopy", arrayClass, int.class, arrayClass, int.class, int.class)); + if (CallArrayCopy.getValue()) { + if (kind == Kind.Object) { + arraycopyCalls.put(kind, ArrayCopySnippets.class.getDeclaredMethod("objectArraycopyUnchecked", arrayClass, int.class, arrayClass, int.class, int.class)); + } else { + arraycopyCalls.put(kind, ArrayCopySnippets.class.getDeclaredMethod(kind + "Arraycopy", arrayClass, int.class, arrayClass, int.class, int.class)); + } + } + } + + static { + try { + addArraycopySnippetMethod(Kind.Byte, byte[].class); + addArraycopySnippetMethod(Kind.Boolean, boolean[].class); + addArraycopySnippetMethod(Kind.Char, char[].class); + addArraycopySnippetMethod(Kind.Short, short[].class); + addArraycopySnippetMethod(Kind.Int, int[].class); + addArraycopySnippetMethod(Kind.Long, long[].class); + addArraycopySnippetMethod(Kind.Float, float[].class); + addArraycopySnippetMethod(Kind.Double, double[].class); + addArraycopySnippetMethod(Kind.Object, Object[].class); + checkcastArraycopySnippet = ArrayCopySnippets.class.getDeclaredMethod("checkcastArraycopy", Object[].class, int.class, Object[].class, int.class, int.class); + genericArraycopySnippet = ArrayCopySnippets.class.getDeclaredMethod("arraycopy", Object.class, int.class, Object.class, int.class, int.class); + } catch (SecurityException | NoSuchMethodException e) { + throw new GraalInternalError(e); + } + } + + public static Method getSnippetForKind(Kind kind, boolean shouldUnroll, boolean exact) { + Method m = null; + if (!shouldUnroll && exact) { + // use hotspot stubs + m = arraycopyCalls.get(kind); + if (m != null) { + return m; + } + } + // use snippets + return arraycopyMethods.get(kind); + } + + private static void checkedCopy(Object src, int srcPos, Object dest, int destPos, int length, Kind baseKind) { + Object nonNullSrc = guardingNonNull(src); + Object nonNullDest = guardingNonNull(dest); + checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); + UnsafeArrayCopyNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, baseKind); + } + + private static int checkArrayType(KlassPointer hub) { + int layoutHelper = readLayoutHelper(hub); + if (probability(SLOW_PATH_PROBABILITY, layoutHelper >= 0)) { + DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); + } + return layoutHelper; + } + + private static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length) { + if (probability(SLOW_PATH_PROBABILITY, srcPos < 0)) { + checkAIOOBECounter.inc(); + DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); + } + if (probability(SLOW_PATH_PROBABILITY, destPos < 0)) { + checkAIOOBECounter.inc(); + DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); + } + if (probability(SLOW_PATH_PROBABILITY, length < 0)) { + checkAIOOBECounter.inc(); + DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); + } + if (probability(SLOW_PATH_PROBABILITY, srcPos + length > ArrayLengthNode.arrayLength(src))) { + checkAIOOBECounter.inc(); + DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); + } + if (probability(SLOW_PATH_PROBABILITY, destPos + length > ArrayLengthNode.arrayLength(dest))) { + checkAIOOBECounter.inc(); + DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); + } + checkSuccessCounter.inc(); + } + + @Snippet + public static void arraycopy(byte[] src, int srcPos, byte[] dest, int destPos, int length) { + byteCounter.inc(); + checkedCopy(src, srcPos, dest, destPos, length, Kind.Byte); + } + + @Snippet + public static void arraycopy(boolean[] src, int srcPos, boolean[] dest, int destPos, int length) { + booleanCounter.inc(); + checkedCopy(src, srcPos, dest, destPos, length, Kind.Boolean); + } + + @Snippet + public static void arraycopy(char[] src, int srcPos, char[] dest, int destPos, int length) { + charCounter.inc(); + checkedCopy(src, srcPos, dest, destPos, length, Kind.Char); + } + + @Snippet + public static void arraycopy(short[] src, int srcPos, short[] dest, int destPos, int length) { + shortCounter.inc(); + checkedCopy(src, srcPos, dest, destPos, length, Kind.Short); + } + + @Snippet + public static void arraycopy(int[] src, int srcPos, int[] dest, int destPos, int length) { + intCounter.inc(); + checkedCopy(src, srcPos, dest, destPos, length, Kind.Int); + } + + @Snippet + public static void arraycopy(float[] src, int srcPos, float[] dest, int destPos, int length) { + floatCounter.inc(); + checkedCopy(src, srcPos, dest, destPos, length, Kind.Float); + } + + @Snippet + public static void arraycopy(long[] src, int srcPos, long[] dest, int destPos, int length) { + longCounter.inc(); + checkedCopy(src, srcPos, dest, destPos, length, Kind.Long); + } + + @Snippet + public static void arraycopy(double[] src, int srcPos, double[] dest, int destPos, int length) { + doubleCounter.inc(); + checkedCopy(src, srcPos, dest, destPos, length, Kind.Double); + } + + @Snippet + public static void arraycopy(Object[] src, int srcPos, Object[] dest, int destPos, int length) { + objectCounter.inc(); + checkedCopy(src, srcPos, dest, destPos, length, Kind.Object); + } + + @Snippet + public static void checkcastArraycopy(Object[] src, int srcPos, Object[] dest, int destPos, int length) { + objectCheckcastCounter.inc(); + Object nonNullSrc = guardingNonNull(src); + Object nonNullDest = guardingNonNull(dest); + checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); + if (probability(SLOW_PATH_PROBABILITY, nonNullSrc == nonNullDest)) { + // no storecheck required. + ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, Kind.Object, false, false); + } else { + KlassPointer destElemKlass = loadHub(nonNullDest); + checkcastArraycopyHelper(srcPos, destPos, length, nonNullSrc, nonNullDest, destElemKlass); + } + } + + private static void checkcastArraycopyHelper(int srcPos, int destPos, int length, Object nonNullSrc, Object nonNullDest, KlassPointer destElemKlass) { + Word superCheckOffset = Word.signed(destElemKlass.readInt(superCheckOffsetOffset(), KLASS_SUPER_CHECK_OFFSET_LOCATION)); + int copiedElements = CheckcastArrayCopyCallNode.checkcastArraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, superCheckOffset, destElemKlass, false); + if (copiedElements != 0) { + // the checkcast stub doesn't throw the ArrayStoreException, but returns the number of + // copied elements (xor'd with -1). + copiedElements ^= -1; + System.arraycopy(nonNullSrc, srcPos + copiedElements, nonNullDest, destPos + copiedElements, length - copiedElements); + } + } + + @Snippet + public static void arraycopy(Object src, int srcPos, Object dest, int destPos, int length) { + Object nonNullSrc = guardingNonNull(src); + Object nonNullDest = guardingNonNull(dest); + KlassPointer srcHub = loadHub(nonNullSrc); + KlassPointer destHub = loadHub(nonNullDest); + if (probability(FAST_PATH_PROBABILITY, srcHub.equal(destHub)) && probability(FAST_PATH_PROBABILITY, nonNullSrc != nonNullDest)) { + int layoutHelper = checkArrayType(srcHub); + final boolean isObjectArray = ((layoutHelper & layoutHelperElementTypePrimitiveInPlace()) == 0); + checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); + if (probability(FAST_PATH_PROBABILITY, isObjectArray)) { + genericObjectExactCallCounter.inc(); + UnsafeArrayCopyNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, Kind.Object); + } else { + genericPrimitiveCallCounter.inc(); + UnsafeArrayCopyNode.arraycopyPrimitive(nonNullSrc, srcPos, nonNullDest, destPos, length, layoutHelper); + } + } else { + genericObjectCallCounter.inc(); + System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length); + } + } + + @NodeIntrinsic(ForeignCallNode.class) + public static native void callArraycopy(@ConstantNodeParameter ForeignCallDescriptor descriptor, Word src, Word dest, Word len); + + private static void callArraycopyTemplate(SnippetCounter counter, Kind kind, boolean aligned, boolean disjoint, Object src, int srcPos, Object dest, int destPos, int length) { + counter.inc(); + Object nonNullSrc = guardingNonNull(src); + Object nonNullDest = guardingNonNull(dest); + checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length); + ArrayCopyCallNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, kind, aligned, disjoint); + } + + @Snippet + public static void objectArraycopyUnchecked(Object[] src, int srcPos, Object[] dest, int destPos, int length) { + callArraycopyTemplate(objectCallCounter, Kind.Object, false, false, src, srcPos, dest, destPos, length); + } + + @Snippet + public static void byteArraycopy(byte[] src, int srcPos, byte[] dest, int destPos, int length) { + callArraycopyTemplate(byteCallCounter, Kind.Byte, false, false, src, srcPos, dest, destPos, length); + } + + @Snippet + public static void booleanArraycopy(boolean[] src, int srcPos, boolean[] dest, int destPos, int length) { + callArraycopyTemplate(booleanCallCounter, Kind.Boolean, false, false, src, srcPos, dest, destPos, length); + } + + @Snippet + public static void charArraycopy(char[] src, int srcPos, char[] dest, int destPos, int length) { + callArraycopyTemplate(charCallCounter, Kind.Char, false, false, src, srcPos, dest, destPos, length); + } + + @Snippet + public static void shortArraycopy(short[] src, int srcPos, short[] dest, int destPos, int length) { + callArraycopyTemplate(shortCallCounter, Kind.Short, false, false, src, srcPos, dest, destPos, length); + } + + @Snippet + public static void intArraycopy(int[] src, int srcPos, int[] dest, int destPos, int length) { + callArraycopyTemplate(intCallCounter, Kind.Int, false, false, src, srcPos, dest, destPos, length); + } + + @Snippet + public static void floatArraycopy(float[] src, int srcPos, float[] dest, int destPos, int length) { + callArraycopyTemplate(floatCallCounter, Kind.Float, false, false, src, srcPos, dest, destPos, length); + } + + @Snippet + public static void longArraycopy(long[] src, int srcPos, long[] dest, int destPos, int length) { + callArraycopyTemplate(longCallCounter, Kind.Long, false, false, src, srcPos, dest, destPos, length); + } + + @Snippet + public static void doubleArraycopy(double[] src, int srcPos, double[] dest, int destPos, int length) { + callArraycopyTemplate(doubleCallCounter, Kind.Double, false, false, src, srcPos, dest, destPos, length); + } + + private static final SnippetCounter.Group checkCounters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy checkInputs") : null; + private static final SnippetCounter checkSuccessCounter = new SnippetCounter(checkCounters, "checkSuccess", "checkSuccess"); + private static final SnippetCounter checkNPECounter = new SnippetCounter(checkCounters, "checkNPE", "checkNPE"); + private static final SnippetCounter checkAIOOBECounter = new SnippetCounter(checkCounters, "checkAIOOBE", "checkAIOOBE"); + + private static final SnippetCounter.Group counters = SnippetCounters.getValue() ? new SnippetCounter.Group("System.arraycopy") : null; + private static final SnippetCounter byteCounter = new SnippetCounter(counters, "byte[]", "arraycopy for byte[] arrays"); + private static final SnippetCounter charCounter = new SnippetCounter(counters, "char[]", "arraycopy for char[] arrays"); + private static final SnippetCounter shortCounter = new SnippetCounter(counters, "short[]", "arraycopy for short[] arrays"); + private static final SnippetCounter intCounter = new SnippetCounter(counters, "int[]", "arraycopy for int[] arrays"); + private static final SnippetCounter booleanCounter = new SnippetCounter(counters, "boolean[]", "arraycopy for boolean[] arrays"); + private static final SnippetCounter longCounter = new SnippetCounter(counters, "long[]", "arraycopy for long[] arrays"); + private static final SnippetCounter objectCounter = new SnippetCounter(counters, "Object[]", "arraycopy for Object[] arrays"); + private static final SnippetCounter objectCheckcastCounter = new SnippetCounter(counters, "Object[]", "arraycopy for non-exact Object[] arrays"); + private static final SnippetCounter floatCounter = new SnippetCounter(counters, "float[]", "arraycopy for float[] arrays"); + private static final SnippetCounter doubleCounter = new SnippetCounter(counters, "double[]", "arraycopy for double[] arrays"); + + private static final SnippetCounter objectCallCounter = new SnippetCounter(counters, "Object[]", "arraycopy call for Object[] arrays"); + + private static final SnippetCounter booleanCallCounter = new SnippetCounter(counters, "boolean[]", "arraycopy call for boolean[] arrays"); + private static final SnippetCounter byteCallCounter = new SnippetCounter(counters, "byte[]", "arraycopy call for byte[] arrays"); + private static final SnippetCounter charCallCounter = new SnippetCounter(counters, "char[]", "arraycopy call for char[] arrays"); + private static final SnippetCounter doubleCallCounter = new SnippetCounter(counters, "double[]", "arraycopy call for double[] arrays"); + private static final SnippetCounter floatCallCounter = new SnippetCounter(counters, "float[]", "arraycopy call for float[] arrays"); + private static final SnippetCounter intCallCounter = new SnippetCounter(counters, "int[]", "arraycopy call for int[] arrays"); + private static final SnippetCounter longCallCounter = new SnippetCounter(counters, "long[]", "arraycopy call for long[] arrays"); + private static final SnippetCounter shortCallCounter = new SnippetCounter(counters, "short[]", "arraycopy call for short[] arrays"); + + private static final SnippetCounter genericPrimitiveCallCounter = new SnippetCounter(counters, "genericPrimitive", "generic arraycopy snippet for primitive arrays"); + private static final SnippetCounter genericObjectExactCallCounter = new SnippetCounter(counters, "genericObjectExact", "generic arraycopy snippet for special object arrays"); + private static final SnippetCounter genericObjectCallCounter = new SnippetCounter(counters, "genericObject", "call to the generic, native arraycopy method"); +} diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/CheckcastArrayCopyCallNode.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/CheckcastArrayCopyCallNode.java Thu Dec 04 23:34:27 2014 +0100 @@ -0,0 +1,130 @@ +/* + * Copyright (c) 2014, 2014, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +//JaCoCo Exclude +package com.oracle.graal.hotspot.replacements.arraycopy; + +import com.oracle.graal.api.meta.*; +import com.oracle.graal.api.runtime.*; +import com.oracle.graal.compiler.common.type.*; +import com.oracle.graal.hotspot.*; +import com.oracle.graal.hotspot.meta.*; +import com.oracle.graal.hotspot.nodes.*; +import com.oracle.graal.nodeinfo.*; +import com.oracle.graal.nodes.*; +import com.oracle.graal.nodes.calc.*; +import com.oracle.graal.nodes.extended.*; +import com.oracle.graal.nodes.spi.*; +import com.oracle.graal.runtime.*; +import com.oracle.graal.word.*; + +@NodeInfo(allowedUsageTypes = {InputType.Memory, InputType.Value}) +public class CheckcastArrayCopyCallNode extends AbstractMemoryCheckpoint implements Lowerable, MemoryCheckpoint.Single { + + @Input ValueNode src; + @Input ValueNode srcPos; + @Input ValueNode dest; + @Input ValueNode destPos; + @Input ValueNode length; + @Input ValueNode destElemKlass; + @Input ValueNode superCheckOffset; + + protected final boolean uninit; + + protected final HotSpotGraalRuntimeProvider runtime; + + public static CheckcastArrayCopyCallNode create(@InjectedNodeParameter HotSpotGraalRuntimeProvider runtime, ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, + ValueNode superCheckOffset, ValueNode destElemKlass, boolean uninit) { + return new CheckcastArrayCopyCallNode(src, srcPos, dest, destPos, length, superCheckOffset, destElemKlass, uninit, runtime); + } + + protected CheckcastArrayCopyCallNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, ValueNode superCheckOffset, ValueNode destElemKlass, boolean uninit, + HotSpotGraalRuntimeProvider runtime) { + super(StampFactory.forKind(Kind.Int)); + this.src = src; + this.srcPos = srcPos; + this.dest = dest; + this.destPos = destPos; + this.length = length; + this.superCheckOffset = superCheckOffset; + this.destElemKlass = destElemKlass; + this.uninit = uninit; + this.runtime = runtime; + } + + public ValueNode getSource() { + return src; + } + + public ValueNode getSourcePosition() { + return srcPos; + } + + public ValueNode getDestination() { + return dest; + } + + public ValueNode getDestinationPosition() { + return destPos; + } + + public ValueNode getLength() { + return length; + } + + public boolean isUninit() { + return uninit; + } + + private ValueNode computeBase(ValueNode base, ValueNode pos) { + FixedWithNextNode basePtr = graph().add(GetObjectAddressNode.create(base)); + graph().addBeforeFixed(this, basePtr); + ValueNode loc = IndexedLocationNode.create(getLocationIdentity(), Kind.Object, runtime.getArrayBaseOffset(Kind.Object), pos, graph(), runtime.getArrayIndexScale(Kind.Object)); + return graph().unique(ComputeAddressNode.create(basePtr, loc, StampFactory.forKind(Kind.Long))); + } + + @Override + public void lower(LoweringTool tool) { + if (graph().getGuardsStage() == StructuredGraph.GuardsStage.AFTER_FSA) { + ForeignCallDescriptor desc = HotSpotHostForeignCallsProvider.lookupCheckcastArraycopyDescriptor(isUninit()); + StructuredGraph graph = graph(); + ValueNode srcAddr = computeBase(getSource(), getSourcePosition()); + ValueNode destAddr = computeBase(getDestination(), getDestinationPosition()); + ValueNode len = getLength(); + if (len.stamp().getStackKind() != Kind.Long) { + len = IntegerConvertNode.convert(len, StampFactory.forKind(Kind.Long), graph()); + } + ForeignCallNode call = graph.add(ForeignCallNode.create(Graal.getRequiredCapability(RuntimeProvider.class).getHostBackend().getForeignCalls(), desc, srcAddr, destAddr, len, + superCheckOffset, destElemKlass)); + call.setStateAfter(stateAfter()); + graph.replaceFixedWithFixed(this, call); + } + } + + @Override + public LocationIdentity getLocationIdentity() { + return NamedLocationIdentity.getArrayLocation(Kind.Object); + } + + @NodeIntrinsic + public static native int checkcastArraycopy(Object src, int srcPos, Object dest, int destPos, int length, Word superCheckOffset, Object destElemKlass, @ConstantNodeParameter boolean uninit); +} diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/UnsafeArrayCopyNode.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/UnsafeArrayCopyNode.java Thu Dec 04 23:34:27 2014 +0100 @@ -0,0 +1,140 @@ +/* + * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.graal.hotspot.replacements.arraycopy; + +import static com.oracle.graal.api.meta.LocationIdentity.*; + +import com.oracle.graal.api.meta.*; +import com.oracle.graal.compiler.common.type.*; +import com.oracle.graal.nodeinfo.*; +import com.oracle.graal.nodes.*; +import com.oracle.graal.nodes.extended.*; +import com.oracle.graal.nodes.spi.*; +import com.oracle.graal.replacements.SnippetTemplate.Arguments; + +@NodeInfo(allowedUsageTypes = {InputType.Memory}) +public class UnsafeArrayCopyNode extends ArrayRangeWriteNode implements Lowerable, MemoryCheckpoint.Single { + + @Input ValueNode src; + @Input ValueNode srcPos; + @Input ValueNode dest; + @Input ValueNode destPos; + @Input ValueNode length; + @OptionalInput ValueNode layoutHelper; + + protected Kind elementKind; + + public static UnsafeArrayCopyNode create(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, ValueNode layoutHelper, Kind elementKind) { + return new UnsafeArrayCopyNode(src, srcPos, dest, destPos, length, layoutHelper, elementKind); + } + + protected UnsafeArrayCopyNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, ValueNode layoutHelper, Kind elementKind) { + super(StampFactory.forVoid()); + assert layoutHelper == null || elementKind == null; + this.src = src; + this.srcPos = srcPos; + this.dest = dest; + this.destPos = destPos; + this.length = length; + this.layoutHelper = layoutHelper; + this.elementKind = elementKind; + } + + public static UnsafeArrayCopyNode create(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, Kind elementKind) { + return new UnsafeArrayCopyNode(src, srcPos, dest, destPos, length, elementKind); + } + + protected UnsafeArrayCopyNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, Kind elementKind) { + this(src, srcPos, dest, destPos, length, null, elementKind); + } + + public static UnsafeArrayCopyNode create(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, ValueNode layoutHelper) { + return new UnsafeArrayCopyNode(src, srcPos, dest, destPos, length, layoutHelper); + } + + protected UnsafeArrayCopyNode(ValueNode src, ValueNode srcPos, ValueNode dest, ValueNode destPos, ValueNode length, ValueNode layoutHelper) { + this(src, srcPos, dest, destPos, length, layoutHelper, null); + } + + @Override + public ValueNode getArray() { + return dest; + } + + @Override + public ValueNode getIndex() { + return destPos; + } + + @Override + public ValueNode getLength() { + return length; + } + + @Override + public boolean isObjectArray() { + return elementKind == Kind.Object; + } + + @Override + public boolean isInitialization() { + return false; + } + + public Kind getElementKind() { + return elementKind; + } + + @Override + public void lower(LoweringTool tool) { + if (graph().getGuardsStage() == StructuredGraph.GuardsStage.AFTER_FSA) { + UnsafeArrayCopySnippets.Templates templates = tool.getReplacements().getSnippetTemplateCache(UnsafeArrayCopySnippets.Templates.class); + templates.lower(this, tool); + } + } + + public void addSnippetArguments(Arguments args) { + args.add("src", src); + args.add("srcPos", srcPos); + args.add("dest", dest); + args.add("destPos", destPos); + args.add("length", length); + if (layoutHelper != null) { + args.add("layoutHelper", layoutHelper); + } + } + + @Override + public LocationIdentity getLocationIdentity() { + if (elementKind != null) { + return NamedLocationIdentity.getArrayLocation(elementKind); + } + return ANY_LOCATION; + } + + @NodeIntrinsic + public static native void arraycopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantNodeParameter Kind elementKind); + + @NodeIntrinsic + public static native void arraycopyPrimitive(Object src, int srcPos, Object dest, int destPos, int length, int layoutHelper); +} diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/UnsafeArrayCopySnippets.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/arraycopy/UnsafeArrayCopySnippets.java Thu Dec 04 23:34:27 2014 +0100 @@ -0,0 +1,310 @@ +/* + * Copyright (c) 2013, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.graal.hotspot.replacements.arraycopy; + +import static com.oracle.graal.api.meta.LocationIdentity.*; +import static com.oracle.graal.hotspot.HotSpotGraalRuntime.*; +import static com.oracle.graal.hotspot.replacements.HotSpotReplacementsUtil.*; +import static com.oracle.graal.nodes.extended.BranchProbabilityNode.*; +import static com.oracle.graal.replacements.SnippetTemplate.*; + +import com.oracle.graal.api.code.*; +import com.oracle.graal.api.meta.*; +import com.oracle.graal.api.replacements.*; +import com.oracle.graal.asm.*; +import com.oracle.graal.hotspot.meta.*; +import com.oracle.graal.hotspot.phases.*; +import com.oracle.graal.nodes.extended.*; +import com.oracle.graal.nodes.spi.*; +import com.oracle.graal.replacements.*; +import com.oracle.graal.replacements.SnippetTemplate.AbstractTemplates; +import com.oracle.graal.replacements.SnippetTemplate.Arguments; +import com.oracle.graal.replacements.SnippetTemplate.SnippetInfo; +import com.oracle.graal.replacements.nodes.*; +import com.oracle.graal.word.*; + +/** + * As opposed to {@link ArrayCopySnippets}, these Snippets do not perform store checks. + */ +public class UnsafeArrayCopySnippets implements Snippets { + private static final boolean supportsUnalignedMemoryAccess = runtime().getTarget().arch.supportsUnalignedMemoryAccess(); + + private static final Kind VECTOR_KIND = Kind.Long; + private static final long VECTOR_SIZE = arrayIndexScale(VECTOR_KIND); + + private static void vectorizedCopy(Object src, int srcPos, Object dest, int destPos, int length, Kind baseKind, LocationIdentity locationIdentity) { + int arrayBaseOffset = arrayBaseOffset(baseKind); + int elementSize = arrayIndexScale(baseKind); + long byteLength = (long) length * elementSize; + long srcOffset = (long) srcPos * elementSize; + long destOffset = (long) destPos * elementSize; + + long preLoopBytes; + long mainLoopBytes; + long postLoopBytes; + + // We can easily vectorize the loop if both offsets have the same alignment. + if (byteLength >= VECTOR_SIZE && (srcOffset % VECTOR_SIZE) == (destOffset % VECTOR_SIZE)) { + preLoopBytes = NumUtil.roundUp(arrayBaseOffset + srcOffset, VECTOR_SIZE) - (arrayBaseOffset + srcOffset); + postLoopBytes = (byteLength - preLoopBytes) % VECTOR_SIZE; + mainLoopBytes = byteLength - preLoopBytes - postLoopBytes; + } else { + // Does the architecture support unaligned memory accesses? + if (supportsUnalignedMemoryAccess) { + preLoopBytes = byteLength % VECTOR_SIZE; + mainLoopBytes = byteLength - preLoopBytes; + postLoopBytes = 0; + } else { + // No. Let's do element-wise copying. + preLoopBytes = byteLength; + mainLoopBytes = 0; + postLoopBytes = 0; + } + } + + if (probability(NOT_FREQUENT_PROBABILITY, src == dest) && probability(NOT_FREQUENT_PROBABILITY, srcPos < destPos)) { + // bad aliased case + srcOffset += byteLength; + destOffset += byteLength; + + // Post-loop + for (long i = 0; i < postLoopBytes; i += elementSize) { + srcOffset -= elementSize; + destOffset -= elementSize; + Object a = UnsafeLoadNode.load(src, arrayBaseOffset + srcOffset, baseKind, locationIdentity); + UnsafeStoreNode.store(dest, arrayBaseOffset + destOffset, a, baseKind, locationIdentity); + } + // Main-loop + for (long i = 0; i < mainLoopBytes; i += VECTOR_SIZE) { + srcOffset -= VECTOR_SIZE; + destOffset -= VECTOR_SIZE; + Long a = UnsafeLoadNode.load(src, arrayBaseOffset + srcOffset, VECTOR_KIND, locationIdentity); + UnsafeStoreNode.store(dest, arrayBaseOffset + destOffset, a, VECTOR_KIND, locationIdentity); + } + // Pre-loop + for (long i = 0; i < preLoopBytes; i += elementSize) { + srcOffset -= elementSize; + destOffset -= elementSize; + Object a = UnsafeLoadNode.load(src, arrayBaseOffset + srcOffset, baseKind, locationIdentity); + UnsafeStoreNode.store(dest, arrayBaseOffset + destOffset, a, baseKind, locationIdentity); + } + } else { + // Pre-loop + for (long i = 0; i < preLoopBytes; i += elementSize) { + Object a = UnsafeLoadNode.load(src, arrayBaseOffset + srcOffset, baseKind, locationIdentity); + UnsafeStoreNode.store(dest, arrayBaseOffset + destOffset, a, baseKind, locationIdentity); + srcOffset += elementSize; + destOffset += elementSize; + } + // Main-loop + for (long i = 0; i < mainLoopBytes; i += VECTOR_SIZE) { + Long a = UnsafeLoadNode.load(src, arrayBaseOffset + srcOffset, VECTOR_KIND, locationIdentity); + UnsafeStoreNode.store(dest, arrayBaseOffset + destOffset, a, VECTOR_KIND, locationIdentity); + srcOffset += VECTOR_SIZE; + destOffset += VECTOR_SIZE; + } + // Post-loop + for (long i = 0; i < postLoopBytes; i += elementSize) { + Object a = UnsafeLoadNode.load(src, arrayBaseOffset + srcOffset, baseKind, locationIdentity); + UnsafeStoreNode.store(dest, arrayBaseOffset + destOffset, a, baseKind, locationIdentity); + srcOffset += elementSize; + destOffset += elementSize; + } + } + } + + @Fold + private static LocationIdentity getArrayLocation(Kind kind) { + return NamedLocationIdentity.getArrayLocation(kind); + } + + @Snippet + public static void arraycopyByte(byte[] src, int srcPos, byte[] dest, int destPos, int length) { + Kind kind = Kind.Byte; + vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); + } + + @Snippet + public static void arraycopyBoolean(boolean[] src, int srcPos, boolean[] dest, int destPos, int length) { + Kind kind = Kind.Boolean; + vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); + } + + @Snippet + public static void arraycopyChar(char[] src, int srcPos, char[] dest, int destPos, int length) { + Kind kind = Kind.Char; + vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); + } + + @Snippet + public static void arraycopyShort(short[] src, int srcPos, short[] dest, int destPos, int length) { + Kind kind = Kind.Short; + vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); + } + + @Snippet + public static void arraycopyInt(int[] src, int srcPos, int[] dest, int destPos, int length) { + Kind kind = Kind.Int; + vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); + } + + @Snippet + public static void arraycopyFloat(float[] src, int srcPos, float[] dest, int destPos, int length) { + Kind kind = Kind.Float; + vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); + } + + @Snippet + public static void arraycopyLong(long[] src, int srcPos, long[] dest, int destPos, int length) { + Kind kind = Kind.Long; + vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); + } + + @Snippet + public static void arraycopyDouble(double[] src, int srcPos, double[] dest, int destPos, int length) { + Kind kind = Kind.Double; + /* + * TODO atomicity problem on 32-bit architectures: The JVM spec requires double values to be + * copied atomically, but not long values. For example, on Intel 32-bit this code is not + * atomic as long as the vector kind remains Kind.Long. + */ + vectorizedCopy(src, srcPos, dest, destPos, length, kind, getArrayLocation(kind)); + } + + /** + * For this kind, Object, we want to avoid write barriers between writes, but instead have them + * at the end of the snippet. This is done by using {@link DirectObjectStoreNode}, and rely on + * {@link WriteBarrierAdditionPhase} to put write barriers after the {@link UnsafeArrayCopyNode} + * with kind Object. + */ + @Snippet + public static void arraycopyObject(Object[] src, int srcPos, Object[] dest, int destPos, int length) { + Kind kind = Kind.Object; + final int scale = arrayIndexScale(kind); + int arrayBaseOffset = arrayBaseOffset(kind); + LocationIdentity arrayLocation = getArrayLocation(kind); + if (src == dest && srcPos < destPos) { // bad aliased case + long start = (long) (length - 1) * scale; + for (long i = start; i >= 0; i -= scale) { + Object a = UnsafeLoadNode.load(src, arrayBaseOffset + i + (long) srcPos * scale, kind, arrayLocation); + DirectObjectStoreNode.storeObject(dest, arrayBaseOffset, i + (long) destPos * scale, a, getArrayLocation(kind)); + } + } else { + long end = (long) length * scale; + for (long i = 0; i < end; i += scale) { + Object a = UnsafeLoadNode.load(src, arrayBaseOffset + i + (long) srcPos * scale, kind, arrayLocation); + DirectObjectStoreNode.storeObject(dest, arrayBaseOffset, i + (long) destPos * scale, a, getArrayLocation(kind)); + } + } + } + + @Snippet + public static void arraycopyPrimitive(Object src, int srcPos, Object dest, int destPos, int length, int layoutHelper) { + int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift()) & layoutHelperLog2ElementSizeMask(); + int headerSize = (layoutHelper >> layoutHelperHeaderSizeShift()) & layoutHelperHeaderSizeMask(); + + Unsigned vectorSize = Word.unsigned(VECTOR_SIZE); + Unsigned srcOffset = Word.unsigned(srcPos).shiftLeft(log2ElementSize).add(headerSize); + Unsigned destOffset = Word.unsigned(destPos).shiftLeft(log2ElementSize).add(headerSize); + Unsigned destStart = destOffset; + Unsigned destEnd = destOffset.add(Word.unsigned(length).shiftLeft(log2ElementSize)); + + Unsigned destVectorEnd = null; + Unsigned nonVectorBytes = null; + Unsigned sizeInBytes = Word.unsigned(length).shiftLeft(log2ElementSize); + if (supportsUnalignedMemoryAccess) { + nonVectorBytes = sizeInBytes.unsignedRemainder(vectorSize); + destVectorEnd = destEnd; + } else { + boolean inPhase = srcOffset.and((int) VECTOR_SIZE - 1).equal(destOffset.and((int) VECTOR_SIZE - 1)); + boolean hasAtLeastOneVector = sizeInBytes.aboveOrEqual(vectorSize); + // We must have at least one full vector, otherwise we must copy each byte separately + if (hasAtLeastOneVector && inPhase) { // If in phase, we can vectorize + nonVectorBytes = vectorSize.subtract(destStart.unsignedRemainder(vectorSize)); + } else { // fallback is byte-wise + nonVectorBytes = sizeInBytes; + } + destVectorEnd = destEnd.subtract(destEnd.unsignedRemainder(vectorSize)); + } + + Unsigned destNonVectorEnd = destStart.add(nonVectorBytes); + while (destOffset.belowThan(destNonVectorEnd)) { + ObjectAccess.writeByte(dest, destOffset, ObjectAccess.readByte(src, srcOffset, ANY_LOCATION), ANY_LOCATION); + destOffset = destOffset.add(1); + srcOffset = srcOffset.add(1); + } + // Unsigned destVectorEnd = destEnd.subtract(destEnd.unsignedRemainder(8)); + while (destOffset.belowThan(destVectorEnd)) { + ObjectAccess.writeWord(dest, destOffset, ObjectAccess.readWord(src, srcOffset, ANY_LOCATION), ANY_LOCATION); + destOffset = destOffset.add(wordSize()); + srcOffset = srcOffset.add(wordSize()); + } + // Do the last bytes each when it is required to have absolute alignment. + while (!supportsUnalignedMemoryAccess && destOffset.belowThan(destEnd)) { + ObjectAccess.writeByte(dest, destOffset, ObjectAccess.readByte(src, srcOffset, ANY_LOCATION), ANY_LOCATION); + destOffset = destOffset.add(1); + srcOffset = srcOffset.add(1); + } + } + + public static class Templates extends AbstractTemplates { + + private final SnippetInfo[] arraycopySnippets; + private final SnippetInfo genericPrimitiveSnippet; + + public Templates(HotSpotProviders providers, TargetDescription target) { + super(providers, providers.getSnippetReflection(), target); + + arraycopySnippets = new SnippetInfo[Kind.values().length]; + arraycopySnippets[Kind.Boolean.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyBoolean"); + arraycopySnippets[Kind.Byte.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyByte"); + arraycopySnippets[Kind.Short.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyShort"); + arraycopySnippets[Kind.Char.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyChar"); + arraycopySnippets[Kind.Int.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyInt"); + arraycopySnippets[Kind.Long.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyLong"); + arraycopySnippets[Kind.Float.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyFloat"); + arraycopySnippets[Kind.Double.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyDouble"); + arraycopySnippets[Kind.Object.ordinal()] = snippet(UnsafeArrayCopySnippets.class, "arraycopyObject"); + + genericPrimitiveSnippet = snippet(UnsafeArrayCopySnippets.class, "arraycopyPrimitive"); + } + + public void lower(UnsafeArrayCopyNode node, LoweringTool tool) { + Kind elementKind = node.getElementKind(); + SnippetInfo snippet; + if (elementKind == null) { + // primitive array of unknown kind + snippet = genericPrimitiveSnippet; + } else { + snippet = arraycopySnippets[elementKind.ordinal()]; + assert snippet != null : "arraycopy snippet for " + elementKind.name() + " not found"; + } + + Arguments args = new Arguments(snippet, node.graph().getGuardsStage(), tool.getLoweringStage()); + node.addSnippetArguments(args); + + SnippetTemplate template = template(args); + template.instantiate(providers.getMetaAccess(), node, DEFAULT_REPLACER, args); + } + } +} diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.graal.lir/src/com/oracle/graal/lir/LIRFrameState.java --- a/graal/com.oracle.graal.lir/src/com/oracle/graal/lir/LIRFrameState.java Thu Dec 04 23:33:24 2014 +0100 +++ b/graal/com.oracle.graal.lir/src/com/oracle/graal/lir/LIRFrameState.java Thu Dec 04 23:34:27 2014 +0100 @@ -134,6 +134,13 @@ } /** + * Updates this reference map with all references that are marked in {@code refMap}. + */ + public void updateUnion(ReferenceMap refMap) { + debugInfo.getReferenceMap().updateUnion(refMap); + } + + /** * Called by the register allocator after all locations are marked. * * @param op The instruction to which this frame state belongs. diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.graal.lir/src/com/oracle/graal/lir/framemap/FrameMap.java --- a/graal/com.oracle.graal.lir/src/com/oracle/graal/lir/framemap/FrameMap.java Thu Dec 04 23:33:24 2014 +0100 +++ b/graal/com.oracle.graal.lir/src/com/oracle/graal/lir/framemap/FrameMap.java Thu Dec 04 23:34:27 2014 +0100 @@ -337,4 +337,16 @@ assert isConstant(location); } } + + public void clearReference(Value location, ReferenceMap refMap) { + LIRKind kind = location.getLIRKind(); + if (isRegister(location)) { + refMap.clearRegister(asRegister(location).getReferenceMapIndex(), kind); + } else if (isStackSlot(location)) { + int offset = offsetForStackSlot(asStackSlot(location)); + refMap.clearStackSlot(offset, kind); + } else { + assert isConstant(location); + } + } } diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.truffle.api.dsl.test/src/com/oracle/truffle/api/dsl/test/ArrayTest.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/graal/com.oracle.truffle.api.dsl.test/src/com/oracle/truffle/api/dsl/test/ArrayTest.java Thu Dec 04 23:34:27 2014 +0100 @@ -0,0 +1,122 @@ +/* + * Copyright (c) 2014, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.truffle.api.dsl.test; + +import org.junit.*; + +import com.oracle.truffle.api.*; +import com.oracle.truffle.api.dsl.*; +import com.oracle.truffle.api.dsl.test.ArrayTestFactory.TestNode1Factory; +import com.oracle.truffle.api.frame.*; +import com.oracle.truffle.api.nodes.*; + +public class ArrayTest { + + @Test + public void testNode1() { + final TestNode1 node = TestNode1Factory.create(null); + RootNode root = new RootNode() { + @Child TestNode1 test = node; + + @Override + public Object execute(VirtualFrame frame) { + return test.executeWith(frame, frame.getArguments()[0]); + } + }; + CallTarget target = Truffle.getRuntime().createCallTarget(root); + + Assert.assertEquals(1, (int) target.call(1)); + Assert.assertArrayEquals(new double[0], (double[]) target.call(new int[0]), 0.0d); + Assert.assertArrayEquals(new double[0], (double[]) target.call(new double[0]), 0.0d); + Assert.assertArrayEquals(new String[0], (String[]) target.call((Object) new String[0])); + } + + @TypeSystemReference(ArrayTypeSystem.class) + abstract static class BaseNode extends Node { + + abstract Object execute(VirtualFrame frame); + + int executeInt(VirtualFrame frame) throws UnexpectedResultException { + return ArrayTypeSystemGen.ARRAYTYPESYSTEM.expectInteger(execute(frame)); + } + + int[] executeIntArray(VirtualFrame frame) throws UnexpectedResultException { + return ArrayTypeSystemGen.ARRAYTYPESYSTEM.expectIntArray(execute(frame)); + } + + String[] executeStringArray(VirtualFrame frame) throws UnexpectedResultException { + return ArrayTypeSystemGen.ARRAYTYPESYSTEM.expectStringArray(execute(frame)); + } + + double[] executeDoubleArray(VirtualFrame frame) throws UnexpectedResultException { + return ArrayTypeSystemGen.ARRAYTYPESYSTEM.expectDoubleArray(execute(frame)); + } + } + + @NodeChild + abstract static class TestNode1 extends BaseNode { + + abstract Object executeWith(VirtualFrame frame, Object operand); + + @Specialization + int doInt(int value) { + return value; + } + + @Specialization + double[] doDoubleArray(double[] value) { + return value; + } + + @Specialization + String[] doStringArray(String[] value) { + return value; + } + + } + + @TypeSystem({int.class, int[].class, double[].class, String[].class, Object[].class}) + public static class ArrayTypeSystem { + + @ImplicitCast + public double[] castFromInt(int[] array) { + double[] newArray = new double[array.length]; + for (int i = 0; i < array.length; i++) { + newArray[i] = array[i]; + } + return newArray; + } + + @TypeCheck + public boolean isIntArray(Object array) { + return array instanceof int[]; + } + + @TypeCast + public int[] asIntArray(Object array) { + return (int[]) array; + } + + } + +} diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.truffle.dsl.processor/src/com/oracle/truffle/dsl/processor/generator/NodeCodeGenerator.java --- a/graal/com.oracle.truffle.dsl.processor/src/com/oracle/truffle/dsl/processor/generator/NodeCodeGenerator.java Thu Dec 04 23:33:24 2014 +0100 +++ b/graal/com.oracle.truffle.dsl.processor/src/com/oracle/truffle/dsl/processor/generator/NodeCodeGenerator.java Thu Dec 04 23:34:27 2014 +0100 @@ -322,12 +322,12 @@ /** * 
      * variant1 $condition != null
-     * 
+     *
      * $type $name = defaultValue($type);
      * if ($condition) {
      *     $name = $value;
      * }
-     * 
+     *
      * variant2 $condition != null
      * $type $name = $value;
      *
@@ -1993,7 +1993,7 @@ } String prefix = expect ? "expect" : "execute"; String suffix = execution.getIndex() > -1 ? String.valueOf(execution.getIndex()) : ""; - return prefix + ElementUtils.firstLetterUpperCase(child.getName()) + ElementUtils.firstLetterUpperCase(ElementUtils.getSimpleName(param.getType())) + suffix; + return prefix + ElementUtils.firstLetterUpperCase(child.getName()) + ElementUtils.firstLetterUpperCase(ElementUtils.getTypeId(param.getType())) + suffix; } private List createExecuteChilds(Parameter param, Set expectTypes) { diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.truffle.dsl.processor/src/com/oracle/truffle/dsl/processor/java/model/CodeTreeBuilder.java --- a/graal/com.oracle.truffle.dsl.processor/src/com/oracle/truffle/dsl/processor/java/model/CodeTreeBuilder.java Thu Dec 04 23:33:24 2014 +0100 +++ b/graal/com.oracle.truffle.dsl.processor/src/com/oracle/truffle/dsl/processor/java/model/CodeTreeBuilder.java Thu Dec 04 23:34:27 2014 +0100 @@ -672,10 +672,6 @@ } public CodeTreeBuilder instanceOf(String var, TypeMirror type) { - TypeElement element = ElementUtils.fromTypeMirror(type); - if (element == null) { - throw new IllegalArgumentException("Cannot call instanceof for a non supported type: " + type.getKind()); - } return instanceOf(singleString(var), singleType(type)); } diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.truffle.dsl.processor/src/com/oracle/truffle/dsl/processor/model/TypeSystemData.java --- a/graal/com.oracle.truffle.dsl.processor/src/com/oracle/truffle/dsl/processor/model/TypeSystemData.java Thu Dec 04 23:33:24 2014 +0100 +++ b/graal/com.oracle.truffle.dsl.processor/src/com/oracle/truffle/dsl/processor/model/TypeSystemData.java Thu Dec 04 23:34:27 2014 +0100 @@ -139,7 +139,7 @@ public TypeData findType(String simpleName) { for (TypeData type : types) { - if (ElementUtils.getSimpleName(type.getBoxedType()).equals(simpleName)) { + if (ElementUtils.getTypeId(type.getBoxedType()).equals(simpleName)) { return type; } } diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.truffle.dsl.processor/src/com/oracle/truffle/dsl/processor/parser/TypeSystemParser.java --- a/graal/com.oracle.truffle.dsl.processor/src/com/oracle/truffle/dsl/processor/parser/TypeSystemParser.java Thu Dec 04 23:33:24 2014 +0100 +++ b/graal/com.oracle.truffle.dsl.processor/src/com/oracle/truffle/dsl/processor/parser/TypeSystemParser.java Thu Dec 04 23:34:27 2014 +0100 @@ -216,7 +216,11 @@ typeData.addError("Invalid type order. The type(s) %s are inherited from a earlier defined type %s.", invalidTypes.get(ElementUtils.getQualifiedName(type)), ElementUtils.getQualifiedName(type)); } - List nextInvalidTypes = ElementUtils.getQualifiedSuperTypeNames(ElementUtils.fromTypeMirror(type)); + TypeElement element = ElementUtils.fromTypeMirror(type); + List nextInvalidTypes = new ArrayList<>(); + if (element != null) { + nextInvalidTypes.addAll(ElementUtils.getQualifiedSuperTypeNames(element)); + } nextInvalidTypes.add(getQualifiedName(type)); for (String qualifiedName : nextInvalidTypes) { diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.truffle.object.basic/src/com/oracle/truffle/object/basic/ShapeBasic.java --- a/graal/com.oracle.truffle.object.basic/src/com/oracle/truffle/object/basic/ShapeBasic.java Thu Dec 04 23:33:24 2014 +0100 +++ b/graal/com.oracle.truffle.object.basic/src/com/oracle/truffle/object/basic/ShapeBasic.java Thu Dec 04 23:34:27 2014 +0100 @@ -30,13 +30,13 @@ super(layout, operations, sharedData, id); } - public ShapeBasic(Layout layout, Object sharedData, ShapeImpl parent, ObjectType operations, PropertyMap propertyMap, Allocator allocator, int id) { - super(layout, parent, operations, sharedData, propertyMap, allocator, id); + public ShapeBasic(Layout layout, Object sharedData, ShapeImpl parent, ObjectType objectType, PropertyMap propertyMap, Transition transition, Allocator allocator, int id) { + super(layout, parent, objectType, sharedData, propertyMap, transition, allocator, id); } @SuppressWarnings("hiding") @Override - protected ShapeImpl createShape(Layout layout, Object sharedData, ShapeImpl parent, ObjectType operations, PropertyMap propertyMap, Allocator allocator, int id) { - return new ShapeBasic(layout, sharedData, parent, operations, propertyMap, allocator, id); + protected ShapeImpl createShape(Layout layout, Object sharedData, ShapeImpl parent, ObjectType objectType, PropertyMap propertyMap, Transition transition, Allocator allocator, int id) { + return new ShapeBasic(layout, sharedData, parent, objectType, propertyMap, transition, allocator, id); } } diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.truffle.object/src/com/oracle/truffle/object/ShapeImpl.java --- a/graal/com.oracle.truffle.object/src/com/oracle/truffle/object/ShapeImpl.java Thu Dec 04 23:33:24 2014 +0100 +++ b/graal/com.oracle.truffle.object/src/com/oracle/truffle/object/ShapeImpl.java Thu Dec 04 23:34:27 2014 +0100 @@ -29,16 +29,18 @@ import com.oracle.truffle.api.CompilerDirectives.TruffleBoundary; import com.oracle.truffle.api.nodes.*; import com.oracle.truffle.api.object.*; +import com.oracle.truffle.api.utilities.*; import com.oracle.truffle.object.LocationImpl.InternalLongLocation; import com.oracle.truffle.object.Locations.ConstantLocation; import com.oracle.truffle.object.Locations.DeclaredDualLocation; import com.oracle.truffle.object.Locations.DeclaredLocation; import com.oracle.truffle.object.Locations.DualLocation; import com.oracle.truffle.object.Locations.ValueLocation; -import com.oracle.truffle.object.Transition.AddTransition; -import com.oracle.truffle.object.Transition.OperationsTransition; +import com.oracle.truffle.object.Transition.AddPropertyTransition; +import com.oracle.truffle.object.Transition.ObjectTypeTransition; import com.oracle.truffle.object.Transition.PropertyTransition; -import com.oracle.truffle.object.Transition.RemoveTransition; +import com.oracle.truffle.object.Transition.RemovePropertyTransition; +import com.oracle.truffle.object.Transition.ReservePrimitiveArrayTransition; /** * Shape objects create a mapping of Property objects to indexes. The mapping of those indexes to an @@ -77,7 +79,7 @@ protected Property[] propertyArray; protected final Assumption validAssumption; - protected final Assumption leafAssumption; + @CompilationFinal protected volatile Assumption leafAssumption; /** * Shape transition map; lazily initialized. @@ -87,18 +89,24 @@ */ private HashMap transitionMap; + private final Transition transitionFromParent; + /** * Private constructor. * - * @see #ShapeImpl(Layout, ShapeImpl, ObjectType, Object, PropertyMap, BaseAllocator, int) + * @param parent predecessor shape + * @param transitionFromParent direct transition from parent shape + * @see #ShapeImpl(Layout, ShapeImpl, ObjectType, Object, PropertyMap, Transition, + * BaseAllocator, int) */ - private ShapeImpl(Layout layout, ShapeImpl parent, ObjectType operations, Object sharedData, PropertyMap propertyMap, int objectArraySize, int objectFieldSize, int primitiveFieldSize, - int primitiveArraySize, boolean hasPrimitiveArray, int id) { + private ShapeImpl(Layout layout, ShapeImpl parent, ObjectType objectType, Object sharedData, PropertyMap propertyMap, Transition transitionFromParent, int objectArraySize, int objectFieldSize, + int primitiveFieldSize, int primitiveArraySize, boolean hasPrimitiveArray, int id) { this.layout = (LayoutImpl) layout; - this.objectType = Objects.requireNonNull(operations); + this.objectType = Objects.requireNonNull(objectType); this.propertyMap = Objects.requireNonNull(propertyMap); this.root = parent != null ? parent.getRoot() : this; this.parent = parent; + this.transitionFromParent = transitionFromParent; this.objectArraySize = objectArraySize; this.objectArrayCapacity = capacityFromSize(objectArraySize); this.objectFieldSize = objectFieldSize; @@ -118,27 +126,26 @@ } this.validAssumption = createValidAssumption(); - this.leafAssumption = createLeafAssumption(); this.id = id; shapeCount.inc(); this.sharedData = sharedData; - this.extraData = operations.createShapeData(this); + this.extraData = objectType.createShapeData(this); debugRegisterShape(this); } - protected ShapeImpl(Layout layout, ShapeImpl parent, ObjectType operations, Object sharedData, PropertyMap propertyMap, Allocator allocator, int id) { - this(layout, parent, operations, sharedData, propertyMap, ((BaseAllocator) allocator).objectArraySize, ((BaseAllocator) allocator).objectFieldSize, + protected ShapeImpl(Layout layout, ShapeImpl parent, ObjectType operations, Object sharedData, PropertyMap propertyMap, Transition transition, Allocator allocator, int id) { + this(layout, parent, operations, sharedData, propertyMap, transition, ((BaseAllocator) allocator).objectArraySize, ((BaseAllocator) allocator).objectFieldSize, ((BaseAllocator) allocator).primitiveFieldSize, ((BaseAllocator) allocator).primitiveArraySize, ((BaseAllocator) allocator).hasPrimitiveArray, id); } @SuppressWarnings("hiding") - protected abstract ShapeImpl createShape(Layout layout, Object sharedData, ShapeImpl parent, ObjectType operations, PropertyMap propertyMap, Allocator allocator, int id); + protected abstract ShapeImpl createShape(Layout layout, Object sharedData, ShapeImpl parent, ObjectType operations, PropertyMap propertyMap, Transition transition, Allocator allocator, int id); protected ShapeImpl(Layout layout, ObjectType operations, Object sharedData, int id) { - this(layout, null, operations, sharedData, PropertyMap.empty(), layout.createAllocator(), id); + this(layout, null, operations, sharedData, PropertyMap.empty(), null, layout.createAllocator(), id); } private static int makePropertyCount(ShapeImpl parent, PropertyMap propertyMap) { @@ -235,7 +242,7 @@ public final ShapeImpl getShapeFromProperty(Object propertyName) { ShapeImpl current = this; while (current != getRoot()) { - if (current.getLastProperty().getKey().equals(propertyName)) { + if (current.getTransitionFromParent() instanceof AddPropertyTransition && ((AddPropertyTransition) current.getTransitionFromParent()).getProperty().getKey().equals(propertyName)) { return current; } current = current.getParent(); @@ -250,7 +257,7 @@ public final ShapeImpl getShapeFromProperty(Property prop) { ShapeImpl current = this; while (current != getRoot()) { - if (current.getLastProperty().equals(prop)) { + if (current.getTransitionFromParent() instanceof AddPropertyTransition && ((AddPropertyTransition) current.getTransitionFromParent()).getProperty().equals(prop)) { return current; } current = current.parent; @@ -280,13 +287,13 @@ return null; } - protected final void addTransition(Transition transition, ShapeImpl next) { - assert next.getParent() == this; + protected final void addDirectTransition(Transition transition, ShapeImpl next) { + assert next.getParent() == this && transition.isDirect(); addTransitionInternal(transition, next); } - public final void addNonlinearTransition(Transition transition, ShapeImpl next) { - assert next.getParent() != this; + public final void addIndirectTransition(Transition transition, ShapeImpl next) { + assert next.getParent() != this && !transition.isDirect(); addTransitionInternal(transition, next); } @@ -311,6 +318,17 @@ return propertyMap; } + private ShapeImpl queryTransition(Transition transition) { + ShapeImpl cachedShape = this.getTransitionMapForRead().get(transition); + if (cachedShape != null) { // Shape already exists? + shapeCacheHitCount.inc(); + return (ShapeImpl) layout.getStrategy().returnCached(cachedShape); + } + shapeCacheMissCount.inc(); + + return null; + } + /** * Add a new property in the map, yielding a new or cached Shape object. * @@ -340,24 +358,21 @@ assert !getPropertyListInternal(false).contains(prop); // invalidatePropertyAssumption(prop.getName()); - Transition.AddTransition key = new Transition.AddTransition(prop); - Map transitionMapForRead = this.getTransitionMapForRead(); - ShapeImpl cachedShape = transitionMapForRead.get(key); - if (cachedShape != null) { // Shape already exists? - shapeCacheHitCount.inc(); - return (ShapeImpl) layout.getStrategy().returnCached(cachedShape); + AddPropertyTransition addTransition = new AddPropertyTransition(prop); + ShapeImpl cachedShape = queryTransition(addTransition); + if (cachedShape != null) { + return cachedShape; } - shapeCacheMissCount.inc(); ShapeImpl oldShape = (ShapeImpl) layout.getStrategy().ensureSpace(this, prop.getLocation()); - ShapeImpl newShape = makeShapeWithAddedProperty(oldShape, key); - oldShape.addTransition(key, newShape); + ShapeImpl newShape = makeShapeWithAddedProperty(oldShape, addTransition); + oldShape.addDirectTransition(addTransition, newShape); return newShape; } protected ShapeImpl cloneRoot(ShapeImpl from, Object newSharedData) { - return createShape(from.layout, newSharedData, null, from.objectType, from.propertyMap, from.allocator(), from.id); + return createShape(from.layout, newSharedData, null, from.objectType, from.propertyMap, null, from.allocator(), from.id); } /** @@ -367,34 +382,29 @@ */ protected final ShapeImpl cloneOnto(ShapeImpl newParent) { ShapeImpl from = this; - ShapeImpl newShape = createShape(newParent.layout, newParent.sharedData, newParent, from.objectType, from.propertyMap, from.allocator(), newParent.id); + ShapeImpl newShape = createShape(newParent.layout, newParent.sharedData, newParent, from.objectType, from.propertyMap, from.transitionFromParent, from.allocator(), newParent.id); shapeCloneCount.inc(); // (aw) need to have this transition for obsolescence - newParent.addTransition(getTransitionFromParent(), newShape); + newParent.addDirectTransition(from.transitionFromParent, newShape); return newShape; } - private Transition getTransitionFromParent() { - for (Map.Entry property : parent.getTransitionMapForRead().entrySet()) { - if (property.getValue() == this) { - return property.getKey(); - } - } - throw new NoSuchElementException(); + public final Transition getTransitionFromParent() { + return transitionFromParent; } /** * Create a new shape that adds a property to the parent shape. */ - private static ShapeImpl makeShapeWithAddedProperty(ShapeImpl parent, AddTransition addTransition) { + private static ShapeImpl makeShapeWithAddedProperty(ShapeImpl parent, AddPropertyTransition addTransition) { Property addend = addTransition.getProperty(); BaseAllocator allocator = parent.allocator().addLocation(addend.getLocation()); PropertyMap newPropertyMap = parent.propertyMap.putCopy(addend); - ShapeImpl newShape = parent.createShape(parent.layout, parent.sharedData, parent, parent.objectType, newPropertyMap, allocator, parent.id); + ShapeImpl newShape = parent.createShape(parent.layout, parent.sharedData, parent, parent.objectType, newPropertyMap, addTransition, allocator, parent.id); assert ((LocationImpl) addend.getLocation()).primitiveArrayCount() == 0 || newShape.hasPrimitiveArray; assert newShape.depth == allocator.depth; return newShape; @@ -403,11 +413,11 @@ /** * Create a new shape that reserves the primitive extension array field. */ - private static ShapeImpl makeShapeWithPrimitiveExtensionArray(ShapeImpl parent) { + private static ShapeImpl makeShapeWithPrimitiveExtensionArray(ShapeImpl parent, Transition transition) { assert parent.getLayout().hasPrimitiveExtensionArray(); assert !parent.hasPrimitiveArray(); BaseAllocator allocator = parent.allocator().addLocation(parent.getLayout().getPrimitiveArrayLocation()); - ShapeImpl newShape = parent.createShape(parent.layout, parent.sharedData, parent, parent.objectType, parent.propertyMap, allocator, parent.id); + ShapeImpl newShape = parent.createShape(parent.layout, parent.sharedData, parent, parent.objectType, parent.propertyMap, transition, allocator, parent.id); assert newShape.hasPrimitiveArray(); assert newShape.depth == allocator.depth; return newShape; @@ -415,17 +425,15 @@ private ShapeImpl addPrimitiveExtensionArray() { assert layout.hasPrimitiveExtensionArray() && !hasPrimitiveArray(); - Transition key = new Transition.ReservePrimitiveArrayTransition(); - ShapeImpl cachedShape = this.getTransitionMapForRead().get(key); + Transition transition = new ReservePrimitiveArrayTransition(); + ShapeImpl cachedShape = queryTransition(transition); if (cachedShape != null) { - shapeCacheHitCount.inc(); - return (ShapeImpl) layout.getStrategy().returnCached(cachedShape); + return cachedShape; } - shapeCacheMissCount.inc(); ShapeImpl oldShape = (ShapeImpl) layout.getStrategy().ensureSpace(this, layout.getPrimitiveArrayLocation()); - ShapeImpl newShape = makeShapeWithPrimitiveExtensionArray(oldShape); - oldShape.addTransition(key, newShape); + ShapeImpl newShape = makeShapeWithPrimitiveExtensionArray(oldShape, transition); + oldShape.addDirectTransition(transition, newShape); return newShape; } @@ -455,8 +463,7 @@ @Override public final List getPropertyList(Pred filter) { LinkedList props = new LinkedList<>(); - next: for (ShapeImpl current = this; current != getRoot(); current = current.parent) { - Property currentProperty = current.getLastProperty(); + next: for (Property currentProperty : this.propertyMap.reverseOrderValues()) { if (!currentProperty.isHidden() && filter.test(currentProperty)) { if (currentProperty.getLocation() instanceof DeclaredLocation) { for (Iterator iter = props.iterator(); iter.hasNext();) { @@ -488,11 +495,11 @@ @Override public final List getPropertyListInternal(boolean ascending) { LinkedList props = new LinkedList<>(); - for (ShapeImpl current = this; current != getRoot(); current = current.parent) { + for (Property current : this.propertyMap.reverseOrderValues()) { if (ascending) { - props.addFirst(current.getLastProperty()); + props.addFirst(current); } else { - props.add(current.getLastProperty()); + props.add(current); } } return props; @@ -507,8 +514,7 @@ @Override public final List getKeyList(Pred filter) { LinkedList keys = new LinkedList<>(); - for (ShapeImpl current = this; current != getRoot(); current = current.parent) { - Property currentProperty = current.getLastProperty(); + for (Property currentProperty : this.propertyMap.reverseOrderValues()) { if (!currentProperty.isHidden() && filter.test(currentProperty) && !currentProperty.isShadow()) { keys.addFirst(currentProperty.getKey()); } @@ -526,12 +532,9 @@ return getKeyList(); } - public final void setTypeTransition(ShapeImpl successorShape, Property before, Property after) { - getTransitionMapForWrite().put(new Transition.TypeTransition(before, after), successorShape); - } - - private static Assumption createValidAssumption() { - return Truffle.getRuntime().createAssumption("valid shape"); + @Override + public final boolean isValid() { + return getValidAssumption().isValid(); } @Override @@ -539,27 +542,47 @@ return validAssumption; } + private static Assumption createValidAssumption() { + return Truffle.getRuntime().createAssumption("valid shape"); + } + + public final void invalidateValidAssumption() { + getValidAssumption().invalidate(); + } + @Override - public final boolean isValid() { - return getValidAssumption().isValid(); + public final boolean isLeaf() { + return leafAssumption == null || leafAssumption.isValid(); + } + + @Override + public final Assumption getLeafAssumption() { + if (leafAssumption == null) { + CompilerDirectives.transferToInterpreterAndInvalidate(); + synchronized (getMutex()) { + if (leafAssumption == null) { + leafAssumption = isLeafHelper() ? createLeafAssumption() : NeverValidAssumption.INSTANCE; + } + } + } + return leafAssumption; + } + + private boolean isLeafHelper() { + return getTransitionMapForRead().isEmpty(); } private static Assumption createLeafAssumption() { return Truffle.getRuntime().createAssumption("leaf shape"); } - @Override - public final Assumption getLeafAssumption() { - return leafAssumption; - } - - @Override - public final boolean isLeaf() { - return getLeafAssumption().isValid(); - } - private void invalidateLeafAssumption() { - getLeafAssumption().invalidate(); + Assumption assumption = leafAssumption; + if (assumption != null) { + assumption.invalidate(); + } else { + leafAssumption = NeverValidAssumption.INSTANCE; + } } @Override @@ -611,24 +634,24 @@ @TruffleBoundary @Override public final ShapeImpl removeProperty(Property prop) { - RemoveTransition transition = new RemoveTransition(prop); - ShapeImpl cachedShape = getTransitionMapForRead().get(transition); + RemovePropertyTransition transition = new RemovePropertyTransition(prop); + ShapeImpl cachedShape = queryTransition(transition); if (cachedShape != null) { - return (ShapeImpl) layout.getStrategy().returnCached(cachedShape); + return cachedShape; } ShapeImpl shape = getShapeFromProperty(prop); if (shape != null) { - List properties = new ArrayList<>(); + List transitionList = new ArrayList<>(); ShapeImpl current = this; while (current != shape) { - properties.add(current.getLastProperty()); + transitionList.add(current.getTransitionFromParent()); current = current.parent; } ShapeImpl newShape = shape.parent; - for (ListIterator iterator = properties.listIterator(properties.size()); iterator.hasPrevious();) { - Property previous = iterator.previous(); - newShape = newShape.append(previous); + for (ListIterator iterator = transitionList.listIterator(transitionList.size()); iterator.hasPrevious();) { + Transition previous = iterator.previous(); + newShape = newShape.applyTransition(previous, true); } getTransitionMapForWrite().put(transition, newShape); @@ -644,6 +667,18 @@ return addProperty(oldProperty.relocate(allocator().moveLocation(oldProperty.getLocation()))); } + public final ShapeImpl applyTransition(Transition transition, boolean append) { + if (transition instanceof AddPropertyTransition) { + return append ? append(((AddPropertyTransition) transition).getProperty()) : addProperty(((AddPropertyTransition) transition).getProperty()); + } else if (transition instanceof ObjectTypeTransition) { + return changeType(((ObjectTypeTransition) transition).getObjectType()); + } else if (transition instanceof ReservePrimitiveArrayTransition) { + return reservePrimitiveExtensionArray(); + } else { + throw new UnsupportedOperationException(); + } + } + @Override public final BaseAllocator allocator() { return layout.getStrategy().createAllocator(this); @@ -653,27 +688,34 @@ * Duplicate shape exchanging existing property with new property. */ @Override - public final ShapeImpl replaceProperty(Property oldProperty, Property newProp) { + public final ShapeImpl replaceProperty(Property oldProperty, Property newProperty) { + Transition replacePropertyTransition = new Transition.ReplacePropertyTransition(oldProperty, newProperty); + ShapeImpl cachedShape = queryTransition(replacePropertyTransition); + if (cachedShape != null) { + return cachedShape; + } + ShapeImpl top = this; - List propertyList = new ArrayList<>(); + List transitionList = new ArrayList<>(); boolean found = false; while (top != getRoot() && !found) { - Property prop = top.getLastProperty(); - propertyList.add(prop); - if (prop.getKey().equals(newProp.getKey())) { + Transition transition = top.getTransitionFromParent(); + transitionList.add(transition); + if (transition instanceof AddPropertyTransition && ((AddPropertyTransition) transition).getProperty().getKey().equals(newProperty.getKey())) { found = true; } top = top.parent; } ShapeImpl newShape = top; - for (ListIterator iterator = propertyList.listIterator(propertyList.size()); iterator.hasPrevious();) { - Property prop = iterator.previous(); - if (prop.getKey().equals(newProp.getKey())) { - newShape = newShape.addProperty(newProp); + for (ListIterator iterator = transitionList.listIterator(transitionList.size()); iterator.hasPrevious();) { + Transition transition = iterator.previous(); + if (transition instanceof AddPropertyTransition && ((AddPropertyTransition) transition).getProperty().getKey().equals(newProperty.getKey())) { + newShape = newShape.addProperty(newProperty); } else { - newShape = newShape.addProperty(prop); + newShape = newShape.applyTransition(transition, false); } } + addIndirectTransition(replacePropertyTransition, newShape); return newShape; } @@ -725,9 +767,6 @@ return newShape; } - /** - * NB: this is not an accurate property count. - */ @Override public final int getPropertyCount() { return propertyCount; @@ -809,19 +848,19 @@ @Override @TruffleBoundary public final ShapeImpl changeType(ObjectType newOps) { - OperationsTransition transition = new OperationsTransition(newOps); - ShapeImpl cachedShape = getTransitionMapForRead().get(transition); + ObjectTypeTransition transition = new ObjectTypeTransition(newOps); + ShapeImpl cachedShape = queryTransition(transition); if (cachedShape != null) { return cachedShape; - } else { - ShapeImpl newShape = createShape(layout, extraData, this, newOps, propertyMap, allocator(), id); - addTransition(transition, newShape); - return newShape; } + + ShapeImpl newShape = createShape(layout, sharedData, this, newOps, propertyMap, transition, allocator(), id); + addDirectTransition(transition, newShape); + return newShape; } @Override - public final Shape reservePrimitiveExtensionArray() { + public final ShapeImpl reservePrimitiveExtensionArray() { if (layout.hasPrimitiveExtensionArray() && !hasPrimitiveArray()) { return addPrimitiveExtensionArray(); } diff -r 12bad81babff -r 5ec45cb4bf22 graal/com.oracle.truffle.object/src/com/oracle/truffle/object/Transition.java --- a/graal/com.oracle.truffle.object/src/com/oracle/truffle/object/Transition.java Thu Dec 04 23:33:24 2014 +0100 +++ b/graal/com.oracle.truffle.object/src/com/oracle/truffle/object/Transition.java Thu Dec 04 23:34:27 2014 +0100 @@ -47,6 +47,8 @@ return true; } + public abstract boolean isDirect(); + public abstract static class PropertyTransition extends Transition { private final Property property; @@ -79,47 +81,62 @@ } } - public static final class AddTransition extends PropertyTransition { - public AddTransition(Property property) { + public static final class AddPropertyTransition extends PropertyTransition { + public AddPropertyTransition(Property property) { super(property); } + + @Override + public boolean isDirect() { + return true; + } } - public static final class RemoveTransition extends PropertyTransition { - public RemoveTransition(Property property) { + public static final class RemovePropertyTransition extends PropertyTransition { + public RemovePropertyTransition(Property property) { super(property); } + + @Override + public boolean isDirect() { + return false; + } } - public static final class OperationsTransition extends Transition { - private final ObjectType operations; + public static final class ObjectTypeTransition extends Transition { + private final ObjectType objectType; - public OperationsTransition(ObjectType operations) { - this.operations = operations; + public ObjectTypeTransition(ObjectType objectType) { + this.objectType = objectType; } - public ObjectType getOperations() { - return operations; + public ObjectType getObjectType() { + return objectType; } @Override public boolean equals(Object other) { - return super.equals(other) && Objects.equals(operations, ((OperationsTransition) other).operations); + return super.equals(other) && Objects.equals(objectType, ((ObjectTypeTransition) other).objectType); } @Override public int hashCode() { final int prime = 31; int result = super.hashCode(); - result = prime * result + ((operations == null) ? 0 : operations.hashCode()); + result = prime * result + ((objectType == null) ? 0 : objectType.hashCode()); return result; } + + @Override + public boolean isDirect() { + return true; + } } - public static final class TypeTransition extends PropertyTransition { + public static final class ReplacePropertyTransition extends PropertyTransition { private final Property after; - public TypeTransition(Property before, Property after) { + public ReplacePropertyTransition(Property before, Property after) { super(before); this.after = after; } @@ -131,11 +148,21 @@ public Property getPropertyAfter() { return after; } + + @Override + public boolean isDirect() { + return false; + } } public static final class ReservePrimitiveArrayTransition extends Transition { public ReservePrimitiveArrayTransition() { } + + @Override + public boolean isDirect() { + return true; + } } public String getShortName() { diff -r 12bad81babff -r 5ec45cb4bf22 make/Makefile --- a/make/Makefile Thu Dec 04 23:33:24 2014 +0100 +++ b/make/Makefile Thu Dec 04 23:34:27 2014 +0100 @@ -306,9 +306,20 @@ $(MAKE_ARGS) $(VM_TARGET) + +ifeq (, $(shell python2.7 --version 2>/dev/null && echo ok)) + ifeq (, $(shell python2.6 --version 2>/dev/null && echo ok)) + PYTHON=python + else + PYTHON=python2.6 + endif +else + PYTHON=python2.7 +endif + # Builds code that can be shared among different build flavors buildshared: - python2.7 -u $(GAMMADIR)/mxtool/mx.py build --no-native --export-dir $(SHARED_DIR) + $(PYTHON) -u $(GAMMADIR)/mxtool/mx.py build --no-native --export-dir $(SHARED_DIR) # Export file rule generic_export: $(EXPORT_LIST) diff -r 12bad81babff -r 5ec45cb4bf22 mx/mx_graal.py --- a/mx/mx_graal.py Thu Dec 04 23:33:24 2014 +0100 +++ b/mx/mx_graal.py Thu Dec 04 23:34:27 2014 +0100 @@ -721,7 +721,7 @@ major, minor = map(int, most_recent_tag_version.split('.')) cached_graal_version = str(major) + '.' + str(minor + 1) + '-' + dev_suffix else: - cached_graal_version = 'unknown-{}-{}'.format(platform.node(), time.strftime('%Y-%m-%d_%H-%M-%S_%Z')) + cached_graal_version = 'unknown-{0}-{1}'.format(platform.node(), time.strftime('%Y-%m-%d_%H-%M-%S_%Z')) return cached_graal_version @@ -1438,7 +1438,7 @@ with VM('server', 'product'): # hosted mode with Task('UnitTests:hosted-product', tasks): - unittest(['--enable-timing', '--verbose']) + unittest(['--enable-timing', '--verbose', '--fail-fast']) with VM('server', 'product'): # hosted mode with Task('UnitTests-BaselineCompiler:hosted-product', tasks): @@ -2113,7 +2113,7 @@ candidates = mx.findclass(args, logToConsole=False, matcher=lambda s, classname: s == classname or classname.endswith('.' + s) or classname.endswith('$' + s)) if len(candidates) > 0: - candidates = mx.select_items(list(mx.OrderedDict.fromkeys(candidates))) + candidates = mx.select_items(sorted(candidates)) else: # mx.findclass can be mistaken, don't give up yet candidates = args @@ -2271,13 +2271,14 @@ assert exists(findbugsJar) nonTestProjects = [p for p in mx.projects() if not p.name.endswith('.test') and not p.name.endswith('.jtt')] outputDirs = map(mx._cygpathU2W, [p.output_dir() for p in nonTestProjects]) + javaCompliance = max([p.javaCompliance for p in nonTestProjects]) findbugsResults = join(_graal_home, 'findbugs.results') cmd = ['-jar', mx._cygpathU2W(findbugsJar), '-textui', '-low', '-maxRank', '15'] if sys.stdout.isatty(): cmd.append('-progress') cmd = cmd + ['-auxclasspath', mx._separatedCygpathU2W(mx.classpath([d.name for d in _jdkDeployedDists] + [p.name for p in nonTestProjects])), '-output', mx._cygpathU2W(findbugsResults), '-exitcode'] + args + outputDirs - exitcode = mx.run_java(cmd, nonZeroIsFatal=False) + exitcode = mx.run_java(cmd, nonZeroIsFatal=False, javaConfig=mx.java(javaCompliance)) if exitcode != 0: with open(findbugsResults) as fp: mx.log(fp.read()) diff -r 12bad81babff -r 5ec45cb4bf22 mxtool/mx.py --- a/mxtool/mx.py Thu Dec 04 23:33:24 2014 +0100 +++ b/mxtool/mx.py Thu Dec 04 23:34:27 2014 +0100 @@ -43,11 +43,33 @@ import shutil, re, xml.dom.minidom import pipes import difflib -from collections import Callable, OrderedDict +from collections import Callable from threading import Thread from argparse import ArgumentParser, REMAINDER from os.path import join, basename, dirname, exists, getmtime, isabs, expandvars, isdir, isfile +# Support for Python 2.6 +def check_output(*popenargs, **kwargs): + process = subprocess.Popen(stdout=subprocess.PIPE, *popenargs, **kwargs) + output, _ = process.communicate() + retcode = process.poll() + if retcode: + cmd = kwargs.get("args") + if cmd is None: + cmd = popenargs[0] + error = subprocess.CalledProcessError(retcode, cmd) + error.output = output + raise error + return output + +try: subprocess.check_output +except: subprocess.check_output = check_output + +try: zipfile.ZipFile.__enter__ +except: + zipfile.ZipFile.__enter__ = lambda self: self + zipfile.ZipFile.__exit__ = lambda self, t, value, traceback: self.close() + _projects = dict() _libs = dict() _jreLibs = dict() @@ -115,103 +137,104 @@ # are sources combined into main archive? unified = self.path == self.sourcesPath - with Archiver(self.path) as arc, Archiver(None if unified else self.sourcesPath) as srcArcRaw: - srcArc = arc if unified else srcArcRaw - services = {} - def overwriteCheck(zf, arcname, source): - if not hasattr(zf, '_provenance'): - zf._provenance = {} - existingSource = zf._provenance.get(arcname, None) - isOverwrite = False - if existingSource and existingSource != source: - if arcname[-1] != os.path.sep: - logv('warning: ' + self.path + ': avoid overwrite of ' + arcname + '\n new: ' + source + '\n old: ' + existingSource) - isOverwrite = True - zf._provenance[arcname] = source - return isOverwrite - - if self.mainClass: - manifest = "Manifest-Version: 1.0\nMain-Class: %s\n\n" % (self.mainClass) - if not overwriteCheck(arc.zf, "META-INF/MANIFEST.MF", "project files"): - arc.zf.writestr("META-INF/MANIFEST.MF", manifest) - - for dep in self.sorted_deps(includeLibs=True): - isCoveredByDependecy = False - for d in self.distDependencies: - if dep in _dists[d].sorted_deps(includeLibs=True, transitive=True): - logv("Excluding {0} from {1} because it's provided by the dependency {2}".format(dep.name, self.path, d)) - isCoveredByDependecy = True - break - - if isCoveredByDependecy: - continue - - if dep.isLibrary(): - l = dep - # merge library jar into distribution jar - logv('[' + self.path + ': adding library ' + l.name + ']') - lpath = l.get_path(resolve=True) - libSourcePath = l.get_source_path(resolve=True) - if lpath: - with zipfile.ZipFile(lpath, 'r') as lp: - for arcname in lp.namelist(): - if arcname.startswith('META-INF/services/') and not arcname == 'META-INF/services/': - service = arcname[len('META-INF/services/'):] - assert '/' not in service - services.setdefault(service, []).extend(lp.read(arcname).splitlines()) - else: - if not overwriteCheck(arc.zf, arcname, lpath + '!' + arcname): - arc.zf.writestr(arcname, lp.read(arcname)) - if srcArc.zf and libSourcePath: - with zipfile.ZipFile(libSourcePath, 'r') as lp: - for arcname in lp.namelist(): - if not overwriteCheck(srcArc.zf, arcname, lpath + '!' + arcname): - srcArc.zf.writestr(arcname, lp.read(arcname)) - elif dep.isProject(): - p = dep - - if self.javaCompliance: - if p.javaCompliance > self.javaCompliance: - abort("Compliance level doesn't match: Distribution {0} requires {1}, but {2} is {3}.".format(self.name, self.javaCompliance, p.name, p.javaCompliance)) - - # skip a Java project if its Java compliance level is "higher" than the configured JDK - jdk = java(p.javaCompliance) - assert jdk - - logv('[' + self.path + ': adding project ' + p.name + ']') - outputDir = p.output_dir() - for root, _, files in os.walk(outputDir): - relpath = root[len(outputDir) + 1:] - if relpath == join('META-INF', 'services'): - for service in files: - with open(join(root, service), 'r') as fp: - services.setdefault(service, []).extend([provider.strip() for provider in fp.readlines()]) - elif relpath == join('META-INF', 'providers'): - for provider in files: - with open(join(root, provider), 'r') as fp: - for service in fp: - services.setdefault(service.strip(), []).append(provider) - else: - for f in files: - arcname = join(relpath, f).replace(os.sep, '/') - if not overwriteCheck(arc.zf, arcname, join(root, f)): - arc.zf.write(join(root, f), arcname) - if srcArc.zf: - sourceDirs = p.source_dirs() - if p.source_gen_dir(): - sourceDirs.append(p.source_gen_dir()) - for srcDir in sourceDirs: - for root, _, files in os.walk(srcDir): - relpath = root[len(srcDir) + 1:] + with Archiver(self.path) as arc: + with Archiver(None if unified else self.sourcesPath) as srcArcRaw: + srcArc = arc if unified else srcArcRaw + services = {} + def overwriteCheck(zf, arcname, source): + if not hasattr(zf, '_provenance'): + zf._provenance = {} + existingSource = zf._provenance.get(arcname, None) + isOverwrite = False + if existingSource and existingSource != source: + if arcname[-1] != os.path.sep: + logv('warning: ' + self.path + ': avoid overwrite of ' + arcname + '\n new: ' + source + '\n old: ' + existingSource) + isOverwrite = True + zf._provenance[arcname] = source + return isOverwrite + + if self.mainClass: + manifest = "Manifest-Version: 1.0\nMain-Class: %s\n\n" % (self.mainClass) + if not overwriteCheck(arc.zf, "META-INF/MANIFEST.MF", "project files"): + arc.zf.writestr("META-INF/MANIFEST.MF", manifest) + + for dep in self.sorted_deps(includeLibs=True): + isCoveredByDependecy = False + for d in self.distDependencies: + if dep in _dists[d].sorted_deps(includeLibs=True, transitive=True): + logv("Excluding {0} from {1} because it's provided by the dependency {2}".format(dep.name, self.path, d)) + isCoveredByDependecy = True + break + + if isCoveredByDependecy: + continue + + if dep.isLibrary(): + l = dep + # merge library jar into distribution jar + logv('[' + self.path + ': adding library ' + l.name + ']') + lpath = l.get_path(resolve=True) + libSourcePath = l.get_source_path(resolve=True) + if lpath: + with zipfile.ZipFile(lpath, 'r') as lp: + for arcname in lp.namelist(): + if arcname.startswith('META-INF/services/') and not arcname == 'META-INF/services/': + service = arcname[len('META-INF/services/'):] + assert '/' not in service + services.setdefault(service, []).extend(lp.read(arcname).splitlines()) + else: + if not overwriteCheck(arc.zf, arcname, lpath + '!' + arcname): + arc.zf.writestr(arcname, lp.read(arcname)) + if srcArc.zf and libSourcePath: + with zipfile.ZipFile(libSourcePath, 'r') as lp: + for arcname in lp.namelist(): + if not overwriteCheck(srcArc.zf, arcname, lpath + '!' + arcname): + srcArc.zf.writestr(arcname, lp.read(arcname)) + elif dep.isProject(): + p = dep + + if self.javaCompliance: + if p.javaCompliance > self.javaCompliance: + abort("Compliance level doesn't match: Distribution {0} requires {1}, but {2} is {3}.".format(self.name, self.javaCompliance, p.name, p.javaCompliance)) + + # skip a Java project if its Java compliance level is "higher" than the configured JDK + jdk = java(p.javaCompliance) + assert jdk + + logv('[' + self.path + ': adding project ' + p.name + ']') + outputDir = p.output_dir() + for root, _, files in os.walk(outputDir): + relpath = root[len(outputDir) + 1:] + if relpath == join('META-INF', 'services'): + for service in files: + with open(join(root, service), 'r') as fp: + services.setdefault(service, []).extend([provider.strip() for provider in fp.readlines()]) + elif relpath == join('META-INF', 'providers'): + for provider in files: + with open(join(root, provider), 'r') as fp: + for service in fp: + services.setdefault(service.strip(), []).append(provider) + else: for f in files: - if f.endswith('.java'): - arcname = join(relpath, f).replace(os.sep, '/') - if not overwriteCheck(srcArc.zf, arcname, join(root, f)): - srcArc.zf.write(join(root, f), arcname) - - for service, providers in services.iteritems(): - arcname = 'META-INF/services/' + service - arc.zf.writestr(arcname, '\n'.join(providers)) + arcname = join(relpath, f).replace(os.sep, '/') + if not overwriteCheck(arc.zf, arcname, join(root, f)): + arc.zf.write(join(root, f), arcname) + if srcArc.zf: + sourceDirs = p.source_dirs() + if p.source_gen_dir(): + sourceDirs.append(p.source_gen_dir()) + for srcDir in sourceDirs: + for root, _, files in os.walk(srcDir): + relpath = root[len(srcDir) + 1:] + for f in files: + if f.endswith('.java'): + arcname = join(relpath, f).replace(os.sep, '/') + if not overwriteCheck(srcArc.zf, arcname, join(root, f)): + srcArc.zf.write(join(root, f), arcname) + + for service, providers in services.iteritems(): + arcname = 'META-INF/services/' + service + arc.zf.writestr(arcname, '\n'.join(providers)) self.notify_updated() @@ -788,69 +811,6 @@ else: return None -# TODO: remove this function once all repos have transitioned -# to the new project format -def _read_projects_file(projectsFile): - suite = OrderedDict() - - suite['projects'] = OrderedDict() - suite['libraries'] = OrderedDict() - suite['jrelibraries'] = OrderedDict() - suite['distributions'] = OrderedDict() - - with open(projectsFile) as f: - prefix = '' - lineNum = 0 - - def error(message): - abort(projectsFile + ':' + str(lineNum) + ': ' + message) - - for line in f: - lineNum = lineNum + 1 - line = line.strip() - if line.endswith('\\'): - prefix = prefix + line[:-1] - continue - if len(prefix) != 0: - line = prefix + line - prefix = '' - if len(line) != 0 and line[0] != '#': - if '=' not in line: - error('non-comment line does not contain an "=" character') - key, value = line.split('=', 1) - - parts = key.split('@') - - if len(parts) == 1: - if parts[0] == 'suite': - suite['name'] = value - elif parts[0] == 'mxversion': - suite['mxversion'] = value - else: - error('Single part property must be "suite": ' + key) - - continue - if len(parts) != 3: - error('Property name does not have 3 parts separated by "@": ' + key) - kind, name, attr = parts - if kind == 'project': - m = suite['projects'] - elif kind == 'library': - m = suite['libraries'] - elif kind == 'jrelibrary': - m = suite['jrelibraries'] - elif kind == 'distribution': - m = suite['distributions'] - else: - error('Property name does not start with "project@", "library@" or "distribution@": ' + key) - - attrs = m.get(name) - if attrs is None: - attrs = OrderedDict() - m[name] = attrs - attrs[attr] = value - return suite - def _load_suite_dict(mxDir): suffix = 1 @@ -883,7 +843,7 @@ # temporarily extend the Python path sys.path.insert(0, mxDir) - snapshot = frozenset(sys.modules.viewkeys()) + snapshot = frozenset(sys.modules.keys()) module = __import__(moduleName) if savedModule: @@ -896,7 +856,7 @@ # For now fail fast if extra modules were loaded. # This can later be relaxed to simply remove the extra modules # from the sys.modules name space if necessary. - extraModules = sys.modules.viewkeys() - snapshot + extraModules = frozenset(sys.modules.keys()) - snapshot assert len(extraModules) == 0, 'loading ' + modulePath + ' caused extra modules to be loaded: ' + ', '.join([m for m in extraModules]) # revert the Python path @@ -906,7 +866,7 @@ abort(modulePath + ' must define a variable named "' + dictName + '"') d = expand(getattr(module, dictName), [dictName]) sections = ['projects', 'libraries', 'jrelibraries', 'distributions'] + (['distribution_extensions'] if suite else ['name', 'mxversion']) - unknown = d.viewkeys() - sections + unknown = frozenset(d.keys()) - frozenset(sections) if unknown: abort(modulePath + ' defines unsupported suite sections: ' + ', '.join(unknown)) @@ -920,7 +880,7 @@ if not existing: suite[s] = additional else: - conflicting = additional.viewkeys() & existing.viewkeys() + conflicting = frozenset(additional.keys()) & frozenset(existing.keys()) if conflicting: abort(modulePath + ' redefines: ' + ', '.join(conflicting)) existing.update(additional) @@ -1102,12 +1062,13 @@ currentAps = zf.read(config).split() if currentAps != aps: logv('[updating ' + config + ' in ' + apsJar + ']') - with Archiver(apsJar) as arc, zipfile.ZipFile(apsJar, 'r') as lp: - for arcname in lp.namelist(): - if arcname == config: - arc.zf.writestr(arcname, '\n'.join(aps)) - else: - arc.zf.writestr(arcname, lp.read(arcname)) + with Archiver(apsJar) as arc: + with zipfile.ZipFile(apsJar, 'r') as lp: + for arcname in lp.namelist(): + if arcname == config: + arc.zf.writestr(arcname, '\n'.join(aps)) + else: + arc.zf.writestr(arcname, lp.read(arcname)) d.add_update_listener(_refineAnnotationProcessorServiceConfig) self.dists.append(d) diff -r 12bad81babff -r 5ec45cb4bf22 src/share/vm/classfile/systemDictionary.cpp --- a/src/share/vm/classfile/systemDictionary.cpp Thu Dec 04 23:33:24 2014 +0100 +++ b/src/share/vm/classfile/systemDictionary.cpp Thu Dec 04 23:34:27 2014 +0100 @@ -96,6 +96,10 @@ oop SystemDictionary::graal_loader() { return _graal_loader; } +void SystemDictionary::init_graal_loader(oop loader) { + assert(UseGraalClassLoader == (loader != NULL), "must be"); + _graal_loader = loader; +} #endif // lazily initialized klass variables @@ -1931,18 +1935,6 @@ } } -#ifdef GRAAL -void SystemDictionary::initialize_preloaded_graal_classes(TRAPS) { - assert(WK_KLASS(CompilerThread_klass) == NULL, "preloaded Graal classes should only be initialized once"); - if (UseGraalClassLoader) { - _graal_loader = GraalRuntime::compute_graal_class_loader(CHECK); - } - - WKID scan = FIRST_GRAAL_WKID; - initialize_wk_klasses_through(LAST_GRAAL_WKID, scan, CHECK); -} -#endif - // Tells if a given klass is a box (wrapper class, such as java.lang.Integer). // If so, returns the basic type it holds. If not, returns T_OBJECT. BasicType SystemDictionary::box_klass_type(Klass* k) { diff -r 12bad81babff -r 5ec45cb4bf22 src/share/vm/classfile/systemDictionary.hpp --- a/src/share/vm/classfile/systemDictionary.hpp Thu Dec 04 23:33:24 2014 +0100 +++ b/src/share/vm/classfile/systemDictionary.hpp Thu Dec 04 23:34:27 2014 +0100 @@ -185,8 +185,7 @@ \ /* Support for Graal */ \ do_klass(BitSet_klass, java_util_BitSet, Opt ) \ - /* Graal classes */ \ - GRAAL_ONLY(do_klass(CompilerThread_klass, com_oracle_graal_compiler_CompilerThread, Graal)) \ + /* Graal classes. These are loaded on-demand. */ \ GRAAL_ONLY(do_klass(Node_klass, com_oracle_graal_graph_Node, Graal)) \ GRAAL_ONLY(do_klass(NodeClass_klass, com_oracle_graal_graph_NodeClass, Graal)) \ GRAAL_ONLY(do_klass(HotSpotCompiledCode_klass, com_oracle_graal_hotspot_HotSpotCompiledCode, Graal)) \ @@ -262,7 +261,7 @@ WKID_LIMIT, #ifdef GRAAL - FIRST_GRAAL_WKID = WK_KLASS_ENUM_NAME(CompilerThread_klass), + FIRST_GRAAL_WKID = WK_KLASS_ENUM_NAME(Node_klass), LAST_GRAAL_WKID = WK_KLASS_ENUM_NAME(AbstractValue_klass), #endif @@ -538,6 +537,8 @@ // Returns the Graal loader. This will be NULL if !UseGraalClassLoader // in which case it's equivalent to the boot loader static oop graal_loader(); + // Sets the Graal loader. This is called at most once. + static void init_graal_loader(oop loader); #endif // Compute the default system loader @@ -705,10 +706,6 @@ public: static bool is_ext_class_loader(Handle class_loader); -#ifdef GRAAL - static void initialize_preloaded_graal_classes(TRAPS); -#endif - private: static Klass* find_shared_class(Symbol* class_name); diff -r 12bad81babff -r 5ec45cb4bf22 src/share/vm/classfile/vmSymbols.hpp --- a/src/share/vm/classfile/vmSymbols.hpp Thu Dec 04 23:33:24 2014 +0100 +++ b/src/share/vm/classfile/vmSymbols.hpp Thu Dec 04 23:34:27 2014 +0100 @@ -291,7 +291,6 @@ \ /* Support for Graal */ \ template(java_util_BitSet, "java/util/BitSet") \ - GRAAL_ONLY(template(com_oracle_graal_compiler_CompilerThread, "com/oracle/graal/compiler/CompilerThread")) \ GRAAL_ONLY(template(com_oracle_graal_graph_Node, "com/oracle/graal/graph/Node")) \ GRAAL_ONLY(template(com_oracle_graal_graph_NodeClass, "com/oracle/graal/graph/NodeClass")) \ GRAAL_ONLY(template(com_oracle_graal_hotspot_HotSpotGraalRuntime, "com/oracle/graal/hotspot/HotSpotGraalRuntime")) \ diff -r 12bad81babff -r 5ec45cb4bf22 src/share/vm/graal/graalCompiler.cpp --- a/src/share/vm/graal/graalCompiler.cpp Thu Dec 04 23:33:24 2014 +0100 +++ b/src/share/vm/graal/graalCompiler.cpp Thu Dec 04 23:34:27 2014 +0100 @@ -115,6 +115,7 @@ return; } + GraalRuntime::ensure_graal_class_loader_is_initialized(); HandleMark hm; ResourceMark rm; JavaValue result(T_VOID); diff -r 12bad81babff -r 5ec45cb4bf22 src/share/vm/graal/graalRuntime.cpp --- a/src/share/vm/graal/graalRuntime.cpp Thu Dec 04 23:33:24 2014 +0100 +++ b/src/share/vm/graal/graalRuntime.cpp Thu Dec 04 23:34:27 2014 +0100 @@ -49,6 +49,8 @@ AMD64_ONLY(guarantee(heap_end < allocation_end, "heap end too close to end of address space (might lead to erroneous TLAB allocations)")); NOT_LP64(error("check TLAB allocation code for address space conflicts")); + ensure_graal_class_loader_is_initialized(); + JavaThread* THREAD = JavaThread::current(); { ThreadToNativeFromVM trans(THREAD); @@ -656,6 +658,13 @@ return value; JRT_END +// private static void Factory.init() +JVM_ENTRY(void, JVM_InitGraalClassLoader(JNIEnv *env, jclass c, jobject loader_handle)) + SystemDictionary::init_graal_loader(JNIHandles::resolve(loader_handle)); + SystemDictionary::WKID scan = SystemDictionary::FIRST_GRAAL_WKID; + SystemDictionary::initialize_wk_klasses_through(SystemDictionary::LAST_GRAAL_WKID, scan, CHECK); +JVM_END + // private static GraalRuntime Graal.initializeRuntime() JVM_ENTRY(jobject, JVM_GetGraalRuntime(JNIEnv *env, jclass c)) return GraalRuntime::get_HotSpotGraalRuntime_jobject(); @@ -748,6 +757,46 @@ return result; JVM_END + +void GraalRuntime::ensure_graal_class_loader_is_initialized() { + // This initialization code is guarded by a static pointer to the Factory class. + // Once it is non-null, the Graal class loader and well know Graal classes are + // guaranteed to have been initialized. By going through the static + // initializer of Factory, we can rely on class initialization semantics to + // synchronize threads racing to do the initialization. + static Klass* _FactoryKlass = NULL; + if (_FactoryKlass == NULL) { + Thread* THREAD = Thread::current(); + TempNewSymbol name = SymbolTable::new_symbol("com/oracle/graal/hotspot/loader/Factory", CHECK_ABORT); + KlassHandle klass = SystemDictionary::resolve_or_fail(name, true, THREAD); + if (HAS_PENDING_EXCEPTION) { + static volatile int seen_error = 0; + if (!seen_error && Atomic::cmpxchg(1, &seen_error, 0) == 0) { + // Only report the failure on the first thread that hits it + abort_on_pending_exception(PENDING_EXCEPTION, "Graal classes are not available"); + } else { + CLEAR_PENDING_EXCEPTION; + // Give first thread time to report the error. + os::sleep(THREAD, 100, false); + vm_abort(false); + } + } + + TempNewSymbol field_name = SymbolTable::new_symbol("useGraalClassLoader", CHECK_ABORT); + fieldDescriptor field_desc; + if (klass->find_field(field_name, vmSymbols::bool_signature(), &field_desc) == NULL) { + ResourceMark rm; + fatal(err_msg("Invalid layout of %s at %s", field_name->as_C_string(), klass->external_name())); + } + + InstanceKlass* ik = InstanceKlass::cast(klass()); + address addr = ik->static_field_addr(field_desc.offset() - InstanceMirrorKlass::offset_of_static_fields()); + *((jboolean *) addr) = (jboolean) UseGraalClassLoader; + klass->initialize(CHECK_ABORT); + _FactoryKlass = klass(); + } +} + jint GraalRuntime::check_arguments(TRAPS) { KlassHandle nullHandle; parse_arguments(nullHandle, THREAD); @@ -800,6 +849,7 @@ } void GraalRuntime::parse_argument(KlassHandle hotSpotOptionsClass, char* arg, TRAPS) { + ensure_graal_class_loader_is_initialized(); char first = arg[0]; char* name; size_t name_len; @@ -1017,17 +1067,6 @@ thread); } -oop GraalRuntime::compute_graal_class_loader(TRAPS) { - assert(UseGraalClassLoader, "must be"); - TempNewSymbol name = SymbolTable::new_symbol("com/oracle/graal/hotspot/loader/Factory", CHECK_NULL); - KlassHandle klass = SystemDictionary::resolve_or_fail(name, true, CHECK_NULL); - - TempNewSymbol getClassLoader = SymbolTable::new_symbol("newClassLoader", CHECK_NULL); - JavaValue result(T_OBJECT); - JavaCalls::call_static(&result, klass, getClassLoader, vmSymbols::void_classloader_signature(), CHECK_NULL); - return (oop) result.get_jobject(); -} - void GraalRuntime::abort_on_pending_exception(Handle exception, const char* message, bool dump_core) { Thread* THREAD = Thread::current(); CLEAR_PENDING_EXCEPTION; diff -r 12bad81babff -r 5ec45cb4bf22 src/share/vm/graal/graalRuntime.hpp --- a/src/share/vm/graal/graalRuntime.hpp Thu Dec 04 23:33:24 2014 +0100 +++ b/src/share/vm/graal/graalRuntime.hpp Thu Dec 04 23:34:27 2014 +0100 @@ -127,6 +127,11 @@ public: + /** + * Ensures that the Graal class loader is initialized and the well known Graal classes are loaded. + */ + static void ensure_graal_class_loader_is_initialized(); + static void initialize_natives(JNIEnv *env, jclass c2vmClass); static bool is_HotSpotGraalRuntime_initialized() { return _HotSpotGraalRuntime_initialized; } @@ -200,11 +205,6 @@ */ static Klass* load_required_class(Symbol* name); - /** - * Creates a separate class loader for classes in graal.jar and graal-truffle.jar. - */ - static oop compute_graal_class_loader(TRAPS); - static BufferBlob* initialize_buffer_blob(); /** diff -r 12bad81babff -r 5ec45cb4bf22 src/share/vm/prims/nativeLookup.cpp --- a/src/share/vm/prims/nativeLookup.cpp Thu Dec 04 23:33:24 2014 +0100 +++ b/src/share/vm/prims/nativeLookup.cpp Thu Dec 04 23:34:27 2014 +0100 @@ -127,6 +127,7 @@ void JNICALL JVM_RegisterPerfMethods(JNIEnv *env, jclass perfclass); void JNICALL JVM_RegisterWhiteBoxMethods(JNIEnv *env, jclass wbclass); #ifdef GRAAL + void JNICALL JVM_InitGraalClassLoader(JNIEnv *env, jclass c, jobject loader); void JNICALL JVM_InitializeGraalNatives(JNIEnv *env, jclass compilerToVMClass); jobject JNICALL JVM_GetGraalRuntime(JNIEnv *env, jclass c); jobject JNICALL JVM_GetGraalServiceImpls(JNIEnv *env, jclass c); @@ -148,6 +149,7 @@ { CC"Java_sun_misc_Perf_registerNatives", NULL, FN_PTR(JVM_RegisterPerfMethods) }, { CC"Java_sun_hotspot_WhiteBox_registerNatives", NULL, FN_PTR(JVM_RegisterWhiteBoxMethods) }, #ifdef GRAAL + { CC"Java_com_oracle_graal_hotspot_loader_Factory_init", NULL, FN_PTR(JVM_InitGraalClassLoader) }, { CC"Java_com_oracle_graal_api_runtime_Graal_initializeRuntime", NULL, FN_PTR(JVM_GetGraalRuntime) }, { CC"Java_com_oracle_graal_api_runtime_Services_getServiceImpls", NULL, FN_PTR(JVM_GetGraalServiceImpls) }, { CC"Java_com_oracle_truffle_api_Truffle_createRuntime", NULL, FN_PTR(JVM_CreateTruffleRuntime) }, diff -r 12bad81babff -r 5ec45cb4bf22 src/share/vm/runtime/thread.cpp --- a/src/share/vm/runtime/thread.cpp Thu Dec 04 23:33:24 2014 +0100 +++ b/src/share/vm/runtime/thread.cpp Thu Dec 04 23:34:27 2014 +0100 @@ -1433,7 +1433,7 @@ bool graal_counters_include(JavaThread* thread) { oop threadObj = thread->threadObj(); - return !GraalCountersExcludeCompiler || (!thread->is_Compiler_thread() && (threadObj == NULL || threadObj->klass() != SystemDictionary::CompilerThread_klass())); + return !GraalCountersExcludeCompiler || !thread->is_Compiler_thread(); } void JavaThread::collect_counters(typeArrayOop array) { @@ -3686,11 +3686,6 @@ // set_init_completed has just been called, causing exceptions not to be shortcut // anymore. We call vm_exit_during_initialization directly instead. SystemDictionary::compute_java_system_loader(THREAD); -#ifdef GRAAL - if (!HAS_PENDING_EXCEPTION) { - SystemDictionary::initialize_preloaded_graal_classes(THREAD); - } -#endif if (HAS_PENDING_EXCEPTION) { vm_exit_during_initialization(Handle(THREAD, PENDING_EXCEPTION)); }