# HG changeset patch
# User Andreas Woess <andreas.woess@jku.at>
# Date 1380196099 -7200
# Node ID ec90fc830e451bba2f3601754d098101cd08ceb4
# Parent  73a886a9564a72dd8f7058d8e3f97c79accab2a9# Parent  91a676d0bbbe5e9ce72df7cd2234d5f3b121d28a
Merge

diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.asm.sparc/src/com/oracle/graal/asm/sparc/SPARCAddress.java
--- a/graal/com.oracle.graal.asm.sparc/src/com/oracle/graal/asm/sparc/SPARCAddress.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.asm.sparc/src/com/oracle/graal/asm/sparc/SPARCAddress.java	Thu Sep 26 13:48:19 2013 +0200
@@ -121,7 +121,7 @@
             throw GraalInternalError.shouldNotReachHere("address has index register");
         }
         // TODO Should we also hide the register save area size here?
-        if (getBase() == sp || getBase() == fp) {
+        if (getBase().equals(sp) || getBase().equals(fp)) {
             return displacement + STACK_BIAS;
         }
         return displacement;
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.compiler.hsail/src/com/oracle/graal/compiler/hsail/HSAILLIRGenerator.java
--- a/graal/com.oracle.graal.compiler.hsail/src/com/oracle/graal/compiler/hsail/HSAILLIRGenerator.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.compiler.hsail/src/com/oracle/graal/compiler/hsail/HSAILLIRGenerator.java	Thu Sep 26 13:48:19 2013 +0200
@@ -143,12 +143,12 @@
             } else {
                 baseRegister = load(base);
             }
-        } else if (base == Value.ILLEGAL) {
+        } else if (base.equals(Value.ILLEGAL)) {
             baseRegister = Value.ILLEGAL;
         } else {
             baseRegister = asAllocatable(base);
         }
-        if (index != Value.ILLEGAL) {
+        if (!index.equals(Value.ILLEGAL)) {
             if (isConstant(index)) {
                 finalDisp += asConstant(index).asLong() * scale;
             } else {
@@ -160,7 +160,7 @@
                 } else {
                     indexRegister = convertedIndex;
                 }
-                if (baseRegister == Value.ILLEGAL) {
+                if (baseRegister.equals(Value.ILLEGAL)) {
                     baseRegister = asAllocatable(indexRegister);
                 } else {
                     baseRegister = emitAdd(baseRegister, indexRegister);
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.compiler.test/src/com/oracle/graal/compiler/test/MemoryScheduleTest.java
--- a/graal/com.oracle.graal.compiler.test/src/com/oracle/graal/compiler/test/MemoryScheduleTest.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.compiler.test/src/com/oracle/graal/compiler/test/MemoryScheduleTest.java	Thu Sep 26 13:48:19 2013 +0200
@@ -447,7 +447,7 @@
     @Test
     public void testLoop4() {
         SchedulePhase schedule = getFinalSchedule("testLoop4Snippet", TestMode.WITHOUT_FRAMESTATES, MemoryScheduling.OPTIMAL);
-        assertReadWithinStartBlock(schedule, false);
+        assertReadWithinStartBlock(schedule, true);
         assertReadWithinReturnBlock(schedule, false);
     }
 
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/HotSpotRuntime.java
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/HotSpotRuntime.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/HotSpotRuntime.java	Thu Sep 26 13:48:19 2013 +0200
@@ -618,7 +618,7 @@
                         value = checkcastNode;
                     }
                 } else {
-                    LoadHubNode arrayClass = graph.unique(new LoadHubNode(array, wordKind, boundsCheck.asNode()));
+                    FloatingReadNode arrayClass = createReadHub(graph, wordKind, array, boundsCheck);
                     LocationNode location = ConstantLocationNode.create(FINAL_LOCATION, wordKind, config.arrayClassElementOffset, graph);
                     /*
                      * Anchor the read of the element klass to the cfg, because it is only valid
@@ -750,6 +750,7 @@
                     for (int lockDepth : commit.getLocks().get(objIndex)) {
                         MonitorEnterNode enter = graph.add(new MonitorEnterNode(allocations[objIndex], lockDepth));
                         graph.addBeforeFixed(commit, enter);
+                        enter.lower(tool);
                     }
                 }
                 for (Node usage : commit.usages().snapshot()) {
@@ -990,13 +991,14 @@
         }
     }
 
-    private static GuardingNode createBoundsCheck(AccessIndexedNode n, LoweringTool tool) {
-        StructuredGraph graph = n.graph();
-        ArrayLengthNode arrayLength = graph.add(new ArrayLengthNode(n.array()));
-        GuardingNode guard = tool.createGuard(graph.unique(new IntegerBelowThanNode(n.index(), arrayLength)), BoundsCheckException, InvalidateReprofile);
-
-        graph.addBeforeFixed(n, arrayLength);
-        return guard;
+    private GuardingNode createBoundsCheck(AccessIndexedNode n, LoweringTool tool) {
+        StructuredGraph g = n.graph();
+        ValueNode array = n.array();
+        Stamp stamp = StampFactory.positiveInt();
+        ReadNode arrayLength = g.add(new ReadNode(array, ConstantLocationNode.create(FINAL_LOCATION, Kind.Int, config.arrayLengthOffset, g), stamp, BarrierType.NONE, false));
+        g.addBeforeFixed(n, arrayLength);
+        tool.createNullCheckGuard(arrayLength, array);
+        return tool.createGuard(g.unique(new IntegerBelowThanNode(n.index(), arrayLength)), BoundsCheckException, InvalidateReprofile);
     }
 
     public ResolvedJavaType lookupJavaType(Class<?> clazz) {
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/nodes/WriteBarrier.java
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/nodes/WriteBarrier.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/nodes/WriteBarrier.java	Thu Sep 26 13:48:19 2013 +0200
@@ -60,8 +60,8 @@
     }
 
     @Override
-    public void lower(LoweringTool generator) {
+    public void lower(LoweringTool tool) {
         assert graph().getGuardsStage() == StructuredGraph.GuardsStage.AFTER_FSA;
-        generator.getRuntime().lower(this, generator);
+        tool.getRuntime().lower(this, tool);
     }
 }
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/CallSiteTargetNode.java
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/CallSiteTargetNode.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/CallSiteTargetNode.java	Thu Sep 26 13:48:19 2013 +0200
@@ -72,7 +72,9 @@
         if (target != null) {
             graph().replaceFixedWithFloating(this, target);
         } else {
-            graph().replaceFixedWithFixed(this, createInvoke());
+            InvokeNode invoke = createInvoke();
+            graph().replaceFixedWithFixed(this, invoke);
+            invoke.lower(tool);
         }
     }
 }
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ReflectionGetCallerClassNode.java
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ReflectionGetCallerClassNode.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ReflectionGetCallerClassNode.java	Thu Sep 26 13:48:19 2013 +0200
@@ -53,7 +53,9 @@
         if (callerClassNode != null) {
             graph().replaceFixedWithFloating(this, callerClassNode);
         } else {
-            graph().replaceFixedWithFixed(this, createInvoke());
+            InvokeNode invoke = createInvoke();
+            graph().replaceFixedWithFixed(this, invoke);
+            invoke.lower(tool);
         }
     }
 
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.lir.hsail/src/com/oracle/graal/lir/hsail/HSAILAddressValue.java
--- a/graal/com.oracle.graal.lir.hsail/src/com/oracle/graal/lir/hsail/HSAILAddressValue.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.lir.hsail/src/com/oracle/graal/lir/hsail/HSAILAddressValue.java	Thu Sep 26 13:48:19 2013 +0200
@@ -66,7 +66,7 @@
     }
 
     public HSAILAddress toAddress() {
-        Register baseReg = base == Value.ILLEGAL ? Register.None : asRegister(base);
+        Register baseReg = base.equals(Value.ILLEGAL) ? Register.None : asRegister(base);
         return new HSAILAddress(baseReg, displacement);
     }
 
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/AbstractBeginNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/AbstractBeginNode.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/AbstractBeginNode.java	Thu Sep 26 13:48:19 2013 +0200
@@ -32,7 +32,7 @@
 import com.oracle.graal.nodes.spi.*;
 import com.oracle.graal.nodes.type.*;
 
-public abstract class AbstractBeginNode extends FixedWithNextNode implements StateSplit, LIRLowerable, Simplifiable, GuardingNode {
+public abstract class AbstractBeginNode extends FixedWithNextNode implements StateSplit, LIRLowerable, Simplifiable, GuardingNode, IterableNodeType {
 
     @Input(notDataflow = true) private FrameState stateAfter;
 
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/GuardingPiNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/GuardingPiNode.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/GuardingPiNode.java	Thu Sep 26 13:48:19 2013 +0200
@@ -95,13 +95,13 @@
     @Override
     public ValueNode canonical(CanonicalizerTool tool) {
         if (stamp() == StampFactory.illegal(object.kind())) {
-            // The condition always fails
+            // The guard always fails
             return graph().add(new DeoptimizeNode(action, reason));
         }
         if (condition instanceof LogicConstantNode) {
             LogicConstantNode c = (LogicConstantNode) condition;
             if (c.getValue() == negated) {
-                // The condition always fails
+                // The guard always fails
                 return graph().add(new DeoptimizeNode(action, reason));
             }
 
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/PiArrayNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/PiArrayNode.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/PiArrayNode.java	Thu Sep 26 13:48:19 2013 +0200
@@ -22,11 +22,14 @@
  */
 package com.oracle.graal.nodes;
 
+import com.oracle.graal.nodes.java.*;
 import com.oracle.graal.nodes.spi.*;
 import com.oracle.graal.nodes.type.*;
 
 /**
- * The {@code UnsafeCastNode} produces the same value as its input, but with a different type.
+ * A {@link PiNode} that also provides an array length in addition to a more refined stamp. A usage
+ * that reads the array length, such as an {@link ArrayLengthNode}, can be canonicalized base on
+ * this information.
  */
 public final class PiArrayNode extends PiNode implements ArrayLengthProvider {
 
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/PiNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/PiNode.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/PiNode.java	Thu Sep 26 13:48:19 2013 +0200
@@ -60,10 +60,6 @@
 
     @Override
     public void generate(LIRGeneratorTool generator) {
-        assert kind() == Kind.Object && object.kind() == Kind.Object;
-        assert ObjectStamp.typeOrNull(this) == null || ObjectStamp.typeOrNull(object) == null || ObjectStamp.typeOrNull(this).isInterface() || ObjectStamp.typeOrNull(object).isInterface() ||
-                        ObjectStamp.typeOrNull(object).isAssignableFrom(ObjectStamp.typeOrNull(this));
-
         if (object.kind() != Kind.Void && object.kind() != Kind.Illegal) {
             generator.setResult(this, generator.operand(object));
         }
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/cfg/ControlFlowGraph.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/cfg/ControlFlowGraph.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/cfg/ControlFlowGraph.java	Thu Sep 26 13:48:19 2013 +0200
@@ -154,12 +154,10 @@
     private void identifyBlocks() {
         // Find all block headers
         int numBlocks = 0;
-        for (Node node : graph.getNodes()) {
-            if (node instanceof AbstractBeginNode) {
-                Block block = new Block((AbstractBeginNode) node);
-                numBlocks++;
-                identifyBlock(block);
-            }
+        for (AbstractBeginNode begin : graph.getNodes(AbstractBeginNode.class)) {
+            Block block = new Block(begin);
+            numBlocks++;
+            identifyBlock(block);
         }
 
         // Compute postorder.
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/debug/DynamicCounterNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/debug/DynamicCounterNode.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/debug/DynamicCounterNode.java	Thu Sep 26 13:48:19 2013 +0200
@@ -183,6 +183,8 @@
 
             graph().addBeforeFixed(this, load);
             graph().addBeforeFixed(this, store);
+            load.lower(tool);
+            store.lower(tool);
         }
         graph().removeFixed(this);
     }
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/java/CheckCastNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/java/CheckCastNode.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/java/CheckCastNode.java	Thu Sep 26 13:48:19 2013 +0200
@@ -115,6 +115,7 @@
                 graph().addBeforeFixed(this, nullGuard);
                 condition = typeTest;
                 stamp = stamp.join(StampFactory.objectNonNull());
+                nullGuard.lower(tool);
             } else {
                 // TODO (ds) replace with probability of null-seen when available
                 double shortCircuitProbability = NOT_FREQUENT_PROBABILITY;
@@ -123,6 +124,7 @@
         }
         GuardingPiNode checkedObject = graph().add(new GuardingPiNode(object, condition, false, forStoreCheck ? ArrayStoreException : ClassCastException, InvalidateReprofile, stamp));
         graph().replaceFixedWithFixed(this, checkedObject);
+        checkedObject.lower(tool);
     }
 
     @Override
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/java/ExceptionObjectNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/java/ExceptionObjectNode.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/java/ExceptionObjectNode.java	Thu Sep 26 13:48:19 2013 +0200
@@ -89,6 +89,7 @@
         graph().addAfterFixed(this, loadException);
         setStateAfter(null);
         setStamp(StampFactory.forVoid());
+        loadException.lower(tool);
     }
 
     @Override
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/spi/Lowerable.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/spi/Lowerable.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/spi/Lowerable.java	Thu Sep 26 13:48:19 2013 +0200
@@ -24,8 +24,16 @@
 
 import com.oracle.graal.nodes.*;
 
+/**
+ * Interface implemented by nodes that can replace themselves with lower level nodes during a phase
+ * that transforms a graph to replace higher level nodes with lower level nodes.
+ */
 public interface Lowerable {
 
+    /**
+     * Expand this node into lower level nodes expressing the same semantics. If the introduced
+     * nodes are themselves lowerable, they should be recursively lowered as part of this call.
+     */
     void lower(LoweringTool tool);
 
     ValueNode asNode();
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.phases.common/src/com/oracle/graal/phases/common/CanonicalizerPhase.java
--- a/graal/com.oracle.graal.phases.common/src/com/oracle/graal/phases/common/CanonicalizerPhase.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.phases.common/src/com/oracle/graal/phases/common/CanonicalizerPhase.java	Thu Sep 26 13:48:19 2013 +0200
@@ -267,7 +267,7 @@
 // @formatter:off
 //     cases:                                           original node:
 //                                         |Floating|Fixed-unconnected|Fixed-connected|
-//                                         -------------------------------------------|
+//                                         --------------------------------------------
 //                                     null|   1    |        X        |       3       |
 //                                         --------------------------------------------
 //                                 Floating|   2    |        X        |       4       |
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.phases.common/src/com/oracle/graal/phases/common/LoweringPhase.java
--- a/graal/com.oracle.graal.phases.common/src/com/oracle/graal/phases/common/LoweringPhase.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.phases.common/src/com/oracle/graal/phases/common/LoweringPhase.java	Thu Sep 26 13:48:19 2013 +0200
@@ -142,31 +142,55 @@
         this.canonicalizer = canonicalizer;
     }
 
-    private static boolean containsLowerable(NodeIterable<Node> nodes) {
-        for (Node n : nodes) {
-            if (n instanceof Lowerable) {
-                return true;
-            }
-        }
-        return false;
+    /**
+     * Checks that second lowering of a given graph did not introduce any new nodes.
+     * 
+     * @param graph a graph that was just {@linkplain #lower lowered}
+     * @throws AssertionError if the check fails
+     */
+    private boolean checkPostLowering(StructuredGraph graph, PhaseContext context) {
+        int expectedMark = graph.getMark();
+        lower(graph, context, 1);
+        int mark = graph.getMark();
+        assert mark == expectedMark : graph + ": a second round in the current lowering phase introduced these new nodes: " + graph.getNewNodes(mark).snapshot();
+        return true;
     }
 
     @Override
     protected void run(final StructuredGraph graph, PhaseContext context) {
-        int i = 0;
-        while (true) {
-            int mark = graph.getMark();
+        lower(graph, context, 0);
+        assert checkPostLowering(graph, context);
+    }
+
+    private void lower(StructuredGraph graph, PhaseContext context, int i) {
+        IncrementalCanonicalizerPhase<PhaseContext> incrementalCanonicalizer = new IncrementalCanonicalizerPhase<>(canonicalizer);
+        incrementalCanonicalizer.appendPhase(new Round(i, context));
+        incrementalCanonicalizer.apply(graph, context);
+        assert graph.verify();
+    }
 
-            IncrementalCanonicalizerPhase<PhaseContext> incrementalCanonicalizer = new IncrementalCanonicalizerPhase<>(canonicalizer);
-            incrementalCanonicalizer.appendPhase(new Round(i++, context));
-            incrementalCanonicalizer.apply(graph, context);
-
-            if (!containsLowerable(graph.getNewNodes(mark))) {
-                // No new lowerable nodes - done!
-                break;
+    /**
+     * Checks that lowering of a given node did not introduce any new {@link Lowerable} nodes that
+     * could be lowered in the current {@link LoweringPhase}. Such nodes must be recursively lowered
+     * as part of lowering {@code node}.
+     * 
+     * @param node a node that was just lowered
+     * @param preLoweringMark the graph mark before {@code node} was lowered
+     * @throws AssertionError if the check fails
+     */
+    private static boolean checkPostNodeLowering(Node node, LoweringToolImpl loweringTool, int preLoweringMark) {
+        StructuredGraph graph = (StructuredGraph) node.graph();
+        int postLoweringMark = graph.getMark();
+        NodeIterable<Node> newNodesAfterLowering = graph.getNewNodes(preLoweringMark);
+        for (Node n : newNodesAfterLowering) {
+            if (n instanceof Lowerable) {
+                ((Lowerable) n).lower(loweringTool);
+                int mark = graph.getMark();
+                assert postLoweringMark == mark : graph + ": lowering of " + node + " produced lowerable " + n + " that should have been recursively lowered as it introduces these new nodes: " +
+                                graph.getNewNodes(postLoweringMark).snapshot();
             }
-            assert graph.verify();
         }
+        return true;
     }
 
     private final class Round extends Phase {
@@ -241,17 +265,23 @@
 
                 if (node instanceof Lowerable) {
                     assert checkUsagesAreScheduled(node);
+                    int preLoweringMark = node.graph().getMark();
                     ((Lowerable) node).lower(loweringTool);
+                    assert checkPostNodeLowering(node, loweringTool, preLoweringMark);
                 }
 
                 if (!nextNode.isAlive()) {
-                    // can happen when the rest of the block is killed by lowering (e.g. by a
-                    // unconditional deopt)
+                    // can happen when the rest of the block is killed by lowering
+                    // (e.g. by an unconditional deopt)
                     break;
                 } else {
                     Node nextLastFixed = nextNode.predecessor();
                     if (!(nextLastFixed instanceof FixedWithNextNode)) {
                         // insert begin node, to have a valid last fixed for next lowerable node.
+                        // This is about lowering a FixedWithNextNode to a control split while this
+                        // FixedWithNextNode is followed by some kind of BeginNode.
+                        // For example the when a FixedGuard followed by a loop exit is lowered to a
+                        // control-split + deopt.
                         BeginNode begin = node.graph().add(new BeginNode());
                         nextLastFixed.replaceFirstSuccessor(nextNode, begin);
                         begin.setNext(nextNode);
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.phases/src/com/oracle/graal/phases/verify/VerifyUsageWithEquals.java
--- a/graal/com.oracle.graal.phases/src/com/oracle/graal/phases/verify/VerifyUsageWithEquals.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.phases/src/com/oracle/graal/phases/verify/VerifyUsageWithEquals.java	Thu Sep 26 13:48:19 2013 +0200
@@ -73,7 +73,7 @@
             if (!isEqualsMethod(graph)) {
                 // bail out if we compare an object of type klass with == or != (except null checks)
                 assert !(checkUsage(cn.x(), cn.y(), context.getRuntime()) && checkUsage(cn.y(), cn.x(), context.getRuntime())) : "Verifcation of " + klass.getName() + " usage failed: Comparing " +
-                                cn.x() + " and" + cn.y() + " in " + graph.method() + " must use .equals() for object equality, not '==' or '!='";
+                                cn.x() + " and " + cn.y() + " in " + graph.method() + " must use .equals() for object equality, not '==' or '!='";
             }
         }
         return true;
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.replacements.amd64/src/com/oracle/graal/replacements/amd64/AMD64ConvertNode.java
--- a/graal/com.oracle.graal.replacements.amd64/src/com/oracle/graal/replacements/amd64/AMD64ConvertNode.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.replacements.amd64/src/com/oracle/graal/replacements/amd64/AMD64ConvertNode.java	Thu Sep 26 13:48:19 2013 +0200
@@ -30,9 +30,8 @@
 
 /**
  * This node has the semantics of the AMD64 conversions. It is used in the lowering of the
- * ConvertNode which, on AMD64 needs a AMD64ConvertNode plus some fixup code that handles the corner
- * cases that differ between AMD64 and Java.
- * 
+ * {@link ConvertNode} which, on AMD64 needs a {@link AMD64ConvertNode} plus some fixup code that
+ * handles the corner cases that differ between AMD64 and Java.
  */
 public class AMD64ConvertNode extends FloatingNode implements ArithmeticLIRLowerable {
 
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.replacements.amd64/src/com/oracle/graal/replacements/amd64/AMD64ConvertSnippets.java
--- a/graal/com.oracle.graal.replacements.amd64/src/com/oracle/graal/replacements/amd64/AMD64ConvertSnippets.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.replacements.amd64/src/com/oracle/graal/replacements/amd64/AMD64ConvertSnippets.java	Thu Sep 26 13:48:19 2013 +0200
@@ -181,6 +181,7 @@
             SnippetTemplate template = template(args);
             Debug.log("Lowering %s in %s: node=%s, template=%s, arguments=%s", convert.opcode, graph, convert, template, args);
             template.instantiate(runtime, replacee, DEFAULT_REPLACER, tool, args);
+            graph.removeFloating(convert);
         }
     }
 }
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.replacements.test/src/com/oracle/graal/replacements/test/PointerTest.java
--- a/graal/com.oracle.graal.replacements.test/src/com/oracle/graal/replacements/test/PointerTest.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.replacements.test/src/com/oracle/graal/replacements/test/PointerTest.java	Thu Sep 26 13:48:19 2013 +0200
@@ -33,8 +33,11 @@
 import com.oracle.graal.nodes.*;
 import com.oracle.graal.nodes.calc.*;
 import com.oracle.graal.nodes.extended.*;
+import com.oracle.graal.phases.*;
+import com.oracle.graal.phases.common.*;
+import com.oracle.graal.phases.tiers.*;
 import com.oracle.graal.replacements.*;
-import com.oracle.graal.replacements.Snippet.*;
+import com.oracle.graal.replacements.Snippet.SnippetInliningPolicy;
 import com.oracle.graal.word.*;
 import com.oracle.graal.word.nodes.*;
 
@@ -399,4 +402,62 @@
         Word.fromObject(o).writeObject(offset, value);
     }
 
+    private void assertNumWordCasts(String snippetName, int expectedWordCasts) {
+        Assumptions assumptions = new Assumptions(true);
+        HighTierContext context = new HighTierContext(runtime(), assumptions, replacements, null, null, OptimisticOptimizations.ALL);
+
+        StructuredGraph graph = parse(snippetName);
+        new CanonicalizerPhase(false).apply(graph, context);
+        Assert.assertEquals(expectedWordCasts, graph.getNodes().filter(WordCastNode.class).count());
+    }
+
+    @Test
+    public void testUnusedFromObject() {
+        assertNumWordCasts("unusedFromObject", 0);
+    }
+
+    @Snippet
+    public static void unusedFromObject(Object o) {
+        Word.fromObject(o);
+    }
+
+    @Test
+    public void testUnusedRawValue() {
+        assertNumWordCasts("unusedRawValue", 0);
+    }
+
+    @Snippet
+    public static void unusedRawValue(Object o) {
+        Word.fromObject(o).rawValue();
+    }
+
+    @Test
+    public void testUsedRawValue() {
+        assertNumWordCasts("usedRawValue", 1);
+    }
+
+    @Snippet
+    public static long usedRawValue(Object o) {
+        return Word.fromObject(o).rawValue();
+    }
+
+    @Test
+    public void testUnusedToObject() {
+        assertNumWordCasts("unusedToObject", 0);
+    }
+
+    @Snippet
+    public static void unusedToObject(Word w) {
+        w.toObject();
+    }
+
+    @Test
+    public void testUsedToObject() {
+        assertNumWordCasts("usedToObject", 1);
+    }
+
+    @Snippet
+    public static Object usedToObject(Word w) {
+        return w.toObject();
+    }
 }
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.replacements/src/com/oracle/graal/replacements/SnippetTemplate.java
--- a/graal/com.oracle.graal.replacements/src/com/oracle/graal/replacements/SnippetTemplate.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.replacements/src/com/oracle/graal/replacements/SnippetTemplate.java	Thu Sep 26 13:48:19 2013 +0200
@@ -391,7 +391,7 @@
     /**
      * Creates a snippet template.
      */
-    protected SnippetTemplate(MetaAccessProvider runtime, Replacements replacements, Arguments args) {
+    protected SnippetTemplate(final MetaAccessProvider runtime, final Replacements replacements, Arguments args) {
         StructuredGraph snippetGraph = replacements.getSnippet(args.info.method);
 
         ResolvedJavaMethod method = snippetGraph.method();
@@ -400,7 +400,7 @@
         PhaseContext context = new PhaseContext(runtime, replacements.getAssumptions(), replacements);
 
         // Copy snippet graph, replacing constant parameters with given arguments
-        StructuredGraph snippetCopy = new StructuredGraph(snippetGraph.name, snippetGraph.method());
+        final StructuredGraph snippetCopy = new StructuredGraph(snippetGraph.name, snippetGraph.method());
         IdentityHashMap<Node, Node> nodeReplacements = new IdentityHashMap<>();
         nodeReplacements.put(snippetGraph.start(), snippetCopy.start());
 
@@ -504,8 +504,13 @@
 
         // Perform lowering on the snippet
         snippetCopy.setGuardsStage(args.cacheKey.guardsStage);
-        PhaseContext c = new PhaseContext(runtime, new Assumptions(false), replacements);
-        new LoweringPhase(new CanonicalizerPhase(true)).apply(snippetCopy, c);
+        Debug.scope("LoweringSnippetTemplate", snippetCopy, new Runnable() {
+
+            public void run() {
+                PhaseContext c = new PhaseContext(runtime, new Assumptions(false), replacements);
+                new LoweringPhase(new CanonicalizerPhase(true)).apply(snippetCopy, c);
+            }
+        });
 
         // Remove all frame states from snippet graph. Snippets must be atomic (i.e. free
         // of side-effects that prevent deoptimizing to a point before the snippet).
@@ -546,6 +551,8 @@
         ReturnNode retNode = null;
         StartNode entryPointNode = snippet.start();
         nodes = new ArrayList<>(snippet.getNodeCount());
+        boolean seenReturn = false;
+        boolean containsMemoryState = false;
         for (Node node : snippet.getNodes()) {
             if (node == entryPointNode || node == entryPointNode.stateAfter()) {
                 // Do nothing.
@@ -557,11 +564,14 @@
                         this.memoryMap = memstate.getMemoryMap();
                         memstate.safeDelete();
                     }
-                    assert snippet.getNodes().filter(ReturnNode.class).count() == 1;
+                    assert !seenReturn : "can handle only one ReturnNode";
+                    seenReturn = true;
+                } else if (node instanceof MemoryState) {
+                    containsMemoryState = true;
                 }
             }
         }
-        assert !containsMemoryState(snippet);
+        assert !containsMemoryState;
 
         this.sideEffectNodes = curSideEffectNodes;
         this.deoptNodes = curDeoptNodes;
@@ -572,10 +582,6 @@
         this.instantiationTimer = Debug.timer("SnippetInstantiationTime[" + method.getName() + "]");
     }
 
-    private static boolean containsMemoryState(StructuredGraph snippet) {
-        return snippet.getNodes().filter(MemoryState.class).count() > 0;
-    }
-
     private static boolean checkAllVarargPlaceholdersAreDeleted(int parameterCount, ConstantNode[] placeholders) {
         for (int i = 0; i < parameterCount; i++) {
             if (placeholders[i] != null) {
@@ -649,7 +655,7 @@
     private final ArrayList<Node> nodes;
 
     /**
-     * mapping of killing locations to memory checkpoints (nodes).
+     * map of killing locations to memory checkpoints (nodes).
      */
     private MemoryMap<Node> memoryMap;
 
@@ -766,11 +772,11 @@
 
                     // lastLocationAccess points into the snippet graph. find a proper
                     // MemoryCheckPoint inside the snippet graph
-                    FloatingReadNode frn = (FloatingReadNode) usage;
-                    Node newlla = mmap.getLastLocationAccess(frn.location().getLocationIdentity());
+                    FloatingReadNode read = (FloatingReadNode) usage;
+                    Node lastAccess = mmap.getLastLocationAccess(read.location().getLocationIdentity());
 
-                    assert newlla != null : "no mapping found for lowerable node " + oldNode + ". (No node in the snippet kill the same location as the lowerable node?)";
-                    frn.setLastLocationAccess(newlla);
+                    assert lastAccess != null : "no mapping found for lowerable node " + oldNode + ". (No node in the snippet kill the same location as the lowerable node?)";
+                    read.setLastLocationAccess(lastAccess);
                 }
             }
         }
@@ -799,7 +805,7 @@
         }
         assert !(replacee instanceof MemoryCheckpoint.Multi) : replacee + " multi not supported (yet)";
 
-        Debug.log("WARNING: %s is not a MemoryCheckpoint, but the snippet graph contains kills (%s). you might want %s to be a MemoryCheckpoint\n", replacee, kills, replacee);
+        Debug.log("WARNING: %s is not a MemoryCheckpoint, but the snippet graph contains kills (%s). You might want %s to be a MemoryCheckpoint", replacee, kills, replacee);
 
         // remove ANY_LOCATION if it's just a kill by the start node
         if (memoryMap.getLastLocationAccess(ANY_LOCATION) instanceof StartNode) {
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.replacements/src/com/oracle/graal/replacements/nodes/MacroNode.java
--- a/graal/com.oracle.graal.replacements/src/com/oracle/graal/replacements/nodes/MacroNode.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.replacements/src/com/oracle/graal/replacements/nodes/MacroNode.java	Thu Sep 26 13:48:19 2013 +0200
@@ -22,6 +22,8 @@
  */
 package com.oracle.graal.replacements.nodes;
 
+import static java.lang.reflect.Modifier.*;
+
 import java.lang.reflect.*;
 
 import com.oracle.graal.api.meta.*;
@@ -106,18 +108,27 @@
 
     @Override
     public void lower(LoweringTool tool) {
-        boolean nullCheck = false;
         StructuredGraph replacementGraph = getLoweredSnippetGraph(tool);
         if (replacementGraph == null) {
             replacementGraph = getLoweredSubstitutionGraph(tool);
-            nullCheck = true;
         }
 
         InvokeNode invoke = replaceWithInvoke();
         assert invoke.verify();
 
         if (replacementGraph != null) {
-            InliningUtil.inline(invoke, replacementGraph, nullCheck);
+            // Pull out the receiver null check so that a replaced
+            // receiver can be lowered if necessary
+            if (!isStatic(targetMethod.getModifiers())) {
+                ValueNode nonNullReceiver = InliningUtil.nonNullReceiver(invoke);
+                if (nonNullReceiver instanceof Lowerable) {
+                    ((Lowerable) nonNullReceiver).lower(tool);
+                }
+            }
+            InliningUtil.inline(invoke, replacementGraph, false);
+            Debug.dump(graph(), "After inlining replacement %s", replacementGraph);
+        } else {
+            invoke.lower(tool);
         }
     }
 
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.truffle/src/com/oracle/graal/truffle/PartialEvaluator.java
--- a/graal/com.oracle.graal.truffle/src/com/oracle/graal/truffle/PartialEvaluator.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.truffle/src/com/oracle/graal/truffle/PartialEvaluator.java	Thu Sep 26 13:48:19 2013 +0200
@@ -223,11 +223,11 @@
                         int nodeCountBefore = graph.getNodeCount();
                         int mark = graph.getMark();
                         InliningUtil.inline(methodCallTargetNode.invoke(), inlineGraph, false);
-                        canonicalizer.applyIncremental(graph, context, mark);
                         if (Debug.isDumpEnabled()) {
                             int nodeCountAfter = graph.getNodeCount();
                             Debug.dump(graph, "After inlining %s %+d (%d)", methodCallTargetNode.targetMethod().toString(), nodeCountAfter - nodeCountBefore, nodeCountAfter);
                         }
+                        canonicalizer.applyIncremental(graph, context, mark);
                         changed = true;
                     }
                 }
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.truffle/src/com/oracle/graal/truffle/TruffleCache.java
--- a/graal/com.oracle.graal.truffle/src/com/oracle/graal/truffle/TruffleCache.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.truffle/src/com/oracle/graal/truffle/TruffleCache.java	Thu Sep 26 13:48:19 2013 +0200
@@ -38,6 +38,7 @@
 import com.oracle.graal.nodes.java.*;
 import com.oracle.graal.nodes.java.MethodCallTargetNode.InvokeKind;
 import com.oracle.graal.nodes.spi.*;
+import com.oracle.graal.nodes.type.*;
 import com.oracle.graal.nodes.util.*;
 import com.oracle.graal.phases.*;
 import com.oracle.graal.phases.common.*;
@@ -200,7 +201,9 @@
     private boolean tryCutOffRuntimeExceptions(MethodCallTargetNode methodCallTargetNode) {
         if (methodCallTargetNode.targetMethod().isConstructor()) {
             ResolvedJavaType runtimeException = metaAccessProvider.lookupJavaType(RuntimeException.class);
-            if (runtimeException.isAssignableFrom(methodCallTargetNode.targetMethod().getDeclaringClass())) {
+            ResolvedJavaType controlFlowException = metaAccessProvider.lookupJavaType(ControlFlowException.class);
+            ResolvedJavaType exceptionType = Objects.requireNonNull(ObjectStamp.typeOrNull(methodCallTargetNode.receiver().stamp()));
+            if (runtimeException.isAssignableFrom(methodCallTargetNode.targetMethod().getDeclaringClass()) && !controlFlowException.isAssignableFrom(exceptionType)) {
                 DeoptimizeNode deoptNode = methodCallTargetNode.graph().add(new DeoptimizeNode(DeoptimizationAction.InvalidateRecompile, DeoptimizationReason.UnreachedCode));
                 FixedNode invokeNode = methodCallTargetNode.invoke().asNode();
                 invokeNode.replaceAtPredecessor(deoptNode);
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.truffle/src/com/oracle/graal/truffle/nodes/asserts/NeverInlineMacroNode.java
--- a/graal/com.oracle.graal.truffle/src/com/oracle/graal/truffle/nodes/asserts/NeverInlineMacroNode.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.truffle/src/com/oracle/graal/truffle/nodes/asserts/NeverInlineMacroNode.java	Thu Sep 26 13:48:19 2013 +0200
@@ -37,5 +37,6 @@
         InvokeNode invoke = createInvoke();
         graph().replaceFixedWithFixed(this, invoke);
         invoke.setUseForInlining(false);
+        invoke.lower(tool);
     }
 }
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.truffle/src/com/oracle/graal/truffle/nodes/frame/FrameAccessNode.java
--- a/graal/com.oracle.graal.truffle/src/com/oracle/graal/truffle/nodes/frame/FrameAccessNode.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.truffle/src/com/oracle/graal/truffle/nodes/frame/FrameAccessNode.java	Thu Sep 26 13:48:19 2013 +0200
@@ -89,7 +89,7 @@
     @Override
     public String toString(Verbosity verbosity) {
         if (verbosity == Verbosity.Name) {
-            return super.toString(verbosity) + getSlotKind().name() + (isConstantFrameSlot() ? " " + getConstantFrameSlot() : "");
+            return super.toString(verbosity) + getSlotKind().name() + (slot != null && isConstantFrameSlot() ? " " + getConstantFrameSlot() : "");
         } else {
             return super.toString(verbosity);
         }
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.truffle/src/com/oracle/graal/truffle/nodes/frame/FrameGetNode.java
--- a/graal/com.oracle.graal.truffle/src/com/oracle/graal/truffle/nodes/frame/FrameGetNode.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.truffle/src/com/oracle/graal/truffle/nodes/frame/FrameGetNode.java	Thu Sep 26 13:48:19 2013 +0200
@@ -88,6 +88,8 @@
             loadNode = graph().add(new UnsafeLoadNode(loadFieldNode, Unsafe.ARRAY_LONG_BASE_OFFSET, slotOffset, getSlotKind()));
         }
         structuredGraph.replaceFixedWithFixed(this, loadNode);
+        loadFieldNode.lower(tool);
+        ((Lowerable) loadNode).lower(tool);
     }
 
     @NodeIntrinsic
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.truffle/src/com/oracle/graal/truffle/nodes/frame/FrameSetNode.java
--- a/graal/com.oracle.graal.truffle/src/com/oracle/graal/truffle/nodes/frame/FrameSetNode.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.truffle/src/com/oracle/graal/truffle/nodes/frame/FrameSetNode.java	Thu Sep 26 13:48:19 2013 +0200
@@ -86,6 +86,8 @@
             storeNode = graph().add(new StoreIndexedNode(loadFieldNode, slotIndex, Kind.Long, value));
         }
         structuredGraph.replaceFixedWithFixed(this, storeNode);
+        loadFieldNode.lower(tool);
+        ((Lowerable) storeNode).lower(tool);
     }
 
     @NodeIntrinsic
diff -r 91a676d0bbbe -r ec90fc830e45 graal/com.oracle.graal.word/src/com/oracle/graal/word/nodes/WordCastNode.java
--- a/graal/com.oracle.graal.word/src/com/oracle/graal/word/nodes/WordCastNode.java	Thu Sep 26 13:47:12 2013 +0200
+++ b/graal/com.oracle.graal.word/src/com/oracle/graal/word/nodes/WordCastNode.java	Thu Sep 26 13:48:19 2013 +0200
@@ -32,7 +32,7 @@
  * Cast between Word and Object that is introduced by the {@link WordTypeRewriterPhase}. It has an
  * impact on the pointer maps for the GC, so it must not be scheduled or optimized away.
  */
-public final class WordCastNode extends FixedWithNextNode implements LIRLowerable {
+public final class WordCastNode extends FixedWithNextNode implements LIRLowerable, Canonicalizable {
 
     public static WordCastNode wordToObject(ValueNode input, Kind wordKind) {
         assert input.kind() == wordKind;
@@ -55,6 +55,14 @@
         return input;
     }
 
+    public ValueNode canonical(CanonicalizerTool tool) {
+        if (usages().count() == 0) {
+            /* If the cast is unused, it can be eliminated. */
+            return input;
+        }
+        return this;
+    }
+
     @Override
     public void generate(LIRGeneratorTool generator) {
         assert kind() != input.kind();
diff -r 91a676d0bbbe -r ec90fc830e45 mx/commands.py
--- a/mx/commands.py	Thu Sep 26 13:47:12 2013 +0200
+++ b/mx/commands.py	Thu Sep 26 13:48:19 2013 +0200
@@ -345,8 +345,8 @@
                 pass
     else:
         if not exists(jdk):
-            if _installed_jdks and mx._opts.verbose:
-                mx.log("Could not find JDK directory at " + jdk)
+            if _installed_jdks:
+                mx.log("The selected JDK directory does not (yet) exist: " + jdk)
             _handle_missing_VM(build, vmToCheck if vmToCheck else 'graal')
 
     if installGraalJar:
diff -r 91a676d0bbbe -r ec90fc830e45 src/cpu/sparc/vm/graalCodeInstaller_sparc.hpp
--- a/src/cpu/sparc/vm/graalCodeInstaller_sparc.hpp	Thu Sep 26 13:47:12 2013 +0200
+++ b/src/cpu/sparc/vm/graalCodeInstaller_sparc.hpp	Thu Sep 26 13:48:19 2013 +0200
@@ -182,4 +182,24 @@
   }
 }
 
+inline void CodeInstaller::pd_relocate_poll(address pc, jint mark) {
+  switch (mark) {
+    case MARK_POLL_NEAR: {
+      fatal("unimplemented");
+    }
+    case MARK_POLL_FAR:
+      _instructions->relocate(pc, relocInfo::poll_type);
+      break;
+    case MARK_POLL_RETURN_NEAR: {
+      fatal("unimplemented");
+    }
+    case MARK_POLL_RETURN_FAR:
+      _instructions->relocate(pc, relocInfo::poll_return_type);
+      break;
+    default:
+      fatal("invalid mark value");
+      break;
+  }
+}
+
 #endif // CPU_SPARC_VM_CODEINSTALLER_SPARC_HPP
diff -r 91a676d0bbbe -r ec90fc830e45 src/cpu/x86/vm/graalCodeInstaller_x86.hpp
--- a/src/cpu/x86/vm/graalCodeInstaller_x86.hpp	Thu Sep 26 13:47:12 2013 +0200
+++ b/src/cpu/x86/vm/graalCodeInstaller_x86.hpp	Thu Sep 26 13:48:19 2013 +0200
@@ -232,7 +232,7 @@
       _instructions->relocate(pc, relocInfo::poll_return_type);
       break;
     default:
-      ShouldNotReachHere();
+      fatal("invalid mark value");
       break;
   }
 }