# HG changeset patch
# User Thomas Wuerthinger <thomas.wuerthinger@oracle.com>
# Date 1340700963 -7200
# Node ID 7ee5a3634003573cd79406e98366172800ac43a0
# Parent  bb94f57c822b065df25f7d380922ce3325bd0cc2# Parent  1d3df3a16940636a94a26c7a1221569f3f6a388f
Merge.

diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.boot/src/com/oracle/graal/boot/meta/ArrayTypeElement.java
--- a/graal/com.oracle.graal.boot/src/com/oracle/graal/boot/meta/ArrayTypeElement.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.boot/src/com/oracle/graal/boot/meta/ArrayTypeElement.java	Tue Jun 26 10:56:03 2012 +0200
@@ -48,14 +48,14 @@
     @Override
     protected void propagateTypesToUsage(BigBang bb, Node use, Set<ResolvedJavaType> set, Element element) {
         LoadIndexedNode load = (LoadIndexedNode) use;
-        ResolvedJavaType declaredType = load.array().stamp().declaredType();
-        if (declaredType == null) {
-            System.out.println("FATAL error: Array access without declared type!");
+        ResolvedJavaType type = load.array().objectStamp().type();
+        if (type == null) {
+            System.out.println("FATAL error: Array access without type!");
             System.out.println(load.array());
             System.out.println(((StructuredGraph) load.graph()).method());
             System.exit(-1);
         }
-        ResolvedJavaType componentType = declaredType.componentType();
+        ResolvedJavaType componentType = type.componentType();
         Set<ResolvedJavaType> newSet = new HashSet<>();
         for (ResolvedJavaType myType : set) {
             if (myType.isSubtypeOf(componentType)) {
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.boot/src/com/oracle/graal/boot/meta/InvokeElement.java
--- a/graal/com.oracle.graal.boot/src/com/oracle/graal/boot/meta/InvokeElement.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.boot/src/com/oracle/graal/boot/meta/InvokeElement.java	Tue Jun 26 10:56:03 2012 +0200
@@ -51,7 +51,7 @@
         int index = 0;
         for (Node arg : methodCallTarget.arguments()) {
             if (arg == sourceNode) {
-                System.out.println("source node " + sourceNode + " is at index " + index + " declaredType=" + ((ValueNode) sourceNode).stamp().declaredType());
+                System.out.println("source node " + sourceNode + " is at index " + index + " stamp=" + ((ValueNode) sourceNode).stamp());
                 unionTypes(bb, sourceNode, newSeenTypes, index);
             }
             ++index;
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/GraalCompiler.java
--- a/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/GraalCompiler.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/GraalCompiler.java	Tue Jun 26 10:56:03 2012 +0200
@@ -134,7 +134,6 @@
 
         if (GraalOptions.Inline && !plan.isPhaseDisabled(InliningPhase.class)) {
             new InliningPhase(target, runtime, null, assumptions, cache, plan, optimisticOpts).apply(graph);
-            new DeadCodeEliminationPhase().apply(graph);
             new PhiStampPhase().apply(graph);
 
             if (GraalOptions.PropagateTypes) {
@@ -153,19 +152,17 @@
 
         plan.runPhases(PhasePosition.HIGH_LEVEL, graph);
 
+        if (GraalOptions.FullUnroll) {
+            new LoopFullUnrollPhase(runtime).apply(graph);
+        }
+
         if (GraalOptions.EscapeAnalysis && !plan.isPhaseDisabled(EscapeAnalysisPhase.class)) {
             new EscapeAnalysisPhase(target, runtime, assumptions, cache, plan, optimisticOpts).apply(graph);
             new PhiStampPhase().apply(graph);
-            if (GraalOptions.OptCanonicalizer) {
-                new CanonicalizerPhase(target, runtime, assumptions).apply(graph);
-            }
         }
-        if (GraalOptions.OptLoops) {
-            if (GraalOptions.OptLoopTransform) {
-                new LoopTransformPhase().apply(graph);
-            }
+        if (GraalOptions.OptLoopTransform) {
+            new LoopTransformHighPhase().apply(graph);
         }
-        new RemoveValueProxyPhase().apply(graph);
         if (GraalOptions.OptCanonicalizer) {
             new CanonicalizerPhase(target, runtime, assumptions).apply(graph);
         }
@@ -191,15 +188,26 @@
             new CheckCastEliminationPhase().apply(graph);
         }
 
+        if (GraalOptions.OptLoopTransform) {
+            new LoopTransformLowPhase().apply(graph);
+        }
+        new RemoveValueProxyPhase().apply(graph);
         if (GraalOptions.OptCanonicalizer) {
             new CanonicalizerPhase(target, runtime, assumptions).apply(graph);
         }
-        new DeadCodeEliminationPhase().apply(graph);
+        if (GraalOptions.CheckCastElimination) {
+            new CheckCastEliminationPhase().apply(graph);
+        }
+
 
         plan.runPhases(PhasePosition.MID_LEVEL, graph);
 
         plan.runPhases(PhasePosition.LOW_LEVEL, graph);
 
+        new DeadCodeEliminationPhase().apply(graph);
+        if (GraalOptions.OptCanonicalizer) {
+            new CanonicalizerPhase(target, runtime, assumptions).apply(graph);
+        }
         // Add safepoints to loops
         if (GraalOptions.GenLoopSafepoints) {
             new LoopSafepointInsertionPhase().apply(graph);
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/GraalOptions.java
--- a/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/GraalOptions.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/GraalOptions.java	Tue Jun 26 10:56:03 2012 +0200
@@ -200,7 +200,6 @@
     public static boolean OptReadElimination                 = true;
     public static boolean OptGVN                             = true;
     public static boolean OptCanonicalizer                   = true;
-    public static boolean OptLoops                           = true;
     public static boolean ScheduleOutOfLoops                 = true;
     public static boolean OptReorderLoops                    = true;
     public static boolean OptEliminateGuards                 = true;
@@ -209,6 +208,11 @@
     public static boolean OptLoopTransform                   = true;
     public static boolean OptSafepointElimination            = true;
 
+    // Loops
+    public static boolean ReassociateInvariants              = true;
+    public static boolean FullUnroll                         = true;
+    public static int FullUnrollMaxNodes                     = 250; // TODO (gd) tune
+
     /**
      * Insert a counter in the method prologue to track the most frequently called methods that were compiled by Graal.
      */
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/gen/DebugInfoBuilder.java
--- a/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/gen/DebugInfoBuilder.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/gen/DebugInfoBuilder.java	Tue Jun 26 10:56:03 2012 +0200
@@ -82,7 +82,7 @@
                                 VirtualObjectState currentField = objectStates.get(vobj);
                                 assert currentField != null;
                                 for (int i = 0; i < vobj.fieldsCount(); i++) {
-                                    values[i] = toCiValue(currentField.fields().get(i));
+                                    values[i] = toCiValue(currentField.fieldValues().get(i));
                                 }
                             }
                         }
@@ -100,7 +100,7 @@
     private BytecodeFrame computeFrameForState(FrameState state, LockScope locks, long leafGraphId) {
         int numLocals = state.localsSize();
         int numStack = state.stackSize();
-        int numLocks = (locks != null && locks.callerState == state.outerFrameState()) ? locks.stateDepth + 1 : 0;
+        int numLocks = (locks != null && locks.inliningIdentifier == state.inliningIdentifier()) ? locks.stateDepth + 1 : 0;
 
         Value[] values = new Value[numLocals + numStack + numLocks];
         for (int i = 0; i < numLocals; i++) {
@@ -112,7 +112,7 @@
 
         LockScope nextLock = locks;
         for (int i = numLocks - 1; i >= 0; i--) {
-            assert locks != null && nextLock.callerState == state.outerFrameState() && nextLock.stateDepth == i;
+            assert locks != null && nextLock.inliningIdentifier == state.inliningIdentifier() && nextLock.stateDepth == i;
 
             Value owner = toCiValue(nextLock.monitor.object());
             Value lockData = nextLock.lockData;
@@ -153,7 +153,7 @@
         } else if (value != null) {
             Debug.metric("StateVariables").increment();
             Value operand = nodeOperands.get(value);
-            assert operand != null && (operand instanceof Variable || operand instanceof Constant);
+            assert operand != null && (operand instanceof Variable || operand instanceof Constant) : operand + " for " + value;
             return operand;
 
         } else {
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/gen/LIRGenerator.java
--- a/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/gen/LIRGenerator.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/gen/LIRGenerator.java	Tue Jun 26 10:56:03 2012 +0200
@@ -23,6 +23,7 @@
 package com.oracle.graal.compiler.gen;
 
 import static com.oracle.graal.api.code.CallingConvention.Type.*;
+import static com.oracle.graal.api.code.ValueUtil.*;
 import static com.oracle.graal.api.meta.Value.*;
 import static com.oracle.graal.lir.LIRValueUtil.*;
 
@@ -45,6 +46,7 @@
 import com.oracle.graal.lir.StandardOp.PhiLabelOp;
 import com.oracle.graal.lir.cfg.*;
 import com.oracle.graal.nodes.*;
+import com.oracle.graal.nodes.FrameState.InliningIdentifier;
 import com.oracle.graal.nodes.PhiNode.PhiType;
 import com.oracle.graal.nodes.calc.*;
 import com.oracle.graal.nodes.extended.*;
@@ -94,12 +96,10 @@
         public final LockScope outer;
 
         /**
-         * The frame state of the caller of the method performing the lock, or null if the outermost method
+         * The identifier of the actual inlined method instance performing the lock, or null if the outermost method
          * performs the lock. This information is used to compute the {@link BytecodeFrame} that this lock belongs to.
-         * We cannot use the actual frame state of the locking method, because it is not unique for a method. The
-         * caller frame states are unique, i.e., all frame states of inlined methods refer to the same caller frame state.
          */
-        public final FrameState callerState;
+        public final InliningIdentifier inliningIdentifier;
 
         /**
          * The number of locks already found for this frame state.
@@ -116,12 +116,12 @@
          */
         public final StackSlot lockData;
 
-        public LockScope(LockScope outer, FrameState callerState, MonitorEnterNode monitor, StackSlot lockData) {
+        public LockScope(LockScope outer, InliningIdentifier inliningIdentifier, MonitorEnterNode monitor, StackSlot lockData) {
             this.outer = outer;
-            this.callerState = callerState;
+            this.inliningIdentifier = inliningIdentifier;
             this.monitor = monitor;
             this.lockData = lockData;
-            if (outer != null && outer.callerState == callerState) {
+            if (outer != null && outer.inliningIdentifier == inliningIdentifier) {
                 this.stateDepth = outer.stateDepth + 1;
             } else {
                 this.stateDepth = 0;
@@ -207,9 +207,15 @@
     }
 
     @Override
+    public RegisterAttributes attributes(Register register) {
+        return frameMap.registerConfig.getAttributesMap()[register.number];
+    }
+
+    @Override
     public Value setResult(ValueNode x, Value operand) {
-        assert (isVariable(operand) && x.kind() == operand.kind) || (isConstant(operand) && x.kind() == operand.kind.stackKind()) : operand.kind + " for node " + x;
-
+        assert (isVariable(operand) && x.kind() == operand.kind) ||
+               (isRegister(operand) && !attributes(asRegister(operand)).isAllocatable()) ||
+               (isConstant(operand) && x.kind() == operand.kind.stackKind()) : operand.kind + " for node " + x;
         assert operand(x) == null : "operand cannot be set twice";
         assert operand != null && isLegal(operand) : "operand must be legal";
         assert operand.kind.stackKind() == x.kind();
@@ -539,7 +545,7 @@
         if (x.eliminated()) {
             // No code is emitted for eliminated locks, but for proper debug information generation we need to
             // register the monitor and its lock data.
-            curLocks = new LockScope(curLocks, x.stateAfter().outerFrameState(), x, lockData);
+            curLocks = new LockScope(curLocks, x.stateAfter().inliningIdentifier(), x, lockData);
             return;
         }
 
@@ -548,7 +554,7 @@
 
         LIRDebugInfo stateBefore = state();
         // The state before the monitor enter is used for null checks, so it must not contain the newly locked object.
-        curLocks = new LockScope(curLocks, x.stateAfter().outerFrameState(), x, lockData);
+        curLocks = new LockScope(curLocks, x.stateAfter().inliningIdentifier(), x, lockData);
         // The state after the monitor enter is used for deoptimization, after the monitor has blocked, so it must contain the newly locked object.
         LIRDebugInfo stateAfter = stateFor(x.stateAfter(), -1);
 
@@ -1023,6 +1029,17 @@
     }
 
     @Override
+    public void emitTypeSwitch(TypeSwitchNode x) {
+        Variable tag = load(operand(x.value()));
+        int len = x.numberOfCases();
+        for (int i = 0; i < len; i++) {
+            ResolvedJavaType type = x.keyAt(i);
+            emitBranch(tag, type.getEncoding(Representation.ObjectHub), Condition.EQ, false, getLIRBlock(x.blockSuccessor(i)), null);
+        }
+        emitJump(getLIRBlock(x.defaultSuccessor()), null);
+    }
+
+    @Override
     public void emitTableSwitch(TableSwitchNode x) {
         Variable value = load(operand(x.value()));
         // TODO: tune the defaults for the controls used to determine what kind of translation to use
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/loop/CountedLoopInfo.java
--- a/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/loop/CountedLoopInfo.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/loop/CountedLoopInfo.java	Tue Jun 26 10:56:03 2012 +0200
@@ -22,6 +22,7 @@
  */
 package com.oracle.graal.compiler.loop;
 
+import com.oracle.graal.compiler.loop.InductionVariable.*;
 import com.oracle.graal.nodes.*;
 import com.oracle.graal.nodes.calc.*;
 
@@ -29,14 +30,17 @@
     private final LoopEx loop;
     private InductionVariable iv;
     private ValueNode end;
+    private boolean oneOff;
 
-    CountedLoopInfo(LoopEx loop, InductionVariable iv, ValueNode end) {
+    CountedLoopInfo(LoopEx loop, InductionVariable iv, ValueNode end, boolean oneOff) {
         this.loop = loop;
         this.iv = iv;
         this.end = end;
+        this.oneOff = oneOff;
     }
 
     public ValueNode maxTripCountNode() {
+        //TODO (gd) stuarte and respect oneOff
         return IntegerArithmeticNode.div(IntegerArithmeticNode.sub(end, iv.initNode()), iv.strideNode());
     }
 
@@ -45,7 +49,9 @@
     }
 
     public long constantMaxTripCount() {
-        return (((ConstantNode) end).asConstant().asLong() - iv.constantInit()) / iv.constantStride();
+        long off = oneOff ? iv.direction() == Direction.Up ? 1 : -1 : 0;
+        long max = (((ConstantNode) end).asConstant().asLong() + off - iv.constantInit()) / iv.constantStride();
+        return Math.max(0, max);
     }
 
     public boolean isExactTripCount() {
@@ -66,4 +72,9 @@
         assert isExactTripCount();
         return constantMaxTripCount();
     }
+
+    @Override
+    public String toString() {
+        return "iv=" + iv + " until " + end + (oneOff ? iv.direction() == Direction.Up ? "+1" : "-1" : "");
+    }
 }
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/loop/LoopEx.java
--- a/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/loop/LoopEx.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/loop/LoopEx.java	Tue Jun 26 10:56:03 2012 +0200
@@ -22,9 +22,14 @@
  */
 package com.oracle.graal.compiler.loop;
 
+import com.oracle.graal.api.code.*;
+import com.oracle.graal.api.meta.*;
+import com.oracle.graal.debug.*;
 import com.oracle.graal.graph.*;
+import com.oracle.graal.graph.iterators.*;
 import com.oracle.graal.lir.cfg.*;
 import com.oracle.graal.nodes.*;
+import com.oracle.graal.nodes.calc.*;
 
 public class LoopEx {
     private final Loop lirLoop;
@@ -105,6 +110,32 @@
 
     @Override
     public String toString() {
-        return isCounted() ? "Counted" : "" + "Loop (depth=" + lirLoop().depth + ") " + loopBegin();
+        return (isCounted() ? "CountedLoop [" + counted() + "] " : "Loop ") + "(depth=" + lirLoop().depth + ") " + loopBegin();
+    }
+
+    private class InvariantPredicate extends NodePredicate {
+        @Override
+        public boolean apply(Node n) {
+            return isOutsideLoop(n);
+        }
+    }
+
+    public void reassociateInvariants() {
+        InvariantPredicate invariant = new InvariantPredicate();
+        StructuredGraph graph = (StructuredGraph) loopBegin().graph();
+        for (BinaryNode binary : whole().nodes().filter(BinaryNode.class)) {
+            if (!BinaryNode.canTryReassociate(binary)) {
+                continue;
+            }
+            BinaryNode result = BinaryNode.reassociate(binary, invariant);
+            if (result != binary) {
+                Debug.log(CodeUtil.format("%H::%n", Debug.contextLookup(ResolvedJavaMethod.class)) + " : Reassociated %s into %s", binary, result);
+                graph.replaceFloating(binary, result);
+            }
+        }
+    }
+
+    public void setCounted(CountedLoopInfo countedLoopInfo) {
+        counted = countedLoopInfo;
     }
 }
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/loop/LoopFragment.java
--- a/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/loop/LoopFragment.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/loop/LoopFragment.java	Tue Jun 26 10:56:03 2012 +0200
@@ -212,20 +212,22 @@
         assert isDuplicate();
         StructuredGraph graph = graph();
         for (BeginNode earlyExit : toHirBlocks(original().loop().lirLoop().exits)) {
-            if (!this.contains(earlyExit)) {
+            FixedNode next = earlyExit.next();
+            if (earlyExit.isDeleted() || !this.contains(earlyExit)) {
                 continue;
             }
             BeginNode newEarlyExit = getDuplicatedNode(earlyExit);
             assert newEarlyExit != null;
             MergeNode merge = graph.add(new MergeNode());
+            merge.setProbability(next.probability());
             EndNode originalEnd = graph.add(new EndNode());
             EndNode newEnd = graph.add(new EndNode());
             merge.addForwardEnd(originalEnd);
             merge.addForwardEnd(newEnd);
-            FixedNode next = earlyExit.next();
             earlyExit.setNext(originalEnd);
             newEarlyExit.setNext(newEnd);
             merge.setNext(next);
+
             FrameState exitState = earlyExit.stateAfter();
             FrameState state = null;
             if (exitState != null) {
@@ -244,7 +246,6 @@
                     PhiNode phi = graph.add(vpn.type() == PhiType.Value ? new PhiNode(vpn.kind(), merge) : new PhiNode(vpn.type(), merge));
                     phi.addInput(vpn);
                     phi.addInput(newVpn);
-                    assert vpn.type() == PhiType.Value;
                     replaceWith = phi;
                 } else {
                     replaceWith = vpn.value();
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/loop/LoopFragmentInside.java
--- a/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/loop/LoopFragmentInside.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/loop/LoopFragmentInside.java	Tue Jun 26 10:56:03 2012 +0200
@@ -28,7 +28,8 @@
 import com.oracle.graal.graph.*;
 import com.oracle.graal.graph.iterators.*;
 import com.oracle.graal.nodes.*;
-import com.oracle.graal.nodes.PhiNode.*;
+import com.oracle.graal.nodes.PhiNode.PhiType;
+import com.oracle.graal.nodes.VirtualState.NodeClosure;
 import com.oracle.graal.nodes.util.*;
 
 
@@ -94,6 +95,7 @@
             GraphUtil.killWithUnusedFloatingInputs(state);
         }
         loop.entryPoint().replaceAtPredecessor(entry);
+        end.setProbability(loop.entryPoint().probability());
         end.setNext(loop.entryPoint());
     }
 
@@ -234,8 +236,8 @@
                 newExitMerge.addForwardEnd(end);
             }
 
-            for (PhiNode phi : loopBegin.phis().snapshot()) {
-                PhiNode firstPhi = graph.add(phi.type() == PhiType.Value ? new PhiNode(phi.kind(), newExitMerge) : new PhiNode(phi.type(), newExitMerge));
+            for (final PhiNode phi : loopBegin.phis().snapshot()) {
+                final PhiNode firstPhi = graph.add(phi.type() == PhiType.Value ? new PhiNode(phi.kind(), newExitMerge) : new PhiNode(phi.type(), newExitMerge));
                 for (EndNode end : newExitMerge.forwardEnds()) {
                     LoopEndNode loopEnd = reverseEnds.get(end);
                     ValueNode prim = prim(phi.valueAt(loopEnd));
@@ -244,7 +246,15 @@
                 }
                 ValueNode initializer = firstPhi;
                 if (duplicateState != null) {
-                    duplicateState.replaceFirstInput(phi, firstPhi); // fix the merge's state after
+                    // fix the merge's state after
+                    duplicateState.applyToNonVirtual(new NodeClosure<ValueNode>() {
+                        @Override
+                        public void apply(Node from, ValueNode node) {
+                            if (node == phi) {
+                                from.replaceFirstInput(phi, firstPhi);
+                            }
+                        }
+                    });
                 }
                 mergedInitializers.put(phi, initializer);
             }
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/loop/LoopPolicies.java
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/loop/LoopPolicies.java	Tue Jun 26 10:56:03 2012 +0200
@@ -0,0 +1,50 @@
+/*
+ * Copyright (c) 2012, 2012, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.graal.compiler.loop;
+
+import com.oracle.graal.compiler.*;
+import com.oracle.graal.nodes.*;
+
+
+public abstract class LoopPolicies {
+    private LoopPolicies() {
+        // does not need to be instantiated
+    }
+
+    // TODO (gd) change when inversion is available
+    public static boolean shouldPeel(LoopEx loop) {
+        LoopBeginNode loopBegin = loop.loopBegin();
+        double entryProbability = loopBegin.forwardEnd().probability();
+        return entryProbability > GraalOptions.MinimumPeelProbability && loop.size() + loopBegin.graph().getNodeCount() < GraalOptions.MaximumDesiredSize;
+    }
+
+    public static boolean shouldFullUnroll(LoopEx loop) {
+        if (!loop.isCounted() || !loop.counted().isConstantMaxTripCount()) {
+            return false;
+        }
+        long exactTrips = loop.counted().constantMaxTripCount();
+        int maxNodes = Math.min(GraalOptions.FullUnrollMaxNodes, GraalOptions.MaximumDesiredSize - loop.loopBegin().graph().getNodeCount());
+        int size = Math.max(1, loop.size() - 1 - loop.loopBegin().phis().count());
+        return size * exactTrips <= maxNodes;
+    }
+}
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/loop/LoopTransformations.java
--- a/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/loop/LoopTransformations.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/loop/LoopTransformations.java	Tue Jun 26 10:56:03 2012 +0200
@@ -22,11 +22,19 @@
  */
 package com.oracle.graal.compiler.loop;
 
+import com.oracle.graal.api.code.*;
+import com.oracle.graal.compiler.*;
 import com.oracle.graal.compiler.phases.*;
 import com.oracle.graal.nodes.*;
 
 
 public abstract class LoopTransformations {
+    private static final int UNROLL_LIMIT = GraalOptions.FullUnrollMaxNodes * 2;
+
+    private LoopTransformations() {
+        // does not need to be instantiated
+    }
+
     public static void invert(LoopEx loop, FixedNode point) {
         LoopFragmentInsideBefore head = loop.insideBefore(point);
         LoopFragmentInsideBefore duplicate = head.duplicate();
@@ -39,14 +47,18 @@
         loop.inside().duplicate().insertBefore(loop);
     }
 
-    public static void fullUnroll(LoopEx loop) {
+    public static void fullUnroll(LoopEx loop, CodeCacheProvider runtime) {
         //assert loop.isCounted(); //TODO (gd) strenghten : counted with known trip count
+        int iterations = 0;
         LoopBeginNode loopBegin = loop.loopBegin();
         StructuredGraph graph = (StructuredGraph) loopBegin.graph();
         while (!loopBegin.isDeleted()) {
             int mark = graph.getMark();
             peel(loop);
-            new CanonicalizerPhase(null, null, null, mark, null).apply(graph);
+            new CanonicalizerPhase(null, runtime, null, mark, null).apply(graph);
+            if (iterations++ > UNROLL_LIMIT || graph.getNodeCount() > GraalOptions.MaximumDesiredSize * 3) {
+                throw new BailoutException("FullUnroll : Graph seems to grow out of proportion");
+            }
         }
     }
 
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/loop/LoopsData.java
--- a/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/loop/LoopsData.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/loop/LoopsData.java	Tue Jun 26 10:56:03 2012 +0200
@@ -23,6 +23,7 @@
 package com.oracle.graal.compiler.loop;
 
 import java.util.*;
+import java.util.concurrent.*;
 
 import com.oracle.graal.compiler.loop.InductionVariable.*;
 import com.oracle.graal.debug.*;
@@ -35,8 +36,14 @@
     private Map<Loop, LoopEx> lirLoopToEx = new IdentityHashMap<>();
     private Map<LoopBeginNode, LoopEx> loopBeginToEx = new IdentityHashMap<>();
 
-    public LoopsData(StructuredGraph graph) {
-        ControlFlowGraph cfg = ControlFlowGraph.compute(graph, true, true, true, false);
+    public LoopsData(final StructuredGraph graph) {
+
+        ControlFlowGraph cfg = Debug.scope("ControlFlowGraph", new Callable<ControlFlowGraph>() {
+            @Override
+            public ControlFlowGraph call() throws Exception {
+                return ControlFlowGraph.compute(graph, true, true, true, false);
+            }
+        });
         for (Loop lirLoop : cfg.getLoops()) {
             LoopEx ex = new LoopEx(lirLoop, this);
             lirLoopToEx.put(lirLoop, ex);
@@ -139,8 +146,7 @@
                         break;
                     default: throw GraalInternalError.shouldNotReachHere();
                 }
-                // TODO (gd)
-                Debug.log("absorb %b %s", oneOff, limit);
+                loop.setCounted(new CountedLoopInfo(loop, iv, limit, oneOff));
             }
         }
     }
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/phases/CanonicalizerPhase.java
--- a/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/phases/CanonicalizerPhase.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/phases/CanonicalizerPhase.java	Tue Jun 26 10:56:03 2012 +0200
@@ -22,8 +22,6 @@
  */
 package com.oracle.graal.compiler.phases;
 
-import java.util.concurrent.*;
-
 import com.oracle.graal.api.code.*;
 import com.oracle.graal.api.meta.*;
 import com.oracle.graal.debug.*;
@@ -32,12 +30,15 @@
 import com.oracle.graal.nodes.*;
 import com.oracle.graal.nodes.calc.*;
 import com.oracle.graal.nodes.spi.*;
+import com.oracle.graal.nodes.type.*;
 import com.oracle.graal.nodes.util.*;
 
 public class CanonicalizerPhase extends Phase {
     private static final int MAX_ITERATION_PER_NODE = 10;
     private static final DebugMetric METRIC_CANONICALIZED_NODES = Debug.metric("CanonicalizedNodes");
     private static final DebugMetric METRIC_CANONICALIZATION_CONSIDERED_NODES = Debug.metric("CanonicalizationConsideredNodes");
+    private static final DebugMetric METRIC_INFER_STAMP_CALLED = Debug.metric("InferStampCalled");
+    private static final DebugMetric METRIC_STAMP_CHANGED = Debug.metric("StampChanged");
     private static final DebugMetric METRIC_SIMPLIFICATION_CONSIDERED_NODES = Debug.metric("SimplificationConsideredNodes");
     public static final DebugMetric METRIC_GLOBAL_VALUE_NUMBERING_HITS = Debug.metric("GlobalValueNumberingHits");
 
@@ -129,29 +130,30 @@
         graph.stopTrackingInputChange();
     }
 
-    private void processNode(Node n, StructuredGraph graph) {
-        if (n.isAlive()) {
+    private void processNode(Node node, StructuredGraph graph) {
+        if (node.isAlive()) {
             METRIC_PROCESSED_NODES.increment();
 
-            if (tryGlobalValueNumbering(n, graph)) {
+            if (tryGlobalValueNumbering(node, graph)) {
                 return;
             }
             int mark = graph.getMark();
-            tryCanonicalize(n, graph, tool);
+            tryCanonicalize(node, graph, tool);
+            tryInferStamp(node, graph);
 
-            for (Node node : graph.getNewNodes(mark)) {
-                workList.add(node);
+            for (Node newNode : graph.getNewNodes(mark)) {
+                workList.add(newNode);
             }
         }
     }
 
-    public static boolean tryGlobalValueNumbering(Node n, StructuredGraph graph) {
-        if (n.getNodeClass().valueNumberable()) {
-            Node newNode = graph.findDuplicate(n);
+    public static boolean tryGlobalValueNumbering(Node node, StructuredGraph graph) {
+        if (node.getNodeClass().valueNumberable()) {
+            Node newNode = graph.findDuplicate(node);
             if (newNode != null) {
-                assert !(n instanceof FixedNode || newNode instanceof FixedNode);
-                n.replaceAtUsages(newNode);
-                n.safeDelete();
+                assert !(node instanceof FixedNode || newNode instanceof FixedNode);
+                node.replaceAtUsages(newNode);
+                node.safeDelete();
                 METRIC_GLOBAL_VALUE_NUMBERING_HITS.increment();
                 Debug.log("GVN applied and new node is %1s", newNode);
                 return true;
@@ -160,14 +162,12 @@
         return false;
     }
 
-    public static void tryCanonicalize(final Node node, StructuredGraph graph, final SimplifierTool tool) {
+    public static void tryCanonicalize(final Node node, final StructuredGraph graph, final SimplifierTool tool) {
         if (node instanceof Canonicalizable) {
             METRIC_CANONICALIZATION_CONSIDERED_NODES.increment();
-            ValueNode canonical = Debug.scope("CanonicalizeNode", node, new Callable<ValueNode>() {
-                public ValueNode call() throws Exception {
-                    return ((Canonicalizable) node).canonical(tool);
-                }
-            });
+            Debug.scope("CanonicalizeNode", node, new Runnable() {
+                public void run() {
+                    ValueNode canonical = ((Canonicalizable) node).canonical(tool);
 //     cases:                                           original node:
 //                                         |Floating|Fixed-unconnected|Fixed-connected|
 //                                         --------------------------------------------
@@ -180,45 +180,47 @@
 //                          Fixed-connected|   2    |        X        |       6       |
 //                                         --------------------------------------------
 //       X: must not happen (checked with assertions)
-            if (canonical == node) {
-                Debug.log("Canonicalizer: work on %s", node);
-            } else {
-                Debug.log("Canonicalizer: replacing %s with %s", node, canonical);
-
-                METRIC_CANONICALIZED_NODES.increment();
-                if (node instanceof FloatingNode) {
-                    if (canonical == null) {
-                        // case 1
-                        graph.removeFloating((FloatingNode) node);
+                    if (canonical == node) {
+                        Debug.log("Canonicalizer: work on %s", node);
                     } else {
-                        // case 2
-                        assert !(canonical instanceof FixedNode) || canonical.predecessor() != null : node + " -> " + canonical +
-                                        " : replacement should be floating or fixed and connected";
-                        graph.replaceFloating((FloatingNode) node, canonical);
-                    }
-                } else {
-                    assert node instanceof FixedWithNextNode && node.predecessor() != null : node + " -> " + canonical + " : node should be fixed & connected (" + node.predecessor() + ")";
-                    if (canonical == null) {
-                        // case 3
-                        graph.removeFixed((FixedWithNextNode) node);
-                    } else if (canonical instanceof FloatingNode) {
-                        // case 4
-                        graph.replaceFixedWithFloating((FixedWithNextNode) node, (FloatingNode) canonical);
-                    } else {
-                        assert canonical instanceof FixedNode;
-                        if (canonical.predecessor() == null) {
-                            assert !canonical.cfgSuccessors().iterator().hasNext() : "replacement " + canonical + " shouldn't have successors";
-                            // case 5
-                            graph.replaceFixedWithFixed((FixedWithNextNode) node, (FixedWithNextNode) canonical);
+                        Debug.log("Canonicalizer: replacing %s with %s", node, canonical);
+
+                        METRIC_CANONICALIZED_NODES.increment();
+                        if (node instanceof FloatingNode) {
+                            if (canonical == null) {
+                                // case 1
+                                graph.removeFloating((FloatingNode) node);
+                            } else {
+                                // case 2
+                                assert !(canonical instanceof FixedNode) || canonical.predecessor() != null : node + " -> " + canonical +
+                                                " : replacement should be floating or fixed and connected";
+                                graph.replaceFloating((FloatingNode) node, canonical);
+                            }
                         } else {
-                            assert canonical.cfgSuccessors().iterator().hasNext() : "replacement " + canonical + " should have successors";
-                            // case 6
-                            node.replaceAtUsages(canonical);
-                            graph.removeFixed((FixedWithNextNode) node);
+                            assert node instanceof FixedWithNextNode && node.predecessor() != null : node + " -> " + canonical + " : node should be fixed & connected (" + node.predecessor() + ")";
+                            if (canonical == null) {
+                                // case 3
+                                graph.removeFixed((FixedWithNextNode) node);
+                            } else if (canonical instanceof FloatingNode) {
+                                // case 4
+                                graph.replaceFixedWithFloating((FixedWithNextNode) node, (FloatingNode) canonical);
+                            } else {
+                                assert canonical instanceof FixedNode;
+                                if (canonical.predecessor() == null) {
+                                    assert !canonical.cfgSuccessors().iterator().hasNext() : "replacement " + canonical + " shouldn't have successors";
+                                    // case 5
+                                    graph.replaceFixedWithFixed((FixedWithNextNode) node, (FixedWithNextNode) canonical);
+                                } else {
+                                    assert canonical.cfgSuccessors().iterator().hasNext() : "replacement " + canonical + " should have successors";
+                                    // case 6
+                                    node.replaceAtUsages(canonical);
+                                    graph.removeFixed((FixedWithNextNode) node);
+                                }
+                            }
                         }
                     }
                 }
-            }
+            });
         } else if (node instanceof Simplifiable) {
             Debug.log("Canonicalizer: simplifying %s", node);
             METRIC_SIMPLIFICATION_CONSIDERED_NODES.increment();
@@ -226,6 +228,30 @@
         }
     }
 
+    /**
+     * Calls {@link ValueNode#inferStamp()} on the node and, if it returns true (which means that the stamp has
+     * changed), re-queues the node's usages . If the stamp has changed then this method also checks if the stamp
+     * now describes a constant integer value, in which case the node is replaced with a constant.
+     */
+    private void tryInferStamp(Node node, StructuredGraph graph) {
+        if (node.isAlive() && node instanceof ValueNode) {
+            ValueNode valueNode = (ValueNode) node;
+            METRIC_INFER_STAMP_CALLED.increment();
+            if (valueNode.inferStamp()) {
+                METRIC_STAMP_CHANGED.increment();
+                if (valueNode.stamp() instanceof IntegerStamp && valueNode.integerStamp().lowerBound() == valueNode.integerStamp().upperBound()) {
+                    ValueNode replacement = ConstantNode.forIntegerKind(valueNode.kind(), valueNode.integerStamp().lowerBound(), graph);
+                    Debug.log("Canonicalizer: replacing %s with %s (inferStamp)", valueNode, replacement);
+                    valueNode.replaceAtUsages(replacement);
+                } else {
+                    for (Node usage : valueNode.usages()) {
+                        workList.addAgain(usage);
+                    }
+                }
+            }
+        }
+    }
+
     private static final class Tool implements SimplifierTool {
 
         private final NodeWorkList nodeWorkSet;
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/phases/InliningPhase.java
--- a/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/phases/InliningPhase.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/phases/InliningPhase.java	Tue Jun 26 10:56:03 2012 +0200
@@ -91,12 +91,17 @@
         }
 
         while (!inlineCandidates.isEmpty() && graph.getNodeCount() < GraalOptions.MaximumDesiredSize) {
-            final InlineInfo info = inlineCandidates.remove();
+            InlineInfo candidate = inlineCandidates.remove();
+            if (!candidate.invoke.node().isAlive()) {
+                continue;
+            }
+            // refresh infos
+            final InlineInfo info = InliningUtil.getInlineInfo(candidate.invoke, candidate.level, runtime, assumptions, this, optimisticOpts);
 
             boolean inline = Debug.scope("InliningDecisions", new Callable<Boolean>() {
                 @Override
                 public Boolean call() throws Exception {
-                    return info.invoke.node().isAlive() && inliningPolicy.isWorthInlining(graph, info);
+                    return info != null && inliningPolicy.isWorthInlining(graph, info);
                 }
             });
 
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/phases/InsertStateAfterPlaceholderPhase.java
--- a/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/phases/InsertStateAfterPlaceholderPhase.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/phases/InsertStateAfterPlaceholderPhase.java	Tue Jun 26 10:56:03 2012 +0200
@@ -29,7 +29,7 @@
 
 public class InsertStateAfterPlaceholderPhase extends Phase {
 
-    private static class PlaceholderNode extends AbstractStateSplit implements StateSplit, Node.IterableNodeType, LIRLowerable {
+    private static class PlaceholderNode extends AbstractStateSplit implements StateSplit, Node.IterableNodeType, LIRLowerable, Canonicalizable {
         public PlaceholderNode() {
             super(StampFactory.forVoid());
         }
@@ -43,6 +43,14 @@
         public boolean hasSideEffect() {
             return false;
         }
+
+        @Override
+        public ValueNode canonical(CanonicalizerTool tool) {
+            if (stateAfter() == null) {
+                return null;
+            }
+            return this;
+        }
     }
 
     @Override
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/phases/LoopFullUnrollPhase.java
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/phases/LoopFullUnrollPhase.java	Tue Jun 26 10:56:03 2012 +0200
@@ -0,0 +1,61 @@
+/*
+ * Copyright (c) 2012, 2012, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.graal.compiler.phases;
+
+import com.oracle.graal.compiler.loop.*;
+import com.oracle.graal.cri.*;
+import com.oracle.graal.debug.*;
+import com.oracle.graal.nodes.*;
+
+
+public class LoopFullUnrollPhase extends Phase {
+    private static final DebugMetric FULLY_UNROLLED_LOOPS = Debug.metric("FullUnrolls");
+    private final ExtendedRiRuntime runtime;
+
+    public LoopFullUnrollPhase(ExtendedRiRuntime runtime) {
+        this.runtime = runtime;
+    }
+
+    @Override
+    protected void run(StructuredGraph graph) {
+        if (graph.hasLoops()) {
+            boolean peeled;
+            do {
+                peeled = false;
+                final LoopsData dataCounted = new LoopsData(graph);
+                dataCounted.detectedCountedLoops();
+                for (final LoopEx loop : dataCounted.countedLoops()) {
+                    if (LoopPolicies.shouldFullUnroll(loop)) {
+                        Debug.log("FullUnroll %s", loop);
+                        LoopTransformations.fullUnroll(loop, runtime);
+                        FULLY_UNROLLED_LOOPS.increment();
+                        Debug.dump(graph, "After fullUnroll %s", loop);
+                        peeled = true;
+                        break;
+                    }
+                }
+            } while(peeled);
+        }
+    }
+
+}
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/phases/LoopTransformHighPhase.java
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/phases/LoopTransformHighPhase.java	Tue Jun 26 10:56:03 2012 +0200
@@ -0,0 +1,44 @@
+/*
+ * Copyright (c) 2012, 2012, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.graal.compiler.phases;
+
+import com.oracle.graal.compiler.loop.*;
+import com.oracle.graal.debug.*;
+import com.oracle.graal.nodes.*;
+
+public class LoopTransformHighPhase extends Phase {
+
+    @Override
+    protected void run(StructuredGraph graph) {
+        if (graph.hasLoops()) {
+            LoopsData data = new LoopsData(graph);
+            for (LoopEx loop : data.outterFirst()) {
+                if (LoopPolicies.shouldPeel(loop)) {
+                    Debug.log("Peeling %s", loop);
+                    LoopTransformations.peel(loop);
+                    Debug.dump(graph, "After peeling %s", loop);
+                }
+            }
+        }
+    }
+}
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/phases/LoopTransformLowPhase.java
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/phases/LoopTransformLowPhase.java	Tue Jun 26 10:56:03 2012 +0200
@@ -0,0 +1,48 @@
+/*
+ * Copyright (c) 2012, 2012, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.graal.compiler.phases;
+
+import com.oracle.graal.compiler.*;
+import com.oracle.graal.compiler.loop.*;
+import com.oracle.graal.debug.*;
+import com.oracle.graal.nodes.*;
+
+public class LoopTransformLowPhase extends Phase {
+
+    @Override
+    protected void run(StructuredGraph graph) {
+        if (graph.hasLoops()) {
+            if (GraalOptions.ReassociateInvariants) {
+                final LoopsData dataReassociate = new LoopsData(graph);
+                Debug.scope("ReassociateInvariants", new Runnable() {
+                    @Override
+                    public void run() {
+                        for (LoopEx loop : dataReassociate.loops()) {
+                            loop.reassociateInvariants();
+                        }
+                    }
+                });
+            }
+        }
+    }
+}
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/phases/LoweringPhase.java
--- a/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/phases/LoweringPhase.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/phases/LoweringPhase.java	Tue Jun 26 10:56:03 2012 +0200
@@ -95,7 +95,7 @@
             if (graph.getNewNodes(mark).filter(FixedNode.class).isEmpty()) {
                 break;
             }
-            graph.verify();
+            assert graph.verify();
             processed.grow();
         }
 
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/phases/PhiStampPhase.java
--- a/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/phases/PhiStampPhase.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/phases/PhiStampPhase.java	Tue Jun 26 10:56:03 2012 +0200
@@ -43,7 +43,7 @@
     }
 
     private void iterativeInferPhi(PhiNode phi) {
-        if (phi.inferStamp()) {
+        if (phi.inferPhiStamp()) {
             for (PhiNode phiUsage : phi.usages().filter(PhiNode.class)) {
                 iterativeInferPhi(phiUsage);
             }
@@ -56,6 +56,6 @@
                 inferPhi(phiInput);
             }
         }
-        phi.inferStamp();
+        phi.inferPhiStamp();
     }
 }
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/target/amd64/AMD64LIRGenerator.java
--- a/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/target/amd64/AMD64LIRGenerator.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/target/amd64/AMD64LIRGenerator.java	Tue Jun 26 10:56:03 2012 +0200
@@ -137,13 +137,18 @@
         Value base = operand(object);
         Value index = Value.IllegalValue;
         int scale = 1;
-        long displacement = location.displacement();
+        int displacement = location.displacement();
 
         if (isConstant(base)) {
-            if (!asConstant(base).isNull()) {
-                displacement += asConstant(base).asLong();
+            if (asConstant(base).isNull()) {
+                base = Value.IllegalValue;
+            } else if (asConstant(base).kind != Kind.Object) {
+                long newDisplacement = displacement + asConstant(base).asLong();
+                if (NumUtil.isInt(newDisplacement)) {
+                    displacement = (int) newDisplacement;
+                    base = Value.IllegalValue;
+                }
             }
-            base = Value.IllegalValue;
         }
 
         if (location instanceof IndexedLocationNode) {
@@ -157,7 +162,7 @@
                 long newDisplacement = displacement + asConstant(index).asLong() * scale;
                 // only use the constant index if the resulting displacement fits into a 32 bit offset
                 if (NumUtil.isInt(newDisplacement)) {
-                    displacement = newDisplacement;
+                    displacement = (int) newDisplacement;
                     index = Value.IllegalValue;
                 } else {
                     // create a temporary variable for the index, the pointer load cannot handle a constant index
@@ -168,7 +173,7 @@
             }
         }
 
-        return new Address(location.getValueKind(), base, index, Address.Scale.fromInt(scale), (int) displacement);
+        return new Address(location.getValueKind(), base, index, Address.Scale.fromInt(scale), displacement);
     }
 
     @Override
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/util/InliningUtil.java
--- a/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/util/InliningUtil.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.compiler/src/com/oracle/graal/compiler/util/InliningUtil.java	Tue Jun 26 10:56:03 2012 +0200
@@ -35,6 +35,7 @@
 import com.oracle.graal.debug.*;
 import com.oracle.graal.graph.*;
 import com.oracle.graal.nodes.*;
+import com.oracle.graal.nodes.FrameState.*;
 import com.oracle.graal.nodes.calc.*;
 import com.oracle.graal.nodes.extended.*;
 import com.oracle.graal.nodes.java.*;
@@ -190,7 +191,7 @@
             ReadHubNode objectClass = graph.add(new ReadHubNode(receiver));
             IsTypeNode isTypeNode = graph.unique(new IsTypeNode(objectClass, type));
             FixedGuardNode guard = graph.add(new FixedGuardNode(isTypeNode, DeoptimizationReason.TypeCheckedInliningViolated, DeoptimizationAction.InvalidateReprofile, invoke.leafGraphId()));
-            AnchorNode anchor = graph.add(new AnchorNode());
+            ValueAnchorNode anchor = graph.add(new ValueAnchorNode());
             assert invoke.predecessor() != null;
 
             ValueNode anchoredReceiver = createAnchoredReceiver(graph, anchor, type, receiver);
@@ -397,41 +398,28 @@
         private FixedNode createDispatchOnType(StructuredGraph graph, ReadHubNode objectClassNode, BeginNode[] calleeEntryNodes, FixedNode unknownTypeSux) {
             assert ptypes.length > 1;
 
-            int lastIndex = ptypes.length - 1;
-            double[] branchProbabilities = convertTypeToBranchProbabilities(ptypes, notRecordedTypeProbability);
-            double nodeProbability = ptypes[lastIndex].probability;
-            IfNode nextNode = createTypeCheck(graph, objectClassNode, ptypes[lastIndex].type, calleeEntryNodes[typesToConcretes[lastIndex]], unknownTypeSux, branchProbabilities[lastIndex], invoke.probability() * nodeProbability);
-            for (int i = lastIndex - 1; i >= 0; i--) {
-                nodeProbability += ptypes[i].probability;
-                nextNode = createTypeCheck(graph, objectClassNode, ptypes[i].type, calleeEntryNodes[typesToConcretes[i]], nextNode, branchProbabilities[i], invoke.probability() * nodeProbability);
-            }
-
-            return nextNode;
-        }
+            ResolvedJavaType[] types = new ResolvedJavaType[ptypes.length];
+            double[] probabilities = new double[ptypes.length + 1];
+            BeginNode[] successors = new BeginNode[ptypes.length + 1];
 
-        private static IfNode createTypeCheck(StructuredGraph graph, ReadHubNode objectClassNode, ResolvedJavaType type, BeginNode tsux, FixedNode nextNode, double tsuxProbability, double probability) {
-            IfNode result;
-            IsTypeNode isTypeNode = graph.unique(new IsTypeNode(objectClassNode, type));
-            if (tsux instanceof MergeNode) {
-                EndNode endNode = graph.add(new EndNode());
-                result = graph.add(new IfNode(isTypeNode, endNode, nextNode, tsuxProbability));
-                ((MergeNode) tsux).addForwardEnd(endNode);
-            } else {
-                result = graph.add(new IfNode(isTypeNode, tsux, nextNode, tsuxProbability));
+            for (int i = 0; i < ptypes.length; i++) {
+                types[i] = ptypes[i].type;
+                probabilities[i] = ptypes[i].probability;
+                FixedNode entry = calleeEntryNodes[typesToConcretes[i]];
+                if (entry instanceof MergeNode) {
+                    EndNode endNode = graph.add(new EndNode());
+                    ((MergeNode) entry).addForwardEnd(endNode);
+                    entry = endNode;
+                }
+                successors[i] = BeginNode.begin(entry);
             }
-            result.setProbability(probability);
-            return result;
-        }
+            assert !(unknownTypeSux instanceof MergeNode);
+            successors[successors.length - 1] = BeginNode.begin(unknownTypeSux);
+            probabilities[successors.length - 1] = notRecordedTypeProbability;
 
-        private static double[] convertTypeToBranchProbabilities(ProfiledType[] ptypes, double notRecordedTypeProbability) {
-            double[] result = new double[ptypes.length];
-            double total = notRecordedTypeProbability;
-            for (int i = ptypes.length - 1; i >= 0; i--) {
-                total += ptypes[i].probability;
-                result[i] = ptypes[i].probability / total;
-            }
-            assert total > 0.99 && total < 1.01;
-            return result;
+            TypeSwitchNode typeSwitch = graph.add(new TypeSwitchNode(objectClassNode, successors, types, probabilities));
+
+            return typeSwitch;
         }
 
         private static BeginNode createInvocationBlock(StructuredGraph graph, Invoke invoke, MergeNode returnMerge, PhiNode returnValuePhi,
@@ -771,6 +759,7 @@
      * @param receiverNullCheck true if a null check needs to be generated for non-static inlinings, false if no such check is required
      */
     public static void inline(Invoke invoke, StructuredGraph inlineGraph, boolean receiverNullCheck) {
+        InliningIdentifier identifier = new InliningIdentifier(inlineGraph.method(), invoke.toString());
         NodeInputList<ValueNode> parameters = invoke.callTarget().arguments();
         StructuredGraph graph = (StructuredGraph) invoke.node().graph();
 
@@ -874,6 +863,7 @@
                         outerFrameState.setDuringCall(true);
                     }
                     frameState.setOuterFrameState(outerFrameState);
+                    frameState.setInliningIdentifier(identifier);
                 }
             }
         }
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.graph/src/com/oracle/graal/graph/iterators/DistinctFilteredNodeIterable.java
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/graal/com.oracle.graal.graph/src/com/oracle/graal/graph/iterators/DistinctFilteredNodeIterable.java	Tue Jun 26 10:56:03 2012 +0200
@@ -0,0 +1,45 @@
+/*
+ * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.graal.graph.iterators;
+
+import java.util.*;
+
+import com.oracle.graal.graph.*;
+
+
+public class DistinctFilteredNodeIterable<T extends Node> extends FilteredNodeIterable<T> {
+
+    public DistinctFilteredNodeIterable(NodeIterable<T> nodeIterable) {
+        super(nodeIterable);
+    }
+
+    @Override
+    public DistinctFilteredNodeIterable<T> distinct() {
+        return this;
+    }
+
+    @Override
+    public Iterator<T> iterator() {
+        return new DistinctPredicatedProxyNodeIterator<>(until, nodeIterable.iterator(), predicate);
+    }
+}
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.graph/src/com/oracle/graal/graph/iterators/FilteredNodeIterable.java
--- a/graal/com.oracle.graal.graph/src/com/oracle/graal/graph/iterators/FilteredNodeIterable.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.graph/src/com/oracle/graal/graph/iterators/FilteredNodeIterable.java	Tue Jun 26 10:56:03 2012 +0200
@@ -27,10 +27,9 @@
 import com.oracle.graal.graph.*;
 
 public class FilteredNodeIterable<T extends Node> extends AbstractNodeIterable<T> {
-    private final NodeIterable<T> nodeIterable;
-    private NodePredicate predicate = NodePredicates.alwaysTrue();
-    private NodePredicate until = NodePredicates.isNull();
-    private boolean distinct;
+    protected final NodeIterable<T> nodeIterable;
+    protected NodePredicate predicate = NodePredicates.alwaysTrue();
+    protected NodePredicate until = NodePredicates.isNull();
     public FilteredNodeIterable(NodeIterable<T> nodeIterable) {
         this.nodeIterable = nodeIterable;
     }
@@ -58,18 +57,15 @@
         return this;
     }
     @Override
-    public FilteredNodeIterable<T> distinct() {
-        distinct = true;
-        return this;
+    public DistinctFilteredNodeIterable<T> distinct() {
+        DistinctFilteredNodeIterable<T> distinct = new DistinctFilteredNodeIterable<>(nodeIterable);
+        distinct.predicate = predicate;
+        distinct.until = until;
+        return distinct;
     }
     @Override
     public Iterator<T> iterator() {
-        final Iterator<T> iterator = nodeIterable.iterator();
-        if (distinct) {
-            return new DistinctPredicatedProxyNodeIterator<>(until, iterator, predicate);
-        } else {
-            return new PredicatedProxyNodeIterator<>(until, iterator, predicate);
-        }
+        return new PredicatedProxyNodeIterator<>(until, nodeIterable.iterator(), predicate);
     }
 
     @SuppressWarnings("unchecked")
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/HotSpotKlassOop.java
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/HotSpotKlassOop.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/HotSpotKlassOop.java	Tue Jun 26 10:56:03 2012 +0200
@@ -22,6 +22,8 @@
  */
 package com.oracle.graal.hotspot;
 
+import com.oracle.graal.api.meta.*;
+
 /**
  * A mechanism for safely conveying a HotSpot klassOop value from the compiler to the C++ code.
  * Such values should not be directly exposed to Java code as they are not real Java
@@ -32,17 +34,14 @@
 
     private static final long serialVersionUID = -5445542223575839143L;
 
-    /**
-     * The Java object from which the klassOop value can be derived (by the C++ code).
-     */
-    public final Class javaMirror;
+    public final ResolvedJavaType type;
 
-    public HotSpotKlassOop(Class javaMirror) {
-        this.javaMirror = javaMirror;
+    public HotSpotKlassOop(ResolvedJavaType type) {
+        this.type = type;
     }
 
     @Override
     public String toString() {
-        return "HotSpotKlassOop<" + javaMirror.getName() + ">";
+        return "HotSpotKlassOop<" + type.toJava().getName() + ">";
     }
 }
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/bridge/VMToCompilerImpl.java
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/bridge/VMToCompilerImpl.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/bridge/VMToCompilerImpl.java	Tue Jun 26 10:56:03 2012 +0200
@@ -232,8 +232,8 @@
         if (queue != null) {
             queue.shutdown();
             if (Debug.isEnabled() && GraalOptions.Dump != null) {
-                // Wait 5 seconds to try and flush out all graph dumps
-                queue.awaitTermination(5, TimeUnit.SECONDS);
+                // Wait 2 seconds to flush out all graph dumps that may be of interest
+                queue.awaitTermination(2, TimeUnit.SECONDS);
             }
         }
     }
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/HotSpotResolvedJavaType.java
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/HotSpotResolvedJavaType.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/HotSpotResolvedJavaType.java	Tue Jun 26 10:56:03 2012 +0200
@@ -27,6 +27,7 @@
 import java.util.*;
 
 import com.oracle.graal.api.meta.*;
+import com.oracle.graal.graph.*;
 import com.oracle.graal.hotspot.*;
 
 /**
@@ -53,8 +54,10 @@
     private ConstantPool constantPool;
     private boolean isInitialized;
     private ResolvedJavaType arrayOfType;
+    private long prototypeHeader;
 
     private HotSpotResolvedJavaType() {
+        throw new GraalInternalError(HotSpotResolvedJavaType.class + " should only be created from C++ code");
     }
 
     @Override
@@ -267,7 +270,7 @@
     @Override
     public synchronized HotSpotKlassOop klassOop() {
         if (klassOopCache == null) {
-            klassOopCache = new HotSpotKlassOop(javaMirror);
+            klassOopCache = new HotSpotKlassOop(this);
         }
         return klassOopCache;
     }
@@ -281,4 +284,8 @@
     public int superCheckOffset() {
         return superCheckOffset;
     }
+
+    public long prototypeHeader() {
+        return prototypeHeader;
+    }
 }
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/HotSpotRuntime.java
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/HotSpotRuntime.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/HotSpotRuntime.java	Tue Jun 26 10:56:03 2012 +0200
@@ -229,7 +229,6 @@
     @Override
     public void lower(Node n, CiLoweringTool tool) {
         StructuredGraph graph = (StructuredGraph) n.graph();
-
         if (n instanceof ArrayLengthNode) {
             ArrayLengthNode arrayLengthNode = (ArrayLengthNode) n;
             SafeReadNode safeReadArrayLength = safeReadArrayLength(arrayLengthNode.array(), StructuredGraph.INVALID_GRAPH_ID);
@@ -367,6 +366,10 @@
             if (shouldLower(graph, GraalOptions.HIRLowerNewInstance)) {
                 newInstanceSnippets.lower((NewInstanceNode) n, tool);
             }
+        } else if (n instanceof TLABAllocateNode) {
+            newInstanceSnippets.lower((TLABAllocateNode) n, tool);
+        } else if (n instanceof InitializeNode) {
+            newInstanceSnippets.lower((InitializeNode) n, tool);
         } else {
             assert false : "Node implementing Lowerable not handled: " + n;
         }
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/HotSpotTypePrimitive.java
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/HotSpotTypePrimitive.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/HotSpotTypePrimitive.java	Tue Jun 26 10:56:03 2012 +0200
@@ -41,7 +41,7 @@
     public HotSpotTypePrimitive(Kind kind) {
         this.kind = kind;
         this.name = String.valueOf(Character.toUpperCase(kind.typeChar));
-        this.klassOop = new HotSpotKlassOop(kind.toJavaClass());
+        this.klassOop = new HotSpotKlassOop(this);
     }
 
     @Override
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/nodes/CastFromHub.java
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/nodes/CastFromHub.java	Tue Jun 26 10:56:03 2012 +0200
@@ -0,0 +1,69 @@
+/*
+ * Copyright (c) 2009, 2011, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.graal.hotspot.nodes;
+
+import com.oracle.graal.api.meta.*;
+import com.oracle.graal.hotspot.*;
+import com.oracle.graal.hotspot.snippets.*;
+import com.oracle.graal.nodes.*;
+import com.oracle.graal.nodes.calc.*;
+import com.oracle.graal.nodes.extended.*;
+import com.oracle.graal.nodes.spi.*;
+import com.oracle.graal.nodes.type.*;
+
+/**
+ * This node is used by the {@link NewInstanceSnippets} to give a formatted new instance its exact type.
+ */
+public final class CastFromHub extends FloatingNode implements Canonicalizable {
+
+    @Input private ValueNode object;
+    @Input private ValueNode hub;
+
+    public ValueNode object() {
+        return object;
+    }
+
+    public CastFromHub(ValueNode object, ValueNode hubObject) {
+        // TODO: the non-nullness should really be derived from 'object' but until
+        // control flow sensitive type analysis is implemented, the object coming
+        // from the TLAB fast path is not non-null
+        super(StampFactory.objectNonNull());
+        this.object = object;
+        this.hub = hubObject;
+    }
+
+    @Override
+    public ValueNode canonical(CanonicalizerTool tool) {
+        if (hub.isConstant()) {
+            ResolvedJavaType type = ((HotSpotKlassOop) this.hub.asConstant().asObject()).type;
+            return graph().unique(new UnsafeCastNode(object, type, true, true));
+        }
+        return this;
+    }
+
+    @SuppressWarnings("unused")
+    @NodeIntrinsic
+    public static <T> T castFromHub(Object object, Object hub) {
+        throw new UnsupportedOperationException("This method may only be compiled with the Graal compiler");
+    }
+}
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/nodes/InitializeNode.java
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/nodes/InitializeNode.java	Tue Jun 26 10:56:03 2012 +0200
@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.graal.hotspot.nodes;
+
+import com.oracle.graal.api.meta.*;
+import com.oracle.graal.cri.*;
+import com.oracle.graal.nodes.*;
+import com.oracle.graal.nodes.spi.*;
+import com.oracle.graal.nodes.type.*;
+
+/**
+ * Initializes the header and body of an uninitialized object cell.
+ * This node calls out to a stub to do both the allocation and formatting
+ * if the memory address it is given is zero/null (e.g. due to
+ * {@linkplain TLABAllocateNode TLAB allocation} failing).
+ */
+public final class InitializeNode extends FixedWithNextNode implements Lowerable {
+
+    @Input private final ValueNode memory;
+    private final ResolvedJavaType type;
+
+    public InitializeNode(ValueNode memory, ResolvedJavaType type) {
+        super(StampFactory.exactNonNull(type));
+        this.memory = memory;
+        this.type = type;
+    }
+
+    public ValueNode memory() {
+        return memory;
+    }
+
+    public ResolvedJavaType type() {
+        return type;
+    }
+
+    @Override
+    public void lower(CiLoweringTool tool) {
+        tool.getRuntime().lower(this, tool);
+    }
+
+    @SuppressWarnings("unused")
+    @NodeIntrinsic
+    public static Object initialize(Object memory, @ConstantNodeParameter ResolvedJavaType type) {
+        throw new UnsupportedOperationException();
+    }
+}
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/nodes/NewInstanceStubCall.java
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/nodes/NewInstanceStubCall.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/nodes/NewInstanceStubCall.java	Tue Jun 26 10:56:03 2012 +0200
@@ -25,6 +25,7 @@
 import com.oracle.graal.api.meta.*;
 import com.oracle.graal.compiler.gen.*;
 import com.oracle.graal.compiler.target.*;
+import com.oracle.graal.hotspot.*;
 import com.oracle.graal.hotspot.target.*;
 import com.oracle.graal.lir.*;
 import com.oracle.graal.nodes.*;
@@ -38,14 +39,26 @@
  */
 public class NewInstanceStubCall extends FixedWithNextNode implements LIRGenLowerable {
 
+    private static final Stamp defaultStamp = StampFactory.objectNonNull();
+
     @Input private final ValueNode hub;
 
     public NewInstanceStubCall(ValueNode hub) {
-        super(StampFactory.objectNonNull());
+        super(defaultStamp);
         this.hub = hub;
     }
 
     @Override
+    public boolean inferStamp() {
+        if (stamp() == defaultStamp && hub.isConstant()) {
+            HotSpotKlassOop klassOop = (HotSpotKlassOop) this.hub.asConstant().asObject();
+            updateStamp(StampFactory.exactNonNull(klassOop.type));
+            return true;
+        }
+        return false;
+    }
+
+    @Override
     public void generate(LIRGenerator gen) {
         Variable result = gen.newVariable(Kind.Object);
         gen.emitMove(gen.operand(hub), AMD64.rdx.asValue(Kind.Object));
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/nodes/RegisterNode.java
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/nodes/RegisterNode.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/nodes/RegisterNode.java	Tue Jun 26 10:56:03 2012 +0200
@@ -44,8 +44,15 @@
 
     @Override
     public void generate(LIRGeneratorTool generator) {
-        Value result = generator.newVariable(kind());
-        generator.emitMove(register.asValue(kind()), result);
+        Value result;
+        if (generator.attributes(register).isAllocatable()) {
+            // The register allocator would prefer us not to tie up an allocatable
+            // register for the complete lifetime of this node.
+            result = generator.newVariable(kind());
+            generator.emitMove(register.asValue(kind()), result);
+        } else {
+            result = register.asValue(kind());
+        }
         generator.setResult(this, result);
     }
 
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/nodes/TLABAllocateNode.java
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/nodes/TLABAllocateNode.java	Tue Jun 26 10:56:03 2012 +0200
@@ -0,0 +1,62 @@
+/*
+ * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.graal.hotspot.nodes;
+
+import com.oracle.graal.api.meta.*;
+import com.oracle.graal.cri.*;
+import com.oracle.graal.nodes.*;
+import com.oracle.graal.nodes.spi.*;
+import com.oracle.graal.nodes.type.*;
+import com.oracle.graal.snippets.*;
+
+/**
+ * Allocates some uninitialized area. This is used for TLAB allocation
+ * only. If allocation fails, zero/null is produced by this node.
+ */
+public final class TLABAllocateNode extends FixedWithNextNode implements Lowerable {
+
+    private final int size;
+
+    public TLABAllocateNode(int size, Kind wordKind) {
+        super(StampFactory.forWord(wordKind, true));
+        this.size = size;
+    }
+
+    public int size() {
+        return size;
+    }
+
+    @Override
+    public void lower(CiLoweringTool tool) {
+        tool.getRuntime().lower(this, tool);
+    }
+
+    /**
+     * @return null if allocation fails
+     */
+    @SuppressWarnings("unused")
+    @NodeIntrinsic
+    public static Word allocate(@ConstantNodeParameter int size, @ConstantNodeParameter Kind wordKind) {
+        throw new UnsupportedOperationException();
+    }
+}
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ArrayCopySnippets.java
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ArrayCopySnippets.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ArrayCopySnippets.java	Tue Jun 26 10:56:03 2012 +0200
@@ -28,11 +28,43 @@
 import com.oracle.graal.nodes.spi.*;
 import com.oracle.graal.nodes.type.*;
 import com.oracle.graal.snippets.*;
+import com.oracle.graal.snippets.Snippet.ConstantParameter;
 import com.oracle.graal.snippets.Snippet.Fold;
 
 
 @SuppressWarnings("unused")
 public class ArrayCopySnippets implements SnippetsInterface{
+    private static final Kind VECTOR_KIND = Kind.Long;
+    private static final long VECTOR_SIZE = arrayIndexScale(Kind.Long);
+
+    //@Snippet
+    public static void vectorizedCopy(Object src, int srcPos, Object dest, int destPos, int length, @ConstantParameter("baseKind") Kind baseKind) {
+        int header = arrayBaseOffset(baseKind);
+        long byteLength = length * arrayIndexScale(baseKind);
+        long nonVectorBytes = byteLength % VECTOR_SIZE;
+        long srcOffset = srcPos * arrayIndexScale(baseKind);
+        long destOffset = destPos * arrayIndexScale(baseKind);
+        if (src == dest && srcPos < destPos) { // bad aliased case
+            for (long i = byteLength - 1; i > byteLength - 1 - nonVectorBytes; i--) {
+                Byte a = UnsafeLoadNode.load(src, header, i + srcOffset, Kind.Byte);
+                UnsafeStoreNode.store(dest, header, i + destOffset, a.byteValue(), Kind.Byte);
+            }
+            long vectorLength = byteLength - nonVectorBytes;
+            for (long i = vectorLength - VECTOR_SIZE; i >= 0; i -= VECTOR_SIZE) {
+                Long a = UnsafeLoadNode.load(src, header, i + srcOffset, VECTOR_KIND);
+                UnsafeStoreNode.store(dest, header, i + destOffset, a.longValue(), VECTOR_KIND);
+            }
+        } else {
+            for (long i = 0; i < nonVectorBytes; i++) {
+                Byte a = UnsafeLoadNode.load(src, header, i + srcOffset, Kind.Byte);
+                UnsafeStoreNode.store(dest, header, i + destOffset, a.byteValue(), Kind.Byte);
+            }
+            for (long i = nonVectorBytes; i < byteLength; i += VECTOR_SIZE) {
+                Long a = UnsafeLoadNode.load(src, header, i + srcOffset, VECTOR_KIND);
+                UnsafeStoreNode.store(dest, header, i + destOffset, a.longValue(), VECTOR_KIND);
+            }
+        }
+    }
 
     @Snippet
     public static void arraycopy(byte[] src, int srcPos, byte[] dest, int destPos, int length) {
@@ -42,34 +74,30 @@
         if (srcPos < 0 || destPos < 0 || length < 0 || srcPos + length > src.length || destPos + length > dest.length) {
             throw new IndexOutOfBoundsException();
         }
-
+        Kind baseKind = Kind.Byte;
+        int header = arrayBaseOffset(baseKind);
+        long byteLength = length * arrayIndexScale(baseKind);
+        long nonVectorBytes = byteLength % VECTOR_SIZE;
+        long srcOffset = srcPos * arrayIndexScale(baseKind);
+        long destOffset = destPos * arrayIndexScale(baseKind);
         if (src == dest && srcPos < destPos) { // bad aliased case
-            if ((length & 0x01) == 0) {
-                if ((length & 0x02) == 0) {
-                    if ((length & 0x04) == 0) {
-                        copyLongsDown(src, srcPos, dest, destPos, length >> 3);
-                    } else {
-                        copyIntsDown(src, srcPos, dest, destPos, length >> 2);
-                    }
-                } else {
-                    copyShortsDown(src, srcPos, dest, destPos, length >> 1);
-                }
-            } else {
-                copyBytesDown(src, srcPos, dest, destPos, length);
+            for (long i = byteLength - 1; i > byteLength - 1 - nonVectorBytes; i--) {
+                Byte a = UnsafeLoadNode.load(src, header, i + srcOffset, Kind.Byte);
+                UnsafeStoreNode.store(dest, header, i + destOffset, a.byteValue(), Kind.Byte);
+            }
+            long vectorLength = byteLength - nonVectorBytes;
+            for (long i = vectorLength - VECTOR_SIZE; i >= 0; i -= VECTOR_SIZE) {
+                Long a = UnsafeLoadNode.load(src, header, i + srcOffset, VECTOR_KIND);
+                UnsafeStoreNode.store(dest, header, i + destOffset, a.longValue(), VECTOR_KIND);
             }
         } else {
-            if ((length & 0x01) == 0) {
-                if ((length & 0x02) == 0) {
-                    if ((length & 0x04) == 0) {
-                        copyLongsUp(src, srcPos, dest, destPos, length >> 3);
-                    } else {
-                        copyIntsUp(src, srcPos, dest, destPos, length >> 2);
-                    }
-                } else {
-                    copyShortsUp(src, srcPos, dest, destPos, length >> 1);
-                }
-            } else {
-                copyBytesUp(src, srcPos, dest, destPos, length);
+            for (long i = 0; i < nonVectorBytes; i++) {
+                Byte a = UnsafeLoadNode.load(src, header, i + srcOffset, Kind.Byte);
+                UnsafeStoreNode.store(dest, header, i + destOffset, a.byteValue(), Kind.Byte);
+            }
+            for (long i = nonVectorBytes; i < byteLength; i += VECTOR_SIZE) {
+                Long a = UnsafeLoadNode.load(src, header, i + srcOffset, VECTOR_KIND);
+                UnsafeStoreNode.store(dest, header, i + destOffset, a.longValue(), VECTOR_KIND);
             }
         }
     }
@@ -82,25 +110,30 @@
         if (srcPos < 0 || destPos < 0 || length < 0 || srcPos + length > src.length || destPos + length > dest.length) {
             throw new IndexOutOfBoundsException();
         }
+        Kind baseKind = Kind.Char;
+        int header = arrayBaseOffset(baseKind);
+        long byteLength = length * arrayIndexScale(baseKind);
+        long nonVectorBytes = byteLength % VECTOR_SIZE;
+        long srcOffset = srcPos * arrayIndexScale(baseKind);
+        long destOffset = destPos * arrayIndexScale(baseKind);
         if (src == dest && srcPos < destPos) { // bad aliased case
-            if ((length & 0x01) == 0) {
-                if ((length & 0x02) == 0) {
-                    copyLongsDown(src, srcPos * 2L, dest, destPos * 2L, length >> 2);
-                } else {
-                    copyIntsDown(src, srcPos * 2L, dest, destPos * 2L, length >> 1);
-                }
-            } else {
-                copyShortsDown(src, srcPos * 2L, dest, destPos * 2L, length);
+            for (long i = byteLength - 1; i > byteLength - 1 - nonVectorBytes; i--) {
+                Byte a = UnsafeLoadNode.load(src, header, i + srcOffset, Kind.Byte);
+                UnsafeStoreNode.store(dest, header, i + destOffset, a.byteValue(), Kind.Byte);
+            }
+            long vectorLength = byteLength - nonVectorBytes;
+            for (long i = vectorLength - VECTOR_SIZE; i >= 0; i -= VECTOR_SIZE) {
+                Long a = UnsafeLoadNode.load(src, header, i + srcOffset, VECTOR_KIND);
+                UnsafeStoreNode.store(dest, header, i + destOffset, a.longValue(), VECTOR_KIND);
             }
         } else {
-            if ((length & 0x01) == 0) {
-                if ((length & 0x02) == 0) {
-                    copyLongsUp(src, srcPos * 2L, dest, destPos * 2L, length >> 2);
-                } else {
-                    copyIntsUp(src, srcPos * 2L, dest, destPos * 2L, length >> 1);
-                }
-            } else {
-                copyShortsUp(src, srcPos * 2L, dest, destPos * 2L, length);
+            for (long i = 0; i < nonVectorBytes; i++) {
+                Byte a = UnsafeLoadNode.load(src, header, i + srcOffset, Kind.Byte);
+                UnsafeStoreNode.store(dest, header, i + destOffset, a.byteValue(), Kind.Byte);
+            }
+            for (long i = nonVectorBytes; i < byteLength; i += VECTOR_SIZE) {
+                Long a = UnsafeLoadNode.load(src, header, i + srcOffset, VECTOR_KIND);
+                UnsafeStoreNode.store(dest, header, i + destOffset, a.longValue(), VECTOR_KIND);
             }
         }
     }
@@ -113,25 +146,30 @@
         if (srcPos < 0 || destPos < 0 || length < 0 || srcPos + length > src.length || destPos + length > dest.length) {
             throw new IndexOutOfBoundsException();
         }
+        Kind baseKind = Kind.Short;
+        int header = arrayBaseOffset(baseKind);
+        long byteLength = length * arrayIndexScale(baseKind);
+        long nonVectorBytes = byteLength % VECTOR_SIZE;
+        long srcOffset = srcPos * arrayIndexScale(baseKind);
+        long destOffset = destPos * arrayIndexScale(baseKind);
         if (src == dest && srcPos < destPos) { // bad aliased case
-            if ((length & 0x01) == 0) {
-                if ((length & 0x02) == 0) {
-                    copyLongsDown(src, srcPos * 2L, dest, destPos * 2L, length >> 2);
-                } else {
-                    copyIntsDown(src, srcPos * 2L, dest, destPos * 2L, length >> 1);
-                }
-            } else {
-                copyShortsDown(src, srcPos * 2L, dest, destPos * 2L, length);
+            for (long i = byteLength - 1; i > byteLength - 1 - nonVectorBytes; i--) {
+                Byte a = UnsafeLoadNode.load(src, header, i + srcOffset, Kind.Byte);
+                UnsafeStoreNode.store(dest, header, i + destOffset, a.byteValue(), Kind.Byte);
+            }
+            long vectorLength = byteLength - nonVectorBytes;
+            for (long i = vectorLength - VECTOR_SIZE; i >= 0; i -= VECTOR_SIZE) {
+                Long a = UnsafeLoadNode.load(src, header, i + srcOffset, VECTOR_KIND);
+                UnsafeStoreNode.store(dest, header, i + destOffset, a.longValue(), VECTOR_KIND);
             }
         } else {
-            if ((length & 0x01) == 0) {
-                if ((length & 0x02) == 0) {
-                    copyLongsUp(src, srcPos * 2L, dest, destPos * 2L, length >> 2);
-                } else {
-                    copyIntsUp(src, srcPos * 2L, dest, destPos * 2L, length >> 1);
-                }
-            } else {
-                copyShortsUp(src, srcPos * 2L, dest, destPos * 2L, length);
+            for (long i = 0; i < nonVectorBytes; i++) {
+                Byte a = UnsafeLoadNode.load(src, header, i + srcOffset, Kind.Byte);
+                UnsafeStoreNode.store(dest, header, i + destOffset, a.byteValue(), Kind.Byte);
+            }
+            for (long i = nonVectorBytes; i < byteLength; i += VECTOR_SIZE) {
+                Long a = UnsafeLoadNode.load(src, header, i + srcOffset, VECTOR_KIND);
+                UnsafeStoreNode.store(dest, header, i + destOffset, a.longValue(), VECTOR_KIND);
             }
         }
     }
@@ -144,17 +182,30 @@
         if (srcPos < 0 || destPos < 0 || length < 0 || srcPos + length > src.length || destPos + length > dest.length) {
             throw new IndexOutOfBoundsException();
         }
+        Kind baseKind = Kind.Int;
+        int header = arrayBaseOffset(baseKind);
+        long byteLength = length * arrayIndexScale(baseKind);
+        long nonVectorBytes = byteLength % VECTOR_SIZE;
+        long srcOffset = srcPos * arrayIndexScale(baseKind);
+        long destOffset = destPos * arrayIndexScale(baseKind);
         if (src == dest && srcPos < destPos) { // bad aliased case
-            if ((length & 0x01) == 0) {
-                copyLongsDown(src, srcPos * 4L, dest, destPos * 4L, length >> 1);
-            } else {
-                copyIntsDown(src, srcPos * 4L, dest, destPos * 4L, length);
+            for (long i = byteLength - 1; i > byteLength - 1 - nonVectorBytes; i--) {
+                Byte a = UnsafeLoadNode.load(src, header, i + srcOffset, Kind.Byte);
+                UnsafeStoreNode.store(dest, header, i + destOffset, a.byteValue(), Kind.Byte);
+            }
+            long vectorLength = byteLength - nonVectorBytes;
+            for (long i = vectorLength - VECTOR_SIZE; i >= 0; i -= VECTOR_SIZE) {
+                Long a = UnsafeLoadNode.load(src, header, i + srcOffset, VECTOR_KIND);
+                UnsafeStoreNode.store(dest, header, i + destOffset, a.longValue(), VECTOR_KIND);
             }
         } else {
-            if ((length & 0x01) == 0) {
-                copyLongsUp(src, srcPos * 4L, dest, destPos * 4L, length >> 1);
-            } else {
-                copyIntsUp(src, srcPos * 4L, dest, destPos * 4L, length);
+            for (long i = 0; i < nonVectorBytes; i++) {
+                Byte a = UnsafeLoadNode.load(src, header, i + srcOffset, Kind.Byte);
+                UnsafeStoreNode.store(dest, header, i + destOffset, a.byteValue(), Kind.Byte);
+            }
+            for (long i = nonVectorBytes; i < byteLength; i += VECTOR_SIZE) {
+                Long a = UnsafeLoadNode.load(src, header, i + srcOffset, VECTOR_KIND);
+                UnsafeStoreNode.store(dest, header, i + destOffset, a.longValue(), VECTOR_KIND);
             }
         }
     }
@@ -167,17 +218,30 @@
         if (srcPos < 0 || destPos < 0 || length < 0 || srcPos + length > src.length || destPos + length > dest.length) {
             throw new IndexOutOfBoundsException();
         }
+        Kind baseKind = Kind.Float;
+        int header = arrayBaseOffset(baseKind);
+        long byteLength = length * arrayIndexScale(baseKind);
+        long nonVectorBytes = byteLength % VECTOR_SIZE;
+        long srcOffset = srcPos * arrayIndexScale(baseKind);
+        long destOffset = destPos * arrayIndexScale(baseKind);
         if (src == dest && srcPos < destPos) { // bad aliased case
-            if ((length & 0x01) == 0) {
-                copyLongsDown(src, srcPos * 4L, dest, destPos * 4L, length >> 1);
-            } else {
-                copyIntsDown(src, srcPos * 4L, dest, destPos * 4L, length);
+            for (long i = byteLength - 1; i > byteLength - 1 - nonVectorBytes; i--) {
+                Byte a = UnsafeLoadNode.load(src, header, i + srcOffset, Kind.Byte);
+                UnsafeStoreNode.store(dest, header, i + destOffset, a.byteValue(), Kind.Byte);
+            }
+            long vectorLength = byteLength - nonVectorBytes;
+            for (long i = vectorLength - VECTOR_SIZE; i >= 0; i -= VECTOR_SIZE) {
+                Long a = UnsafeLoadNode.load(src, header, i + srcOffset, VECTOR_KIND);
+                UnsafeStoreNode.store(dest, header, i + destOffset, a.longValue(), VECTOR_KIND);
             }
         } else {
-            if ((length & 0x01) == 0) {
-                copyLongsUp(src, srcPos * 4L, dest, destPos * 4L, length >> 1);
-            } else {
-                copyIntsUp(src, srcPos * 4L, dest, destPos * 4L, length);
+            for (long i = 0; i < nonVectorBytes; i++) {
+                Byte a = UnsafeLoadNode.load(src, header, i + srcOffset, Kind.Byte);
+                UnsafeStoreNode.store(dest, header, i + destOffset, a.byteValue(), Kind.Byte);
+            }
+            for (long i = nonVectorBytes; i < byteLength; i += VECTOR_SIZE) {
+                Long a = UnsafeLoadNode.load(src, header, i + srcOffset, VECTOR_KIND);
+                UnsafeStoreNode.store(dest, header, i + destOffset, a.longValue(), VECTOR_KIND);
             }
         }
     }
@@ -190,10 +254,21 @@
         if (srcPos < 0 || destPos < 0 || length < 0 || srcPos + length > src.length || destPos + length > dest.length) {
             throw new IndexOutOfBoundsException();
         }
+        Kind baseKind = Kind.Long;
+        int header = arrayBaseOffset(baseKind);
+        long byteLength = length * arrayIndexScale(baseKind);
+        long srcOffset = srcPos * arrayIndexScale(baseKind);
+        long destOffset = destPos * arrayIndexScale(baseKind);
         if (src == dest && srcPos < destPos) { // bad aliased case
-            copyLongsDown(src, srcPos * 8L, dest, destPos * 8L, length);
+            for (long i = byteLength - VECTOR_SIZE; i >= 0; i -= VECTOR_SIZE) {
+                Long a = UnsafeLoadNode.load(src, header, i + srcOffset, VECTOR_KIND);
+                UnsafeStoreNode.store(dest, header, i + destOffset, a.longValue(), VECTOR_KIND);
+            }
         } else {
-            copyLongsUp(src, srcPos * 8L, dest, destPos * 8L, length);
+            for (long i = 0; i < byteLength; i += VECTOR_SIZE) {
+                Long a = UnsafeLoadNode.load(src, header, i + srcOffset, VECTOR_KIND);
+                UnsafeStoreNode.store(dest, header, i + destOffset, a.longValue(), VECTOR_KIND);
+            }
         }
     }
 
@@ -205,10 +280,21 @@
         if (srcPos < 0 || destPos < 0 || length < 0 || srcPos + length > src.length || destPos + length > dest.length) {
             throw new IndexOutOfBoundsException();
         }
+        Kind baseKind = Kind.Double;
+        int header = arrayBaseOffset(baseKind);
+        long byteLength = length * arrayIndexScale(baseKind);
+        long srcOffset = srcPos * arrayIndexScale(baseKind);
+        long destOffset = destPos * arrayIndexScale(baseKind);
         if (src == dest && srcPos < destPos) { // bad aliased case
-            copyLongsDown(src, srcPos * 8L, dest, destPos * 8L, length);
+            for (long i = byteLength - VECTOR_SIZE; i >= 0; i -= VECTOR_SIZE) {
+                Long a = UnsafeLoadNode.load(src, header, i + srcOffset, VECTOR_KIND);
+                UnsafeStoreNode.store(dest, header, i + destOffset, a.longValue(), VECTOR_KIND);
+            }
         } else {
-            copyLongsUp(src, srcPos * 8L, dest, destPos * 8L, length);
+            for (long i = 0; i < byteLength; i += VECTOR_SIZE) {
+                Long a = UnsafeLoadNode.load(src, header, i + srcOffset, VECTOR_KIND);
+                UnsafeStoreNode.store(dest, header, i + destOffset, a.longValue(), VECTOR_KIND);
+            }
         }
     }
 
@@ -241,42 +327,6 @@
         }
     }
 
-    @Snippet
-    public static void copyBytesDown(Object src, int srcPos, Object dest, int destPos, int length)  {
-        int header = arrayBaseOffset(Kind.Byte);
-        for (long i = length - 1; i >= 0; i--) {
-            Byte a = UnsafeLoadNode.load(src, header, i + srcPos, Kind.Byte);
-            UnsafeStoreNode.store(dest, header, i + destPos, a.byteValue(), Kind.Byte);
-        }
-    }
-
-    @Snippet
-    public static void copyShortsDown(Object src, long srcOffset, Object dest, long destOffset, int length)  {
-        int header = arrayBaseOffset(Kind.Short);
-        for (long i = (length - 1) * 2; i >= 0; i -= 2) {
-            Character a = UnsafeLoadNode.load(src, header, i + srcOffset, Kind.Short);
-            UnsafeStoreNode.store(dest, header, i + destOffset, a.charValue(), Kind.Short);
-        }
-    }
-
-    @Snippet
-    public static void copyIntsDown(Object src, long srcOffset, Object dest, long destOffset, int length)  {
-        int header = arrayBaseOffset(Kind.Int);
-        for (long i = (length - 1) * 4; i >= 0; i -= 4) {
-            Integer a = UnsafeLoadNode.load(src, header, i + srcOffset, Kind.Int);
-            UnsafeStoreNode.store(dest, header, i + destOffset, a.intValue(), Kind.Int);
-        }
-    }
-
-    @Snippet
-    public static void copyLongsDown(Object src, long srcOffset, Object dest, long destOffset, int length)  {
-        int header = arrayBaseOffset(Kind.Long);
-        for (long i = (length - 1) * 8; i >= 0; i -= 8) {
-            Long a = UnsafeLoadNode.load(src, header, i + srcOffset, Kind.Long);
-            UnsafeStoreNode.store(dest, header, i + destOffset, a.longValue(), Kind.Long);
-        }
-    }
-
     // Does NOT perform store checks
     @Snippet
     public static void copyObjectsDown(Object src, long srcOffset, Object dest, long destOffset, int length)  {
@@ -287,57 +337,6 @@
             DirectObjectStoreNode.store(dest, header, i + destOffset, a);
         }
     }
-    /**
-     * Copies {@code length} bytes from {@code src} starting at {@code srcPos} to {@code dest} starting at {@code destPos}.
-     * @param src source object
-     * @param srcPos source offset
-     * @param dest destination object
-     * @param destPos destination offset
-     * @param length number of bytes to copy
-     */
-    @Snippet
-    public static void copyBytesUp(Object src, int srcPos, Object dest, int destPos, int length)  {
-        int header = arrayBaseOffset(Kind.Byte);
-        for (long i = 0; i < length; i++) {
-            Byte a = UnsafeLoadNode.load(src, header, i + srcPos, Kind.Byte);
-            UnsafeStoreNode.store(dest, header, i + destPos, a.byteValue(), Kind.Byte);
-        }
-    }
-
-    /**
-     * Copies {@code length} shorts from {@code src} starting at offset {@code srcOffset} (in bytes) to {@code dest} starting at offset {@code destOffset} (in bytes).
-     * @param src
-     * @param srcOffset (in bytes)
-     * @param dest
-     * @param destOffset (in bytes)
-     * @param length  (in shorts)
-     */
-    @Snippet
-    public static void copyShortsUp(Object src, long srcOffset, Object dest, long destOffset, int length)  {
-        int header = arrayBaseOffset(Kind.Short);
-        for (long i = 0; i < length * 2L; i += 2) {
-            Character a = UnsafeLoadNode.load(src, header, i + srcOffset, Kind.Short);
-            UnsafeStoreNode.store(dest, header, i + destOffset, a.charValue(), Kind.Short);
-        }
-    }
-
-    @Snippet
-    public static void copyIntsUp(Object src, long srcOffset, Object dest, long destOffset, int length)  {
-        int header = arrayBaseOffset(Kind.Int);
-        for (long i = 0; i < length * 4L; i += 4) {
-            Integer a = UnsafeLoadNode.load(src, header, i + srcOffset, Kind.Int);
-            UnsafeStoreNode.store(dest, header, i + destOffset, a.intValue(), Kind.Int);
-        }
-    }
-
-    @Snippet
-    public static void copyLongsUp(Object src, long srcOffset, Object dest, long destOffset, int length)  {
-        int header = arrayBaseOffset(Kind.Long);
-        for (long i = 0; i < length * 8L; i += 8) {
-            Long a = UnsafeLoadNode.load(src, header, i + srcOffset, Kind.Long);
-            UnsafeStoreNode.store(dest, header, i + destOffset, a.longValue(), Kind.Long);
-        }
-    }
 
     // Does NOT perform store checks
     @Snippet
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/CheckCastSnippets.java
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/CheckCastSnippets.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/CheckCastSnippets.java	Tue Jun 26 10:56:03 2012 +0200
@@ -34,7 +34,6 @@
 import com.oracle.graal.api.code.*;
 import com.oracle.graal.api.meta.*;
 import com.oracle.graal.compiler.*;
-import com.oracle.graal.compiler.phases.*;
 import com.oracle.graal.cri.*;
 import com.oracle.graal.debug.*;
 import com.oracle.graal.graph.*;
@@ -385,7 +384,6 @@
             SnippetTemplate template = cache.get(key);
             Debug.log("Lowering checkcast in %s: node=%s, template=%s, arguments=%s", graph, checkcast, template, arguments);
             template.instantiate(runtime, checkcast, checkcast, arguments);
-            new DeadCodeEliminationPhase().apply(graph);
         }
 
         private static HotSpotKlassOop[] createHints(TypeCheckHints hints) {
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/NewInstanceSnippets.java
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/NewInstanceSnippets.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/NewInstanceSnippets.java	Tue Jun 26 10:56:03 2012 +0200
@@ -22,10 +22,9 @@
  */
 package com.oracle.graal.hotspot.snippets;
 
+import static com.oracle.graal.hotspot.nodes.CastFromHub.*;
 import static com.oracle.graal.hotspot.nodes.RegisterNode.*;
 import static com.oracle.graal.hotspot.snippets.DirectObjectStoreNode.*;
-import static com.oracle.graal.nodes.calc.Condition.*;
-import static com.oracle.graal.nodes.extended.UnsafeCastNode.*;
 import static com.oracle.graal.nodes.extended.UnsafeLoadNode.*;
 import static com.oracle.graal.snippets.SnippetTemplate.Arguments.*;
 import static com.oracle.graal.snippets.nodes.ExplodeLoopNode.*;
@@ -33,7 +32,6 @@
 
 import com.oracle.graal.api.code.*;
 import com.oracle.graal.api.meta.*;
-import com.oracle.graal.compiler.phases.*;
 import com.oracle.graal.cri.*;
 import com.oracle.graal.debug.*;
 import com.oracle.graal.graph.*;
@@ -55,50 +53,32 @@
  */
 public class NewInstanceSnippets implements SnippetsInterface {
 
-    private static final boolean LOG_ALLOCATION = Boolean.getBoolean("graal.traceAllocation");
+    @Snippet
+    public static Word allocate(@ConstantParameter("size") int size) {
+        Word thread = asWord(register(r15, wordKind()));
+        Word top = loadWord(thread, threadTlabTopOffset());
+        Word end = loadWord(thread, threadTlabEndOffset());
+        Word newTop = top.plus(size);
+        if (newTop.belowOrEqual(end)) {
+            store(thread, 0, threadTlabTopOffset(), newTop);
+            return top;
+        }
+        return Word.zero();
+    }
 
     @Snippet
-    public static Object newInstance(
+    public static Object initialize(
+                    @Parameter("memory") Word memory,
                     @Parameter("hub") Object hub,
-                    @ConstantParameter("size") int size,
-                    @ConstantParameter("checkInit") boolean checkInit,
-                    @ConstantParameter("useTLAB") boolean useTLAB,
-                    @ConstantParameter("logType") String logType) {
-
-        if (checkInit) {
-            int klassState = load(hub, 0, klassStateOffset(), Kind.Int);
-            if (klassState != klassStateFullyInitialized()) {
-                if (logType != null) {
-                    Log.print(logType);
-                    Log.println(" - uninit alloc");
-                }
-                return verifyOop(NewInstanceStubCall.call(hub));
-            }
-        }
+                    @Parameter("prototypeHeader") Word headerPrototype,
+                    @ConstantParameter("size") int size) {
 
-        if (useTLAB) {
-            Word thread = asWord(register(r15, wordKind()));
-            Word top = loadWord(thread, threadTlabTopOffset());
-            Word end = loadWord(thread, threadTlabEndOffset());
-            Word newTop = top.plus(size);
-            if (newTop.cmp(BE, end)) {
-                Object instance = cast(top, Object.class);
-                store(thread, 0, threadTlabTopOffset(), newTop);
-                formatInstance(hub, size, instance);
-                if (logType != null) {
-                    Log.print(logType);
-                    Log.print(" - fast alloc at ");
-                    Log.printlnAddress(instance);
-                }
-                return verifyOop(instance);
-            }
+        if (memory == Word.zero()) {
+            return NewInstanceStubCall.call(hub);
         }
-
-        if (logType != null) {
-            Log.print(logType);
-            Log.println(" - slow alloc");
-        }
-        return verifyOop(NewInstanceStubCall.call(hub));
+        formatObject(hub, size, memory, headerPrototype);
+        Object instance = memory.toObject();
+        return castFromHub(verifyOop(instance), hub);
     }
 
     private static Object verifyOop(Object object) {
@@ -109,23 +89,36 @@
     }
 
     private static Word asWord(Object object) {
-        return cast(object, Word.class);
+        return Word.fromObject(object);
     }
 
-    private static Word loadWord(Object object, int offset) {
-        return cast(load(object, 0, offset, wordKind()), Word.class);
+    private static Word loadWord(Word address, int offset) {
+        Object value = loadObject(address, 0, offset, true);
+        return asWord(value);
     }
 
     /**
-     * Formats the header of a created instance and zeroes out its body.
+     * Maximum size of an object whose body is initialized by a sequence of
+     * zero-stores to its fields. Larger objects have their bodies initialized
+     * in a loop.
+     */
+    private static final int MAX_UNROLLED_OBJECT_INITIALIZATION_SIZE = 10 * wordSize();
+
+    /**
+     * Formats some allocated memory with an object header zeroes out the rest.
      */
-    private static void formatInstance(Object hub, int size, Object instance) {
-        Word headerPrototype = cast(load(hub, 0, instanceHeaderPrototypeOffset(), wordKind()), Word.class);
-        store(instance, 0, 0, headerPrototype);
-        store(instance, 0, hubOffset(), hub);
-        explodeLoop();
-        for (int offset = 2 * wordSize(); offset < size; offset += wordSize()) {
-            store(instance, 0, offset, 0);
+    private static void formatObject(Object hub, int size, Word memory, Word headerPrototype) {
+        store(memory, 0, 0, headerPrototype);
+        store(memory, 0, hubOffset(), hub);
+        if (size <= MAX_UNROLLED_OBJECT_INITIALIZATION_SIZE) {
+            explodeLoop();
+            for (int offset = 2 * wordSize(); offset < size; offset += wordSize()) {
+                store(memory, 0, offset, 0);
+            }
+        } else {
+            for (int offset = 2 * wordSize(); offset < size; offset += wordSize()) {
+                store(memory, 0, offset, 0);
+            }
         }
     }
 
@@ -135,16 +128,6 @@
     }
 
     @Fold
-    private static int klassStateOffset() {
-        return HotSpotGraalRuntime.getInstance().getConfig().klassStateOffset;
-    }
-
-    @Fold
-    private static int klassStateFullyInitialized() {
-        return HotSpotGraalRuntime.getInstance().getConfig().klassStateFullyInitialized;
-    }
-
-    @Fold
     private static int threadTlabTopOffset() {
         return HotSpotGraalRuntime.getInstance().getConfig().threadTlabTopOffset;
     }
@@ -165,11 +148,6 @@
     }
 
     @Fold
-    private static int instanceHeaderPrototypeOffset() {
-        return HotSpotGraalRuntime.getInstance().getConfig().instanceHeaderPrototypeOffset;
-    }
-
-    @Fold
     private static int hubOffset() {
         return HotSpotGraalRuntime.getInstance().getConfig().hubOffset;
     }
@@ -177,7 +155,8 @@
     public static class Templates {
 
         private final Cache cache;
-        private final ResolvedJavaMethod newInstance;
+        private final ResolvedJavaMethod allocate;
+        private final ResolvedJavaMethod initialize;
         private final CodeCacheProvider runtime;
         private final boolean useTLAB;
 
@@ -186,7 +165,8 @@
             this.cache = new Cache(runtime);
             this.useTLAB = useTLAB;
             try {
-                newInstance = runtime.getResolvedJavaMethod(NewInstanceSnippets.class.getDeclaredMethod("newInstance", Object.class, int.class, boolean.class, boolean.class, String.class));
+                allocate = runtime.getResolvedJavaMethod(NewInstanceSnippets.class.getDeclaredMethod("allocate", int.class));
+                initialize = runtime.getResolvedJavaMethod(NewInstanceSnippets.class.getDeclaredMethod("initialize", Word.class, Object.class, Word.class, int.class));
             } catch (NoSuchMethodException e) {
                 throw new GraalInternalError(e);
             }
@@ -200,17 +180,50 @@
             StructuredGraph graph = (StructuredGraph) newInstanceNode.graph();
             HotSpotResolvedJavaType type = (HotSpotResolvedJavaType) newInstanceNode.instanceClass();
             HotSpotKlassOop hub = type.klassOop();
-            int instanceSize = type.instanceSize();
-            assert (instanceSize % wordSize()) == 0;
-            assert instanceSize >= 0;
-            Key key = new Key(newInstance).add("size", instanceSize).add("checkInit", !type.isInitialized()).add("useTLAB", useTLAB).add("logType", LOG_ALLOCATION ? type.name() : null);
-            Arguments arguments = arguments("hub", hub);
+            int size = type.instanceSize();
+            assert (size % wordSize()) == 0;
+            assert size >= 0;
+
+            ValueNode memory;
+            if (!useTLAB) {
+                memory = ConstantNode.forObject(null, runtime, graph);
+            } else {
+                TLABAllocateNode tlabAllocateNode = graph.add(new TLABAllocateNode(size, wordKind()));
+                graph.addBeforeFixed(newInstanceNode, tlabAllocateNode);
+                memory = tlabAllocateNode;
+            }
+            InitializeNode initializeNode = graph.add(new InitializeNode(memory, type));
+            graph.replaceFixedWithFixed(newInstanceNode, initializeNode);
+        }
+
+        @SuppressWarnings("unused")
+        public void lower(TLABAllocateNode tlabAllocateNode, CiLoweringTool tool) {
+            StructuredGraph graph = (StructuredGraph) tlabAllocateNode.graph();
+            int size = tlabAllocateNode.size();
+            assert (size % wordSize()) == 0;
+            assert size >= 0;
+            Key key = new Key(allocate).add("size", size);
+            Arguments arguments = new Arguments();
             SnippetTemplate template = cache.get(key);
-            Debug.log("Lowering newInstance in %s: node=%s, template=%s, arguments=%s", graph, newInstanceNode, template, arguments);
-            //System.out.printf("Lowering newInstance in %s: node=%s, template=%s, arguments=%s%n", graph, newInstanceNode, template, arguments);
-            //DebugScope.getConfig().addToContext(graph.method());
-            template.instantiate(runtime, newInstanceNode, newInstanceNode, arguments);
-            new DeadCodeEliminationPhase().apply(graph);
+            Debug.log("Lowering fastAllocate in %s: node=%s, template=%s, arguments=%s", graph, tlabAllocateNode, template, arguments);
+            template.instantiate(runtime, tlabAllocateNode, tlabAllocateNode, arguments);
+        }
+
+        @SuppressWarnings("unused")
+        public void lower(InitializeNode initializeNode, CiLoweringTool tool) {
+            StructuredGraph graph = (StructuredGraph) initializeNode.graph();
+            HotSpotResolvedJavaType type = (HotSpotResolvedJavaType) initializeNode.type();
+            HotSpotKlassOop hub = type.klassOop();
+            int size = type.instanceSize();
+            assert (size % wordSize()) == 0;
+            assert size >= 0;
+            Key key = new Key(initialize).add("size", size);
+            ValueNode memory = initializeNode.memory();
+            //assert memory instanceof AllocateNode || memory instanceof ConstantNode : memory;
+            Arguments arguments = arguments("memory", memory).add("hub", hub).add("prototypeHeader", type.prototypeHeader());
+            SnippetTemplate template = cache.get(key);
+            Debug.log("Lowering initialize in %s: node=%s, template=%s, arguments=%s", graph, initializeNode, template, arguments);
+            template.instantiate(runtime, initializeNode, initializeNode, arguments);
         }
     }
 }
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/target/AMD64VerifyOopStubCallOp.java
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/target/AMD64VerifyOopStubCallOp.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/target/AMD64VerifyOopStubCallOp.java	Tue Jun 26 10:56:03 2012 +0200
@@ -49,7 +49,7 @@
         // r13: (in) object
         if (object != AMD64.r13) {
             masm.push(AMD64.r13);
-            masm.movl(AMD64.r13, object);
+            masm.movq(AMD64.r13, object);
         }
         AMD64Call.directCall(tasm, masm, HotSpotGraalRuntime.getInstance().getConfig().verifyOopStub, info);
         if (object != AMD64.r13) {
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.java/src/com/oracle/graal/java/FrameStateBuilder.java
--- a/graal/com.oracle.graal.java/src/com/oracle/graal/java/FrameStateBuilder.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.java/src/com/oracle/graal/java/FrameStateBuilder.java	Tue Jun 26 10:56:03 2012 +0200
@@ -116,7 +116,7 @@
     }
 
     public FrameState create(int bci) {
-        return graph.add(new FrameState(method, bci, locals, stack, stackSize, rethrowException, false));
+        return graph.add(new FrameState(method, bci, locals, stack, stackSize, rethrowException, false, null));
     }
 
     public FrameStateBuilder copy() {
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.java/src/com/oracle/graal/java/GraphBuilderPhase.java
--- a/graal/com.oracle.graal.java/src/com/oracle/graal/java/GraphBuilderPhase.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.java/src/com/oracle/graal/java/GraphBuilderPhase.java	Tue Jun 26 10:56:03 2012 +0200
@@ -662,7 +662,7 @@
 
     void genNewInstance(int cpi) {
         JavaType type = lookupType(cpi, NEW);
-        if (type instanceof ResolvedJavaType) {
+        if (type instanceof ResolvedJavaType && ((ResolvedJavaType) type).isInitialized()) {
             NewInstanceNode n = currentGraph.add(new NewInstanceNode((ResolvedJavaType) type));
             frameState.apush(append(n));
         } else {
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.jtt/src/com/oracle/graal/jtt/bytecode/BC_ldiv2.java
--- a/graal/com.oracle.graal.jtt/src/com/oracle/graal/jtt/bytecode/BC_ldiv2.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.jtt/src/com/oracle/graal/jtt/bytecode/BC_ldiv2.java	Tue Jun 26 10:56:03 2012 +0200
@@ -27,6 +27,8 @@
 /*
  */
 public class BC_ldiv2 {
+    public static long MIN = Long.MIN_VALUE;
+    public static long MAX = Long.MAX_VALUE;
 
     public static long test(long a, long b) {
         return a / b;
@@ -34,12 +36,16 @@
 
     @Test
     public void run0() throws Throwable {
-        Assert.assertEquals(-9223372036854775808L, test(-9223372036854775808L, -1));
+        Assert.assertEquals(MIN, test(MIN, -1));
     }
 
     @Test
     public void run1() throws Throwable {
-        Assert.assertEquals(-9223372036854775808L, test(-9223372036854775808L, 1));
+        Assert.assertEquals(MIN, test(MIN, 1));
     }
 
+    @Test
+    public void run2() throws Throwable {
+        Assert.assertEquals(-1, test(MIN, MAX));
+    }
 }
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.jtt/src/com/oracle/graal/jtt/optimize/ArrayCopy04.java
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/graal/com.oracle.graal.jtt/src/com/oracle/graal/jtt/optimize/ArrayCopy04.java	Tue Jun 26 10:56:03 2012 +0200
@@ -0,0 +1,137 @@
+/*
+ * Copyright (c) 2011, 2012, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.graal.jtt.optimize;
+
+import org.junit.*;
+
+/*
+ * Tests calls to the array copy method.
+ */
+public class ArrayCopy04 {
+
+    public static byte[] array = new byte[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
+    public static byte[] array0 = new byte[]{0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10};
+
+    @Before
+    public void setUp() {
+        System.currentTimeMillis();
+        for (int i = 0; i < array.length; i++) {
+            array[i] = array0[i];
+        }
+    }
+
+    public static byte[] test(int srcPos, int destPos, int length) {
+        System.arraycopy(array, srcPos, array, destPos, length);
+        return array;
+    }
+
+    @Test
+    public void run0() throws Throwable {
+        Assert.assertArrayEquals(new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, test(0, 0, 0));
+    }
+
+    @Test(expected = java.lang.IndexOutOfBoundsException.class)
+    public void run1() throws Throwable {
+        test(0, 0, -1);
+    }
+
+    @Test(expected = java.lang.IndexOutOfBoundsException.class)
+    public void run2() throws Throwable {
+        test(-1, 0, 0);
+    }
+
+    @Test(expected = java.lang.IndexOutOfBoundsException.class)
+    public void run3() throws Throwable {
+        test(0, -1, 0);
+    }
+
+    @Test
+    public void run4() throws Throwable {
+        Assert.assertArrayEquals(new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, test(0, 0, 2));
+    }
+
+    @Test(expected = java.lang.IndexOutOfBoundsException.class)
+    public void run5() throws Throwable {
+        test(0, 1, 11);
+    }
+
+    @Test(expected = java.lang.IndexOutOfBoundsException.class)
+    public void run6() throws Throwable {
+        test(1, 0, 11);
+    }
+
+    @Test(expected = java.lang.IndexOutOfBoundsException.class)
+    public void run7() throws Throwable {
+        test(1, 1, -1);
+    }
+
+    @Test
+    public void run8() throws Throwable {
+        Assert.assertArrayEquals(new byte[] {0, 0, 1, 3, 4, 5, 6, 7, 8, 9, 10}, test(0, 1, 2));
+    }
+
+    @Test
+    public void run9() throws Throwable {
+        Assert.assertArrayEquals(new byte[] {1, 2, 2, 3, 4, 5, 6, 7, 8, 9, 10}, test(1, 0, 2));
+    }
+
+    @Test
+    public void run10() throws Throwable {
+        Assert.assertArrayEquals(new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, test(1, 1, 2));
+    }
+
+    @Test
+    public void run11() throws Throwable {
+        Assert.assertArrayEquals(new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, test(0, 0, 6));
+    }
+
+    @Test
+    public void run12() throws Throwable {
+        Assert.assertArrayEquals(new byte[] {0, 0, 1, 2, 3, 4, 6, 7, 8, 9, 10}, test(0, 1, 5));
+    }
+
+    @Test
+    public void run13() throws Throwable {
+        Assert.assertArrayEquals(new byte[] {1, 2, 3, 4, 5, 5, 6, 7, 8, 9, 10}, test(1, 0, 5));
+    }
+
+    @Test
+    public void run14() throws Throwable {
+        Assert.assertArrayEquals(new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, test(1, 1, 5));
+    }
+
+    @Test
+    public void run15() throws Throwable {
+        Assert.assertArrayEquals(new byte[] {0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10}, test(0, 0, 11));
+    }
+
+    @Test
+    public void run16() throws Throwable {
+        Assert.assertArrayEquals(new byte[] {0, 0, 1, 2, 3, 4, 5, 6, 7, 8, 9}, test(0, 1, 10));
+    }
+
+    @Test
+    public void run17() throws Throwable {
+        Assert.assertArrayEquals(new byte[] {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 10}, test(1, 0, 10));
+    }
+}
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.lir.amd64/src/com/oracle/graal/lir/amd64/AMD64Arithmetic.java
--- a/graal/com.oracle.graal.lir.amd64/src/com/oracle/graal/lir/amd64/AMD64Arithmetic.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.lir.amd64/src/com/oracle/graal/lir/amd64/AMD64Arithmetic.java	Tue Jun 26 10:56:03 2012 +0200
@@ -397,7 +397,7 @@
                         masm.movq(AMD64.rdx, java.lang.Long.MIN_VALUE);
                         masm.cmpq(AMD64.rax, AMD64.rdx);
                         masm.jcc(ConditionFlag.notEqual, normalCase);
-                        masm.cmpl(asRegister(src), -1);
+                        masm.cmpq(asRegister(src), -1);
                         masm.jcc(ConditionFlag.equal, continuation);
                         masm.bind(normalCase);
                     }
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/AnchorNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/AnchorNode.java	Mon Jun 18 00:29:37 2012 +0200
+++ /dev/null	Thu Jan 01 00:00:00 1970 +0000
@@ -1,50 +0,0 @@
-/*
- * Copyright (c) 2009, 2011, Oracle and/or its affiliates. All rights reserved.
- * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
- *
- * This code is free software; you can redistribute it and/or modify it
- * under the terms of the GNU General Public License version 2 only, as
- * published by the Free Software Foundation.
- *
- * This code is distributed in the hope that it will be useful, but WITHOUT
- * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
- * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
- * version 2 for more details (a copy is included in the LICENSE file that
- * accompanied this code).
- *
- * You should have received a copy of the GNU General Public License version
- * 2 along with this work; if not, write to the Free Software Foundation,
- * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
- *
- * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
- * or visit www.oracle.com if you need additional information or have any
- * questions.
- */
-package com.oracle.graal.nodes;
-
-import com.oracle.graal.nodes.spi.*;
-import com.oracle.graal.nodes.type.*;
-
-/**
- * The {@code AnchorNode} can be used a lower bound for a guard. It can also be used as an upper bound if no other FixedNode can be used for that purpose.
- * The guards that should be kept above this node need to be added to the {@link #dependencies()} collection.
- */
-public final class AnchorNode extends FixedWithNextNode implements LIRLowerable, Canonicalizable {
-
-    public AnchorNode() {
-        super(StampFactory.dependency());
-    }
-
-    @Override
-    public ValueNode canonical(CanonicalizerTool tool) {
-        if (this.usages().size() == 0 && dependencies().isEmpty()) {
-            return null;
-        }
-        return this;
-    }
-
-    @Override
-    public void generate(LIRGeneratorTool gen) {
-        // Currently, there is nothing to emit since anchors are only a structural element with no execution semantics.
-    }
-}
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/FixedGuardNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/FixedGuardNode.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/FixedGuardNode.java	Tue Jun 26 10:56:03 2012 +0200
@@ -26,6 +26,7 @@
 import com.oracle.graal.api.meta.*;
 import com.oracle.graal.cri.*;
 import com.oracle.graal.graph.*;
+import com.oracle.graal.nodes.extended.*;
 import com.oracle.graal.nodes.spi.*;
 import com.oracle.graal.nodes.type.*;
 
@@ -92,8 +93,7 @@
 
     @Override
     public void lower(CiLoweringTool tool) {
-        AnchorNode newAnchor = graph().add(new AnchorNode());
-        newAnchor.dependencies().add(tool.createGuard(condition, deoptReason, action, negated, leafGraphId));
+        ValueAnchorNode newAnchor = graph().add(new ValueAnchorNode(tool.createGuard(condition, deoptReason, action, negated, leafGraphId)));
         ((StructuredGraph) graph()).replaceFixedWithFixed(this, newAnchor);
     }
 
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/FrameState.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/FrameState.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/FrameState.java	Tue Jun 26 10:56:03 2012 +0200
@@ -37,6 +37,25 @@
  */
 public final class FrameState extends VirtualState implements Node.IterableNodeType, LIRLowerable {
 
+    /**
+     * An instance of this class is an identifier for all nodes that were generated by one specific inlining operation.
+     * It is used to generate the correct debug information for nested locks.
+     */
+    public static final class InliningIdentifier {
+        private final ResolvedJavaMethod method;
+        private final String context;
+
+        public InliningIdentifier(ResolvedJavaMethod method, String context) {
+            this.method = method;
+            this.context = context;
+        }
+
+        @Override
+        public String toString() {
+            return method + "@" + context;
+        }
+    }
+
     protected final int localsSize;
 
     protected final int stackSize;
@@ -46,6 +65,12 @@
     private boolean duringCall;
 
     /**
+     * This object identifies the concrete inlining operation that produced this frame state.
+     * It is set during inlining, therefore for the outermost frame states of a graph this field is null.
+     */
+    private InliningIdentifier inliningIdentifier;
+
+    /**
      * This BCI should be used for frame states that are built for code with no meaningful BCI.
      */
     public static final int UNKNOWN_BCI = -4;
@@ -92,7 +117,7 @@
      * @param stackSize size of the stack
      * @param rethrowException if true the VM should re-throw the exception on top of the stack when deopt'ing using this framestate
      */
-    public FrameState(ResolvedJavaMethod method, int bci, List<ValueNode> values, int stackSize, boolean rethrowException, boolean duringCall, List<VirtualObjectState> virtualObjectMappings) {
+    public FrameState(ResolvedJavaMethod method, int bci, List<ValueNode> values, int stackSize, boolean rethrowException, boolean duringCall, InliningIdentifier inliningIdentifier, List<VirtualObjectState> virtualObjectMappings) {
         assert stackSize >= 0;
         assert (bci >= 0 && method != null) || (bci < 0 && method == null && values.isEmpty());
         this.method = method;
@@ -103,6 +128,7 @@
         this.virtualObjectMappings = new NodeInputList<>(this, virtualObjectMappings);
         this.rethrowException = rethrowException;
         this.duringCall = duringCall;
+        this.inliningIdentifier = inliningIdentifier;
         assert !rethrowException || stackSize == 1 : "must have exception on top of the stack";
     }
 
@@ -111,10 +137,10 @@
      * @param bci marker bci, needs to be < 0
      */
     public FrameState(int bci) {
-        this(null, bci, Collections.<ValueNode>emptyList(), 0, false, false, Collections.<VirtualObjectState>emptyList());
+        this(null, bci, Collections.<ValueNode>emptyList(), 0, false, false, null, Collections.<VirtualObjectState>emptyList());
     }
 
-    public FrameState(ResolvedJavaMethod method, int bci, ValueNode[] locals, ValueNode[] stack, int stackSize, boolean rethrowException, boolean duringCall) {
+    public FrameState(ResolvedJavaMethod method, int bci, ValueNode[] locals, ValueNode[] stack, int stackSize, boolean rethrowException, boolean duringCall, InliningIdentifier inliningIdentifier) {
         this.method = method;
         this.bci = bci;
         this.localsSize = locals.length;
@@ -132,6 +158,7 @@
         this.virtualObjectMappings = new NodeInputList<>(this);
         this.rethrowException = rethrowException;
         this.duringCall = duringCall;
+        this.inliningIdentifier = inliningIdentifier;
         assert !rethrowException || stackSize == 1 : "must have exception on top of the stack";
     }
 
@@ -148,6 +175,14 @@
         this.outerFrameState = x;
     }
 
+    public InliningIdentifier inliningIdentifier() {
+        return inliningIdentifier;
+    }
+
+    public void setInliningIdentifier(InliningIdentifier inliningIdentifier) {
+        this.inliningIdentifier = inliningIdentifier;
+    }
+
     public boolean rethrowException() {
         return rethrowException;
     }
@@ -176,7 +211,7 @@
         return virtualObjectMappings.get(i);
     }
 
-    public Iterable<VirtualObjectState> virtualObjectMappings() {
+    public NodeInputList<VirtualObjectState> virtualObjectMappings() {
         return virtualObjectMappings;
     }
 
@@ -184,7 +219,9 @@
      * Gets a copy of this frame state.
      */
     public FrameState duplicate(int newBci) {
-        return duplicate(newBci, false);
+        FrameState other = graph().add(new FrameState(method, newBci, values, stackSize, rethrowException, duringCall, inliningIdentifier, virtualObjectMappings));
+        other.setOuterFrameState(outerFrameState());
+        return other;
     }
 
     /**
@@ -194,12 +231,21 @@
         return duplicate(bci);
     }
 
-    public FrameState duplicate(int newBci, boolean duplicateOuter) {
-        FrameState other = graph().add(new FrameState(method, newBci, values, stackSize, rethrowException, duringCall, virtualObjectMappings));
+    /**
+     * Duplicates a FrameState, along with a deep copy of all connected VirtualState (outer FrameStates,
+     * VirtualObjectStates, ...).
+     */
+    @Override
+    public FrameState duplicateWithVirtualState() {
         FrameState newOuterFrameState = outerFrameState();
-        if (duplicateOuter && newOuterFrameState != null) {
-            newOuterFrameState = newOuterFrameState.duplicate(newOuterFrameState.bci, duplicateOuter);
+        if (newOuterFrameState != null) {
+            newOuterFrameState = newOuterFrameState.duplicateWithVirtualState();
         }
+        ArrayList<VirtualObjectState> newVirtualMappings = new ArrayList<>(virtualObjectMappings.size());
+        for (VirtualObjectState state : virtualObjectMappings) {
+            newVirtualMappings.add(state.duplicateWithVirtualState());
+        }
+        FrameState other = graph().add(new FrameState(method, bci, values, stackSize, rethrowException, duringCall, inliningIdentifier, newVirtualMappings));
         other.setOuterFrameState(newOuterFrameState);
         return other;
     }
@@ -221,7 +267,7 @@
         }
         Collections.addAll(copy, pushedValues);
 
-        FrameState other = graph().add(new FrameState(method, newBci, copy, copy.size() - localsSize, newRethrowException, false, virtualObjectMappings));
+        FrameState other = graph().add(new FrameState(method, newBci, copy, copy.size() - localsSize, newRethrowException, false, inliningIdentifier, virtualObjectMappings));
         other.setOuterFrameState(outerFrameState());
         return other;
     }
@@ -342,4 +388,17 @@
         }
         return super.verify();
     }
+
+    @Override
+    public void applyToNonVirtual(NodeClosure< ? super ValueNode> closure) {
+        for (ValueNode value : values.nonNull()) {
+            closure.apply(this, value);
+        }
+        for (VirtualObjectState state : virtualObjectMappings) {
+            state.applyToNonVirtual(closure);
+        }
+        if (outerFrameState() != null) {
+            outerFrameState().applyToNonVirtual(closure);
+        }
+    }
 }
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/PhiNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/PhiNode.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/PhiNode.java	Tue Jun 26 10:56:03 2012 +0200
@@ -84,16 +84,19 @@
         return values;
     }
 
+    @Override
     public boolean inferStamp() {
-        Stamp newStamp = StampFactory.meet(values());
-        if (stamp().equals(newStamp)) {
+        if (type == PhiType.Value) {
+            return inferPhiStamp();
+        } else {
             return false;
-        } else {
-            setStamp(newStamp);
-            return true;
         }
     }
 
+    public boolean inferPhiStamp() {
+        return updateStamp(StampTool.meet(values()));
+    }
+
     @Override
     public boolean verify() {
         assertTrue(merge() != null, "missing merge");
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/StructuredGraph.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/StructuredGraph.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/StructuredGraph.java	Tue Jun 26 10:56:03 2012 +0200
@@ -173,7 +173,7 @@
     }
 
     public boolean hasLoops() {
-        return getNodes(LoopBeginNode.class).iterator().hasNext();
+        return getNodes(LoopBeginNode.class).isNotEmpty();
     }
 
     public void removeFloating(FloatingNode node) {
@@ -323,6 +323,7 @@
             reduceTrivialMerge(begin);
         } else { // convert to merge
             MergeNode merge = this.add(new MergeNode());
+            merge.setProbability(begin.probability());
             this.replaceFixedWithFixed(begin, merge);
         }
     }
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/ValueNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/ValueNode.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/ValueNode.java	Tue Jun 26 10:56:03 2012 +0200
@@ -78,12 +78,39 @@
         this.stamp = stamp;
     }
 
+    /**
+     * Checks if the given stamp is different than the current one ({@code newStamp.equals(oldStamp) == false}). If it
+     * is different then the new stamp will become the current stamp for this node.
+     *
+     * @return true if the stamp has changed, false otherwise.
+     */
+    protected final boolean updateStamp(Stamp newStamp) {
+        if (newStamp.equals(stamp)) {
+            return false;
+        } else {
+            stamp = newStamp;
+            return true;
+        }
+    }
+
+    /**
+     * This method can be overridden by subclasses of {@link ValueNode} if they need to recompute their stamp if their
+     * inputs change. A typical implementation will compute the stamp and pass it to {@link #updateStamp(Stamp)}, whose
+     * return value can be used as the result of this method.
+     *
+     * @return true if the stamp has changed, false otherwise.
+     */
+    public boolean inferStamp() {
+        return false;
+    }
+
     public Kind kind() {
         return stamp.kind();
     }
 
     /**
      * Checks whether this value is a constant (i.e. it is of type {@link ConstantNode}.
+     *
      * @return {@code true} if this value is a constant
      */
     public final boolean isConstant() {
@@ -103,6 +130,7 @@
 
     /**
      * Checks whether this value represents the null constant.
+     *
      * @return {@code true} if this value represents the null constant
      */
     public final boolean isNullConstant() {
@@ -111,8 +139,8 @@
 
     /**
      * Convert this value to a constant if it is a constant, otherwise return null.
-     * @return the {@link Constant} represented by this value if it is a constant; {@code null}
-     * otherwise
+     *
+     * @return the {@link Constant} represented by this value if it is a constant; {@code null} otherwise
      */
     public final Constant asConstant() {
         if (this instanceof ConstantNode) {
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/ValueProxyNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/ValueProxyNode.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/ValueProxyNode.java	Tue Jun 26 10:56:03 2012 +0200
@@ -22,8 +22,8 @@
  */
 package com.oracle.graal.nodes;
 
-import com.oracle.graal.graph.Node;
-import com.oracle.graal.graph.Node.*;
+import com.oracle.graal.graph.*;
+import com.oracle.graal.graph.Node.ValueNumberable;
 import com.oracle.graal.nodes.PhiNode.PhiType;
 import com.oracle.graal.nodes.calc.*;
 import com.oracle.graal.nodes.type.*;
@@ -51,8 +51,13 @@
     }
 
     @Override
+    public boolean inferStamp() {
+        return updateStamp(value.stamp());
+    }
+
+    @Override
     public Stamp stamp() {
-        return value.stamp();
+        return value().stamp();
     }
 
     public BeginNode proxyPoint() {
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/VirtualState.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/VirtualState.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/VirtualState.java	Tue Jun 26 10:56:03 2012 +0200
@@ -30,4 +30,12 @@
  */
 public abstract class VirtualState extends Node {
 
+    public interface NodeClosure<T extends Node> {
+        void apply(Node usage, T node);
+    }
+
+    public abstract VirtualState duplicateWithVirtualState();
+
+    public abstract void applyToNonVirtual(NodeClosure<? super ValueNode> closure);
+
 }
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/AndNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/AndNode.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/AndNode.java	Tue Jun 26 10:56:03 2012 +0200
@@ -26,6 +26,7 @@
 import com.oracle.graal.graph.*;
 import com.oracle.graal.nodes.*;
 import com.oracle.graal.nodes.spi.*;
+import com.oracle.graal.nodes.type.*;
 
 @NodeInfo(shortName = "&")
 public final class AndNode extends LogicNode implements Canonicalizable, LIRLowerable {
@@ -35,6 +36,11 @@
     }
 
     @Override
+    public boolean inferStamp() {
+        return updateStamp(StampTool.and(x().integerStamp(), y().integerStamp()));
+    }
+
+    @Override
     public ValueNode canonical(CanonicalizerTool tool) {
         if (x() == y()) {
             return x();
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/CompareNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/CompareNode.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/CompareNode.java	Tue Jun 26 10:56:03 2012 +0200
@@ -80,11 +80,11 @@
     }
 
 
-    private ValueNode optimizeMaterialize(Constant constant, MaterializeNode materializeNode, CodeCacheProvider runtime, Condition cond) {
-        Constant trueConstant = materializeNode.trueValue().asConstant();
-        Constant falseConstant = materializeNode.falseValue().asConstant();
+    private ValueNode optimizeConditional(Constant constant, ConditionalNode conditionalNode, CodeCacheProvider runtime, Condition cond) {
+        Constant trueConstant = conditionalNode.trueValue().asConstant();
+        Constant falseConstant = conditionalNode.falseValue().asConstant();
 
-        if (falseConstant != null && trueConstant != null) {
+        if (falseConstant != null && trueConstant != null && runtime != null) {
             Boolean trueResult = cond.foldCondition(trueConstant, constant, runtime, unorderedIsTrue());
             Boolean falseResult = cond.foldCondition(falseConstant, constant, runtime, unorderedIsTrue());
 
@@ -96,11 +96,11 @@
                 } else {
                     if (trueUnboxedResult) {
                         assert falseUnboxedResult == false;
-                        return materializeNode.condition();
+                        return conditionalNode.condition();
                     } else {
                         assert falseUnboxedResult == true;
                         negateUsages();
-                        return materializeNode.condition();
+                        return conditionalNode.condition();
 
                     }
                 }
@@ -114,18 +114,18 @@
     }
 
     public ValueNode canonical(CanonicalizerTool tool) {
-        if (x().isConstant() && y().isConstant()) {
+        if (x().isConstant() && y().isConstant() && tool.runtime() != null) {
             return ConstantNode.forBoolean(condition().foldCondition(x().asConstant(), y().asConstant(), tool.runtime(), unorderedIsTrue()), graph());
         }
         if (x().isConstant()) {
-            if (y() instanceof MaterializeNode) {
-                return optimizeMaterialize(x().asConstant(), (MaterializeNode) y(), tool.runtime(), condition().mirror());
+            if (y() instanceof ConditionalNode) {
+                return optimizeConditional(x().asConstant(), (ConditionalNode) y(), tool.runtime(), condition().mirror());
             } else if (y() instanceof NormalizeCompareNode) {
                 return optimizeNormalizeCmp(x().asConstant(), (NormalizeCompareNode) y(), true);
             }
         } else if (y().isConstant()) {
-            if (x() instanceof MaterializeNode) {
-                return optimizeMaterialize(y().asConstant(), (MaterializeNode) x(), tool.runtime(), condition());
+            if (x() instanceof ConditionalNode) {
+                return optimizeConditional(y().asConstant(), (ConditionalNode) x(), tool.runtime(), condition());
             } else if (x() instanceof NormalizeCompareNode) {
                 return optimizeNormalizeCmp(y().asConstant(), (NormalizeCompareNode) x(), false);
             }
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/Condition.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/Condition.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/Condition.java	Tue Jun 26 10:56:03 2012 +0200
@@ -259,7 +259,7 @@
      * @param unorderedIsTrue true if an undecided float comparison should result in "true"
      * @return true if the comparison is known to be true, false if the comparison is known to be false
      */
-    public boolean foldCondition(Constant lt, Constant rt, CodeCacheProvider runtime, boolean unorderedIsTrue) {
+    public boolean foldCondition(Constant lt, Constant rt, MetaAccessProvider runtime, boolean unorderedIsTrue) {
         switch (lt.kind) {
             case Boolean:
             case Byte:
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/ConditionalNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/ConditionalNode.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/ConditionalNode.java	Tue Jun 26 10:56:03 2012 +0200
@@ -22,6 +22,7 @@
  */
 package com.oracle.graal.nodes.calc;
 
+import com.oracle.graal.api.meta.*;
 import com.oracle.graal.nodes.*;
 import com.oracle.graal.nodes.spi.*;
 
@@ -53,6 +54,17 @@
 
     @Override
     public ValueNode canonical(CanonicalizerTool tool) {
+        // this optimizes the case where a value that can only be 0 or 1 is materialized to 0 or 1
+        if (x().isConstant() && y().isConstant() && condition instanceof IntegerEqualsNode) {
+            IntegerEqualsNode equals = (IntegerEqualsNode) condition;
+            if (equals.y().isConstant() && equals.y().asConstant().equals(Constant.INT_0)) {
+                if (equals.x().integerStamp().mask() == 1) {
+                    if (x().asConstant().equals(Constant.INT_0) && y().asConstant().equals(Constant.INT_1)) {
+                        return equals.x();
+                    }
+                }
+            }
+        }
         if (condition instanceof ConstantNode) {
             ConstantNode c = (ConstantNode) condition;
             if (c.asConstant().asBoolean()) {
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/IntegerAddNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/IntegerAddNode.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/IntegerAddNode.java	Tue Jun 26 10:56:03 2012 +0200
@@ -27,6 +27,7 @@
 import com.oracle.graal.nodes.*;
 import com.oracle.graal.nodes.spi.*;
 import com.oracle.graal.nodes.spi.types.*;
+import com.oracle.graal.nodes.type.*;
 
 @NodeInfo(shortName = "+")
 public class IntegerAddNode extends IntegerArithmeticNode implements Canonicalizable, LIRLowerable, TypeFeedbackProvider {
@@ -36,6 +37,11 @@
     }
 
     @Override
+    public boolean inferStamp() {
+        return updateStamp(StampTool.add(x().integerStamp(), y().integerStamp()));
+    }
+
+    @Override
     public ValueNode canonical(CanonicalizerTool tool) {
         if (x().isConstant() && !y().isConstant()) {
             return graph().unique(new IntegerAddNode(kind(), y(), x()));
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/IntegerDivNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/IntegerDivNode.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/IntegerDivNode.java	Tue Jun 26 10:56:03 2012 +0200
@@ -22,10 +22,12 @@
  */
 package com.oracle.graal.nodes.calc;
 
+import com.oracle.graal.api.code.*;
 import com.oracle.graal.api.meta.*;
 import com.oracle.graal.graph.*;
 import com.oracle.graal.nodes.*;
 import com.oracle.graal.nodes.spi.*;
+import com.oracle.graal.nodes.type.*;
 
 @NodeInfo(shortName = "/")
 public final class IntegerDivNode extends IntegerArithmeticNode implements Canonicalizable, LIRLowerable {
@@ -35,6 +37,11 @@
     }
 
     @Override
+    public boolean inferStamp() {
+        return updateStamp(StampTool.div(x().integerStamp(), y().integerStamp()));
+    }
+
+    @Override
     public ValueNode canonical(CanonicalizerTool tool) {
         if (x().isConstant() && y().isConstant()) {
             long yConst = y().asConstant().asLong();
@@ -52,6 +59,33 @@
             if (c == 1) {
                 return x();
             }
+            if (c == -1) {
+                return graph().unique(new NegateNode(x()));
+            }
+            long abs = Math.abs(c);
+            if (CodeUtil.isPowerOf2(abs)) {
+                ValueNode dividend = x();
+                IntegerStamp stampX = x().integerStamp();
+                int log2 = CodeUtil.log2(abs);
+                // no rounding if dividend is negative or if its low bits are always 0
+                if (stampX.lowerBound() < 0 || (stampX.mask() & (abs - 1)) != 0) {
+                    int bits;
+                    if (kind().isInt()) {
+                        bits = 32;
+                    } else {
+                        assert kind().isLong();
+                        bits = 64;
+                    }
+                    RightShiftNode sign = graph().unique(new RightShiftNode(kind(), x(), ConstantNode.forInt(bits - 1, graph())));
+                    UnsignedRightShiftNode round = graph().unique(new UnsignedRightShiftNode(kind(), sign, ConstantNode.forInt(bits - log2, graph())));
+                    dividend = IntegerArithmeticNode.add(dividend, round);
+                }
+                RightShiftNode shift = graph().unique(new RightShiftNode(kind(), dividend, ConstantNode.forInt(log2, graph())));
+                if (c < 0) {
+                    return graph().unique(new NegateNode(shift));
+                }
+                return shift;
+            }
         }
         return this;
     }
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/IntegerLessThanNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/IntegerLessThanNode.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/IntegerLessThanNode.java	Tue Jun 26 10:56:03 2012 +0200
@@ -77,6 +77,9 @@
         } else if (x().integerStamp().lowerBound() >= y().integerStamp().upperBound()) {
             return ConstantNode.forBoolean(false, graph());
         }
+        if (x().integerStamp().lowerBound() >= 0 && y().integerStamp().lowerBound() >= 0) {
+            return graph().unique(new IntegerBelowThanNode(x(), y()));
+        }
         return super.canonical(tool);
     }
 }
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/IntegerMulNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/IntegerMulNode.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/IntegerMulNode.java	Tue Jun 26 10:56:03 2012 +0200
@@ -55,8 +55,14 @@
             if (c == 0) {
                 return ConstantNode.defaultForKind(kind(), graph());
             }
-            if (c > 0 && CodeUtil.isPowerOf2(c)) {
-                return graph().unique(new LeftShiftNode(kind(), x(), ConstantNode.forInt(CodeUtil.log2(c), graph())));
+            long abs = Math.abs(c);
+            if (abs > 0 && CodeUtil.isPowerOf2(abs)) {
+                LeftShiftNode shift = graph().unique(new LeftShiftNode(kind(), x(), ConstantNode.forInt(CodeUtil.log2(abs), graph())));
+                if (c < 0) {
+                    return graph().unique(new NegateNode(shift));
+                } else {
+                    return shift;
+                }
             }
             // canonicalize expressions like "(a * 1) * 2"
             return BinaryNode.reassociate(this, ValueNode.isConstantPredicate());
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/IntegerRemNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/IntegerRemNode.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/IntegerRemNode.java	Tue Jun 26 10:56:03 2012 +0200
@@ -22,6 +22,7 @@
  */
 package com.oracle.graal.nodes.calc;
 
+import com.oracle.graal.api.code.*;
 import com.oracle.graal.api.meta.*;
 import com.oracle.graal.graph.*;
 import com.oracle.graal.nodes.*;
@@ -51,6 +52,8 @@
             long c = y().asConstant().asLong();
             if (c == 1 || c == -1) {
                 return ConstantNode.forIntegerKind(kind(), 0, graph());
+            } else if (c > 0 && CodeUtil.isPowerOf2(c) && x().integerStamp().lowerBound() >= 0) {
+                return graph().unique(new AndNode(kind(), x(), ConstantNode.forIntegerKind(kind(), c - 1, graph())));
             }
         }
         return this;
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/IntegerSubNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/IntegerSubNode.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/IntegerSubNode.java	Tue Jun 26 10:56:03 2012 +0200
@@ -26,6 +26,7 @@
 import com.oracle.graal.graph.*;
 import com.oracle.graal.nodes.*;
 import com.oracle.graal.nodes.spi.*;
+import com.oracle.graal.nodes.type.*;
 
 @NodeInfo(shortName = "-")
 public final class IntegerSubNode extends IntegerArithmeticNode implements Canonicalizable, LIRLowerable {
@@ -35,6 +36,11 @@
     }
 
     @Override
+    public boolean inferStamp() {
+        return updateStamp(StampTool.sub(x().integerStamp(), y().integerStamp()));
+    }
+
+    @Override
     public ValueNode canonical(CanonicalizerTool tool) {
         if (x() == y()) {
             return ConstantNode.forIntegerKind(kind(), 0, graph());
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/LeftShiftNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/LeftShiftNode.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/LeftShiftNode.java	Tue Jun 26 10:56:03 2012 +0200
@@ -26,6 +26,7 @@
 import com.oracle.graal.graph.*;
 import com.oracle.graal.nodes.*;
 import com.oracle.graal.nodes.spi.*;
+import com.oracle.graal.nodes.type.*;
 
 @NodeInfo(shortName = "<<")
 public final class LeftShiftNode extends ShiftNode implements Canonicalizable, LIRLowerable {
@@ -35,6 +36,11 @@
     }
 
     @Override
+    public boolean inferStamp() {
+        return updateStamp(StampTool.leftShift(x().integerStamp(), y().integerStamp()));
+    }
+
+    @Override
     public ValueNode canonical(CanonicalizerTool tool) {
         if (y().isConstant()) {
             int amount = y().asConstant().asInt();
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/LogicNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/LogicNode.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/LogicNode.java	Tue Jun 26 10:56:03 2012 +0200
@@ -38,6 +38,7 @@
      */
     public LogicNode(Kind kind, ValueNode x, ValueNode y) {
         super(kind, x, y);
+        assert kind == Kind.Int || kind == Kind.Long;
     }
 
     public static LogicNode and(ValueNode v1, ValueNode v2) {
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/NegateNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/NegateNode.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/NegateNode.java	Tue Jun 26 10:56:03 2012 +0200
@@ -37,13 +37,18 @@
         return x;
     }
 
+    @Override
+    public boolean inferStamp() {
+        return updateStamp(StampTool.negate(x().stamp()));
+    }
+
     /**
      * Creates new NegateNode instance.
      *
      * @param x the instruction producing the value that is input to this instruction
      */
     public NegateNode(ValueNode x) {
-        super(x.stamp() instanceof IntegerStamp ? StampFactory.negate((IntegerStamp) x.stamp()) : StampFactory.forKind(x.kind()));
+        super(StampTool.negate(x.stamp()));
         this.x = x;
     }
 
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/OrNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/OrNode.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/OrNode.java	Tue Jun 26 10:56:03 2012 +0200
@@ -26,6 +26,7 @@
 import com.oracle.graal.graph.*;
 import com.oracle.graal.nodes.*;
 import com.oracle.graal.nodes.spi.*;
+import com.oracle.graal.nodes.type.*;
 
 @NodeInfo(shortName = "|")
 public final class OrNode extends LogicNode implements Canonicalizable, LIRLowerable {
@@ -35,6 +36,11 @@
     }
 
     @Override
+    public boolean inferStamp() {
+        return updateStamp(StampTool.or(x().integerStamp(), y().integerStamp()));
+    }
+
+    @Override
     public ValueNode canonical(CanonicalizerTool tool) {
         if (x() == y()) {
             return x();
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/RightShiftNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/RightShiftNode.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/RightShiftNode.java	Tue Jun 26 10:56:03 2012 +0200
@@ -36,6 +36,9 @@
 
     @Override
     public ValueNode canonical(CanonicalizerTool tool) {
+        if (x().integerStamp().lowerBound() >= 0) {
+            return graph().unique(new UnsignedRightShiftNode(kind(), x(), y()));
+        }
         if (y().isConstant()) {
             int amount = y().asConstant().asInt();
             int originalAmout = amount;
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/UnsignedRightShiftNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/UnsignedRightShiftNode.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/UnsignedRightShiftNode.java	Tue Jun 26 10:56:03 2012 +0200
@@ -26,6 +26,7 @@
 import com.oracle.graal.graph.*;
 import com.oracle.graal.nodes.*;
 import com.oracle.graal.nodes.spi.*;
+import com.oracle.graal.nodes.type.*;
 
 @NodeInfo(shortName = ">>>")
 public final class UnsignedRightShiftNode extends ShiftNode implements Canonicalizable, LIRLowerable {
@@ -35,6 +36,11 @@
     }
 
     @Override
+    public boolean inferStamp() {
+        return updateStamp(StampTool.unsignedRightShift(x().integerStamp(), y().integerStamp()));
+    }
+
+    @Override
     public ValueNode canonical(CanonicalizerTool tool) {
         if (y().isConstant()) {
             int amount = y().asConstant().asInt();
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/extended/IndexedLocationNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/extended/IndexedLocationNode.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/extended/IndexedLocationNode.java	Tue Jun 26 10:56:03 2012 +0200
@@ -69,8 +69,8 @@
     @Override
     public ValueNode canonical(CanonicalizerTool tool) {
         Constant constantIndex = index.asConstant();
-        if (constantIndex != null && constantIndex.kind.stackKind().isInt()) {
-            long constantIndexLong = constantIndex.asInt();
+        if (constantIndex != null) {
+            long constantIndexLong = constantIndex.asLong();
             if (indexScalingEnabled) {
                 if (tool.target() == null) {
                     return this;
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/extended/UnsafeCastNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/extended/UnsafeCastNode.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/extended/UnsafeCastNode.java	Tue Jun 26 10:56:03 2012 +0200
@@ -23,7 +23,6 @@
 package com.oracle.graal.nodes.extended;
 
 import com.oracle.graal.api.meta.*;
-import com.oracle.graal.cri.*;
 import com.oracle.graal.nodes.*;
 import com.oracle.graal.nodes.calc.*;
 import com.oracle.graal.nodes.spi.*;
@@ -32,7 +31,7 @@
 /**
  * The {@code UnsafeCastNode} produces the same value as its input, but with a different type.
  */
-public final class UnsafeCastNode extends FloatingNode implements Canonicalizable, Lowerable, LIRLowerable {
+public final class UnsafeCastNode extends FloatingNode implements Canonicalizable, LIRLowerable {
 
     @Input private ValueNode object;
     private ResolvedJavaType toType;
@@ -41,28 +40,49 @@
         return object;
     }
 
+    public UnsafeCastNode(ValueNode object, ResolvedJavaType toType, boolean exactType, boolean nonNull) {
+        super(toType.kind().isObject() ? new ObjectStamp(toType, exactType, nonNull) : StampFactory.forKind(toType.kind()));
+        this.object = object;
+        this.toType = toType;
+    }
+
     public UnsafeCastNode(ValueNode object, ResolvedJavaType toType) {
-        super(StampFactory.declared(toType, object.stamp().nonNull()));
+        super(toType.kind().isObject() ? StampFactory.declared(toType, object.stamp().nonNull()) : StampFactory.forKind(toType.kind()));
         this.object = object;
         this.toType = toType;
     }
 
     @Override
     public ValueNode canonical(CanonicalizerTool tool) {
-        if (object != null && object.kind().isObject() && object.objectStamp().type() != null && object.objectStamp().type().isSubtypeOf(toType)) {
-            return object;
+        if (object != null) {
+            if (object.kind().isObject()) {
+                if (object.objectStamp().type() != null && object.objectStamp().type().isSubtypeOf(toType)) {
+                    if (!isNarrower(objectStamp(), object.objectStamp())) {
+                        return object;
+                    }
+                }
+            } else if (object.kind() == kind()) {
+                // Removes redundant casts introduced by WordTypeRewriterPhase
+                return object;
+            }
         }
         return this;
     }
 
-
-    @Override
-    public void lower(CiLoweringTool tool) {
-        if (object.kind() == kind()) {
-            ((StructuredGraph) graph()).replaceFloating(this, object);
-        } else {
-            // Cannot remove an unsafe cast between two different kinds
+    /**
+     * Determines if one object stamp is narrower than another in terms of nullness and exactness.
+     *
+     * @return true if x is definitely non-null and y's nullness is unknown OR
+     *                  x's type is exact and the exactness of y's type is unknown
+     */
+    private static boolean isNarrower(ObjectStamp x, ObjectStamp y) {
+        if (x.nonNull() && !y.nonNull()) {
+            return true;
         }
+        if (x.isExactType() && !y.isExactType()) {
+            return true;
+        }
+        return false;
     }
 
     @SuppressWarnings("unused")
@@ -71,6 +91,12 @@
         throw new UnsupportedOperationException("This method may only be compiled with the Graal compiler");
     }
 
+    @SuppressWarnings("unused")
+    @NodeIntrinsic
+    public static <T> T cast(Object object, @ConstantNodeParameter Class<?> toType, @ConstantNodeParameter boolean exactType, @ConstantNodeParameter boolean nonNull) {
+        throw new UnsupportedOperationException("This method may only be compiled with the Graal compiler");
+    }
+
     @Override
     public void generate(LIRGeneratorTool generator) {
         Value result = generator.newVariable(kind());
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/extended/ValueAnchorNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/extended/ValueAnchorNode.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/extended/ValueAnchorNode.java	Tue Jun 26 10:56:03 2012 +0200
@@ -52,20 +52,16 @@
     }
 
     @Override
-    public boolean verify() {
-        assertTrue(usages().isEmpty(), "upwards dependencies should target BeginNodes only");
-        return super.verify();
-    }
-
-    @Override
     public ValueNode canonical(CanonicalizerTool tool) {
         if (this.predecessor() instanceof ValueAnchorNode) {
-            // transfer values and remove
             ValueAnchorNode previousAnchor = (ValueAnchorNode) this.predecessor();
-            for (ValueNode node : dependencies().nonNull().distinct()) {
-                previousAnchor.addAnchoredNode(node);
+            if (previousAnchor.usages().isEmpty()) { // avoid creating cycles
+                // transfer values and remove
+                for (ValueNode node : dependencies().nonNull().distinct()) {
+                    previousAnchor.addAnchoredNode(node);
+                }
+                return previousAnchor;
             }
-            return null;
         }
         for (Node node : dependencies().nonNull().and(isNotA(BeginNode.class))) {
             if (node instanceof ConstantNode) {
@@ -83,6 +79,9 @@
             }
             return this; // still necessary
         }
-        return null; // no node which require an anchor found
+        if (usages().isEmpty()) {
+            return null; // no node which require an anchor found
+        }
+        return this;
     }
 }
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/java/NewArrayNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/java/NewArrayNode.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/java/NewArrayNode.java	Tue Jun 26 10:56:03 2012 +0200
@@ -102,7 +102,7 @@
             EscapeField[] fields = new EscapeField[length];
             for (int i = 0; i < length; i++) {
                 Integer representation = i;
-                fields[i] = new EscapeField("[" + i + "]", representation, ((NewArrayNode) node).elementType());
+                fields[i] = new EscapeField(Integer.toString(i), representation, ((NewArrayNode) node).elementType());
             }
             return fields;
         }
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/java/TypeSwitchNode.java
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/java/TypeSwitchNode.java	Tue Jun 26 10:56:03 2012 +0200
@@ -0,0 +1,66 @@
+/*
+ * Copyright (c) 2009, 2011, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.graal.nodes.java;
+
+import com.oracle.graal.api.meta.*;
+import com.oracle.graal.nodes.*;
+import com.oracle.graal.nodes.extended.*;
+import com.oracle.graal.nodes.spi.*;
+
+/**
+ * The {@code TypeSwitchNode} performs a lookup based on the type of the input value.
+ * The type comparison is an exact type comparison, not an instanceof.
+ */
+public final class TypeSwitchNode extends SwitchNode implements LIRLowerable, Simplifiable {
+
+    private final ResolvedJavaType[] keys;
+
+    public TypeSwitchNode(ValueNode value, BeginNode[] successors, ResolvedJavaType[] keys, double[] probability) {
+        super(value, successors, probability);
+        assert successors.length == keys.length + 1;
+        assert successors.length == probability.length;
+        this.keys = keys;
+    }
+
+    public TypeSwitchNode(ValueNode value, ResolvedJavaType[] keys, double[] switchProbability) {
+        this(value, new BeginNode[switchProbability.length], keys, switchProbability);
+    }
+
+    public ResolvedJavaType keyAt(int i) {
+        return keys[i];
+    }
+
+    public int keysLength() {
+        return keys.length;
+    }
+
+    @Override
+    public void generate(LIRGeneratorTool gen) {
+        gen.emitTypeSwitch(this);
+    }
+
+    @Override
+    public void simplify(SimplifierTool tool) {
+        // TODO(ls) perform simplifications based on the type of value
+    }
+}
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/spi/LIRGeneratorTool.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/spi/LIRGeneratorTool.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/spi/LIRGeneratorTool.java	Tue Jun 26 10:56:03 2012 +0200
@@ -48,6 +48,8 @@
      */
     public abstract boolean canStoreConstant(Constant c);
 
+    public abstract RegisterAttributes attributes(Register register);
+
     public abstract Value operand(ValueNode object);
     public abstract Value newVariable(Kind kind);
     public abstract Value setResult(ValueNode x, Value operand);
@@ -92,6 +94,7 @@
 
     public abstract void emitLookupSwitch(LookupSwitchNode i);
     public abstract void emitTableSwitch(TableSwitchNode i);
+    public abstract void emitTypeSwitch(TypeSwitchNode i);
 
     public abstract void emitInvoke(Invoke i);
     public abstract void emitRuntimeCall(RuntimeCallNode i);
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/type/IntegerStamp.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/type/IntegerStamp.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/type/IntegerStamp.java	Tue Jun 26 10:56:03 2012 +0200
@@ -23,32 +23,56 @@
 package com.oracle.graal.nodes.type;
 
 import com.oracle.graal.api.meta.*;
+import com.oracle.graal.nodes.*;
 
-
+/**
+ * Describes the possible values of a {@link ValueNode} that produces an int or long result.
+ *
+ * The description consists of (inclusive) lower and upper bounds and a bit-mask.
+ */
 public class IntegerStamp extends Stamp {
 
     private final long lowerBound;
     private final long upperBound;
+    private final long mask;
 
     public IntegerStamp(Kind kind) {
-        this(kind, kind.minValue(), kind.maxValue());
+        this(kind, kind.minValue(), kind.maxValue(), defaultMask(kind));
     }
 
-    public IntegerStamp(Kind kind, long lowerBound, long upperBound) {
+    public IntegerStamp(Kind kind, long lowerBound, long upperBound, long mask) {
         super(kind);
         assert lowerBound <= upperBound;
+        assert lowerBound >= kind.minValue();
+        assert upperBound <= kind.maxValue();
+        assert (mask & defaultMask(kind)) == mask;
         this.lowerBound = lowerBound;
         this.upperBound = upperBound;
+        this.mask = mask;
     }
 
+    /**
+     * The (inclusive) lower bound on the value described by this stamp.
+     */
     public long lowerBound() {
         return lowerBound;
     }
 
+    /**
+     * The (inclusive) upper bound on the value described by this stamp.
+     */
     public long upperBound() {
         return upperBound;
     }
 
+    /**
+     * This bit-mask describes the bits that can be set in the value described by this stamp. It is primarily used to
+     * represent values that are multiples of a known power of two.
+     */
+    public long mask() {
+        return mask;
+    }
+
     @Override
     public String toString() {
         StringBuilder str = new StringBuilder();
@@ -58,6 +82,9 @@
         } else if (lowerBound != kind().minValue() || upperBound != kind().maxValue()) {
             str.append(" [").append(lowerBound).append(" - ").append(upperBound).append(']');
         }
+        if (mask != defaultMask(kind())) {
+            str.append(" #").append(Long.toHexString(mask));
+        }
         return str.toString();
     }
 
@@ -73,10 +100,11 @@
         assert kind() == other.kind();
         long meetUpperBound = Math.max(upperBound, other.upperBound);
         long meetLowerBound = Math.min(lowerBound, other.lowerBound);
-        if (meetLowerBound == lowerBound && meetUpperBound == upperBound) {
+        long meetMask = mask | other.mask;
+        if (meetLowerBound == lowerBound && meetUpperBound == upperBound && meetMask == mask) {
             return this;
         } else {
-            return new IntegerStamp(kind(), meetLowerBound, meetUpperBound);
+            return new IntegerStamp(kind(), meetLowerBound, meetUpperBound, meetMask);
         }
     }
 
@@ -98,12 +126,27 @@
             return false;
         }
         IntegerStamp other = (IntegerStamp) obj;
-        if (lowerBound != other.lowerBound || upperBound != other.upperBound) {
+        if (lowerBound != other.lowerBound || upperBound != other.upperBound || mask != other.mask) {
             return false;
         }
         return true;
     }
 
+    public static long defaultMask(Kind kind) {
+        if (kind == Kind.Int) {
+            return 0xFFFFFFFFL;
+        } else {
+            return 0xFFFFFFFFFFFFFFFFL;
+        }
+    }
+
+    public static long maskFor(Kind kind, long lowerBound, long upperBound) {
+        long mask = lowerBound | upperBound;
+        if (mask == 0) {
+            return 0;
+        } else {
+            return ((-1L) >>> Long.numberOfLeadingZeros(mask)) & defaultMask(kind);
+        }
+    }
 
 }
-
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/type/ObjectStamp.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/type/ObjectStamp.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/type/ObjectStamp.java	Tue Jun 26 10:56:03 2012 +0200
@@ -53,16 +53,6 @@
     }
 
     @Override
-    public ResolvedJavaType exactType() {
-        return exactType ? type : null;
-    }
-
-    @Override
-    public ResolvedJavaType declaredType() {
-        return type;
-    }
-
-    @Override
     public String toString() {
         StringBuilder str = new StringBuilder();
         str.append(kind().typeChar);
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/type/Stamp.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/type/Stamp.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/type/Stamp.java	Tue Jun 26 10:56:03 2012 +0200
@@ -25,7 +25,9 @@
 import com.oracle.graal.api.meta.*;
 import com.oracle.graal.nodes.spi.types.*;
 
-
+/**
+ * A stamp is the basis for a type system over the nodes in a graph.
+ */
 public abstract class Stamp {
 
     private final Kind kind;
@@ -50,14 +52,6 @@
         return false;
     }
 
-    public ResolvedJavaType exactType() {
-        return null;
-    }
-
-    public ResolvedJavaType declaredType() {
-        return null;
-    }
-
     public abstract boolean alwaysDistinct(Stamp other);
 
     public abstract Stamp meet(Stamp other);
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/type/StampFactory.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/type/StampFactory.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/type/StampFactory.java	Tue Jun 26 10:56:03 2012 +0200
@@ -22,8 +22,6 @@
  */
 package com.oracle.graal.nodes.type;
 
-import java.util.*;
-
 import com.oracle.graal.api.meta.*;
 import com.oracle.graal.graph.*;
 import com.oracle.graal.nodes.type.GenericStamp.GenericStampType;
@@ -40,7 +38,7 @@
     private static final Stamp conditionStamp = new GenericStamp(GenericStampType.Condition);
     private static final Stamp voidStamp = new GenericStamp(GenericStampType.Void);
 
-    private static final Stamp positiveInt = forInt(0, Integer.MAX_VALUE);
+    private static final Stamp positiveInt = forInteger(Kind.Int, 0, Integer.MAX_VALUE, Integer.MAX_VALUE);
 
     private static void setCache(Kind kind, Stamp stamp) {
         stampCache[kind.ordinal()] = stamp;
@@ -63,6 +61,10 @@
         setCache(Kind.Void, voidStamp);
     }
 
+    public static Stamp forWord(Kind wordKind, boolean nonNull) {
+        return new WordStamp(wordKind, nonNull);
+    }
+
     public static Stamp forKind(Kind kind) {
         assert stampCache[kind.stackKind().ordinal()] != null : "unexpected forKind(" + kind + ")";
         return stampCache[kind.stackKind().ordinal()];
@@ -96,12 +98,8 @@
         return positiveInt;
     }
 
-    public static Stamp forInt(int lowerBound, int upperBound) {
-        return new IntegerStamp(Kind.Int, lowerBound, upperBound);
-    }
-
-    public static Stamp forLong(long lowerBound, long upperBound) {
-        return new IntegerStamp(Kind.Long, lowerBound, upperBound);
+    public static Stamp forInteger(Kind kind, long lowerBound, long upperBound, long mask) {
+        return new IntegerStamp(kind, lowerBound, upperBound, mask);
     }
 
     public static Stamp forConstant(Constant value) {
@@ -109,10 +107,8 @@
         if (value.kind == Kind.Object) {
             throw new GraalInternalError("unexpected kind: %s", value.kind);
         } else {
-            if (value.kind == Kind.Int) {
-                return forInt(value.asInt(), value.asInt());
-            } else if (value.kind == Kind.Long) {
-                return forLong(value.asLong(), value.asLong());
+            if (value.kind == Kind.Int || value.kind == Kind.Long) {
+                return forInteger(value.kind, value.asLong(), value.asLong(), value.asLong() & IntegerStamp.defaultMask(value.kind));
             }
             return forKind(value.kind.stackKind());
         }
@@ -128,14 +124,6 @@
         }
     }
 
-    public static Stamp negate(IntegerStamp stamp) {
-        Kind kind = stamp.kind();
-        if (stamp.lowerBound() != kind.minValue()) {
-            return new IntegerStamp(kind, -stamp.upperBound(), -stamp.lowerBound());
-        }
-        return forKind(kind);
-    }
-
     public static Stamp object() {
         return objectStamp;
     }
@@ -166,22 +154,4 @@
     public static Stamp exactNonNull(ResolvedJavaType type) {
         return new ObjectStamp(type, true, true);
     }
-
-    public static Stamp or(Collection<? extends StampProvider> values) {
-        return meet(values);
-    }
-
-    public static Stamp meet(Collection<? extends StampProvider> values) {
-        if (values.size() == 0) {
-            return forVoid();
-        } else {
-            Iterator< ? extends StampProvider> iterator = values.iterator();
-            Stamp stamp = iterator.next().stamp();
-
-            while (iterator.hasNext()) {
-                stamp = stamp.meet(iterator.next().stamp());
-            }
-            return stamp;
-        }
-    }
 }
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/type/StampTool.java
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/type/StampTool.java	Tue Jun 26 10:56:03 2012 +0200
@@ -0,0 +1,168 @@
+/*
+ * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.graal.nodes.type;
+
+import java.util.*;
+
+import com.oracle.graal.api.meta.*;
+
+/**
+ * Helper class that is used to keep all stamp-related operations in one place.
+ */
+// TODO(ls) maybe move the contents into IntegerStamp
+public class StampTool {
+
+    public static Stamp negate(Stamp stamp) {
+        Kind kind = stamp.kind();
+        if (stamp instanceof IntegerStamp) {
+            IntegerStamp integerStamp = (IntegerStamp) stamp;
+            if (integerStamp.lowerBound() != kind.minValue()) {
+                // TODO(ls) check if the mask calculation is correct...
+                return new IntegerStamp(kind, -integerStamp.upperBound(), -integerStamp.lowerBound(), IntegerStamp.defaultMask(kind) & (integerStamp.mask() | -integerStamp.mask()));
+            }
+        }
+        return StampFactory.forKind(kind);
+    }
+
+    public static Stamp meet(Collection<? extends StampProvider> values) {
+        Iterator< ? extends StampProvider> iterator = values.iterator();
+        if (iterator.hasNext()) {
+            Stamp stamp = iterator.next().stamp();
+            while (iterator.hasNext()) {
+                stamp = stamp.meet(iterator.next().stamp());
+            }
+            return stamp;
+        } else {
+            return StampFactory.forVoid();
+        }
+    }
+
+    public static Stamp add(IntegerStamp stamp1, IntegerStamp stamp2) {
+        Kind kind = stamp1.kind();
+        assert kind == stamp2.kind();
+        if (addOverflow(stamp1.lowerBound(), stamp2.lowerBound(), kind)) {
+            return StampFactory.forKind(kind);
+        }
+        if (addOverflow(stamp1.upperBound(), stamp2.upperBound(), kind)) {
+            return StampFactory.forKind(kind);
+        }
+        long lowerBound = stamp1.lowerBound() + stamp2.lowerBound();
+        long upperBound = stamp1.upperBound() + stamp2.upperBound();
+        long mask = IntegerStamp.maskFor(kind, lowerBound, upperBound) & (stamp1.mask() | stamp2.mask());
+
+        return StampFactory.forInteger(kind, lowerBound, upperBound, mask);
+    }
+
+    public static Stamp sub(IntegerStamp stamp1, IntegerStamp stamp2) {
+        return add(stamp1, (IntegerStamp) StampTool.negate(stamp2));
+    }
+
+    public static Stamp div(IntegerStamp stamp1, IntegerStamp stamp2) {
+        Kind kind = stamp1.kind();
+        if (stamp2.lowerBound() > 0) {
+            long lowerBound = stamp1.lowerBound() / stamp2.lowerBound();
+            long upperBound = stamp1.upperBound() / stamp2.lowerBound();
+            return StampFactory.forInteger(kind, lowerBound, upperBound, IntegerStamp.maskFor(kind, lowerBound, upperBound));
+        }
+        return StampFactory.forKind(kind);
+    }
+
+    private static boolean addOverflow(long x, long y, Kind kind) {
+        long result = x + y;
+        if (kind == Kind.Long) {
+            return ((x ^ result) & (y ^ result)) < 0;
+        } else {
+            assert kind == Kind.Int;
+            return result > Integer.MAX_VALUE || result < Integer.MIN_VALUE;
+        }
+    }
+
+    private static final long INTEGER_SIGN_BIT = 0x80000000L;
+    private static final long LONG_SIGN_BIT = 0x8000000000000000L;
+
+    private static Stamp stampForMask(Kind kind, long mask) {
+        long lowerBound;
+        long upperBound;
+        if (kind == Kind.Int && (mask & INTEGER_SIGN_BIT) != 0) {
+            // the mask is negative
+            lowerBound = Integer.MIN_VALUE;
+            upperBound = mask ^ INTEGER_SIGN_BIT;
+        } else if (kind == Kind.Long && (mask & LONG_SIGN_BIT) != 0) {
+            // the mask is negative
+            lowerBound = Long.MIN_VALUE;
+            upperBound = mask ^ LONG_SIGN_BIT;
+        } else {
+            lowerBound = 0;
+            upperBound = mask;
+        }
+        return StampFactory.forInteger(kind, lowerBound, upperBound, mask);
+    }
+
+    public static Stamp and(IntegerStamp stamp1, IntegerStamp stamp2) {
+        Kind kind = stamp1.kind();
+        long mask = stamp1.mask() & stamp2.mask();
+        return stampForMask(kind, mask);
+    }
+
+    public static Stamp or(IntegerStamp stamp1, IntegerStamp stamp2) {
+        Kind kind = stamp1.kind();
+        long mask = stamp1.mask() | stamp2.mask();
+        return stampForMask(kind, mask);
+    }
+
+    public static Stamp unsignedRightShift(IntegerStamp value, IntegerStamp shift) {
+        Kind kind = value.kind();
+        if (shift.lowerBound() == shift.upperBound()) {
+            long shiftMask = kind == Kind.Int ? 0x1FL : 0x3FL;
+            long shiftCount = shift.lowerBound() & shiftMask;
+            long lowerBound;
+            long upperBound;
+            if (value.lowerBound() < 0) {
+                lowerBound = 0;
+                upperBound = IntegerStamp.defaultMask(kind) >>> shiftCount;
+            } else {
+                lowerBound = value.lowerBound() >>> shiftCount;
+                upperBound = value.upperBound() >>> shiftCount;
+            }
+            long mask = value.mask() >>> shiftCount;
+            return StampFactory.forInteger(kind, lowerBound, upperBound, mask);
+        }
+        long mask = IntegerStamp.maskFor(kind, value.lowerBound(), value.upperBound());
+        return stampForMask(kind, mask);
+    }
+
+    public static Stamp leftShift(IntegerStamp value, IntegerStamp shift) {
+        Kind kind = value.kind();
+        int shiftBits = kind == Kind.Int ? 5 : 6;
+        long shiftMask = kind == Kind.Int ? 0x1FL : 0x3FL;
+        if ((shift.lowerBound() >>> shiftBits) == (shift.upperBound() >>> shiftBits)) {
+            long mask = 0;
+            for (long i = shift.lowerBound() & shiftMask; i <= (shift.upperBound() & shiftMask); i++) {
+                mask |= value.mask() << i;
+            }
+            mask &= IntegerStamp.defaultMask(kind);
+            return stampForMask(kind, mask);
+        }
+        return StampFactory.forKind(kind);
+    }
+}
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/type/WordStamp.java
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/type/WordStamp.java	Tue Jun 26 10:56:03 2012 +0200
@@ -0,0 +1,89 @@
+/*
+ * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.graal.nodes.type;
+
+import com.oracle.graal.api.meta.*;
+
+/**
+ * Models the word type.
+ */
+public class WordStamp extends Stamp {
+
+    private final boolean nonNull;
+
+    public WordStamp(Kind wordKind, boolean nonNull) {
+        super(wordKind);
+        this.nonNull = nonNull;
+    }
+
+    @Override
+    public boolean nonNull() {
+        return nonNull;
+    }
+
+    @Override
+    public String toString() {
+        StringBuilder str = new StringBuilder();
+        str.append(kind().typeChar);
+        str.append(nonNull ? "!" : "");
+        return str.toString();
+    }
+
+    @Override
+    public boolean alwaysDistinct(Stamp otherStamp) {
+        return false;
+    }
+
+    @Override
+    public Stamp meet(Stamp otherStamp) {
+        WordStamp other = (WordStamp) otherStamp;
+        if (other.nonNull == nonNull) {
+            return this;
+        } else {
+            return new WordStamp(kind(), nonNull && other.nonNull);
+        }
+    }
+
+    @Override
+    public int hashCode() {
+        final int prime = 31;
+        int result = 1;
+        result = prime * result + (nonNull ? 1231 : 1237);
+        return result;
+    }
+
+    @Override
+    public boolean equals(Object obj) {
+        if (this == obj) {
+            return true;
+        }
+        if (obj == null || getClass() != obj.getClass()) {
+            return false;
+        }
+        WordStamp other = (WordStamp) obj;
+        if (nonNull != other.nonNull) {
+            return false;
+        }
+        return true;
+    }
+}
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/virtual/VirtualObjectState.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/virtual/VirtualObjectState.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/virtual/VirtualObjectState.java	Tue Jun 26 10:56:03 2012 +0200
@@ -22,6 +22,8 @@
  */
 package com.oracle.graal.nodes.virtual;
 
+import java.util.*;
+
 import com.oracle.graal.graph.*;
 import com.oracle.graal.nodes.*;
 import com.oracle.graal.nodes.spi.*;
@@ -32,24 +34,42 @@
 public final class VirtualObjectState extends VirtualState implements Node.IterableNodeType, LIRLowerable {
 
     @Input private VirtualObjectNode object;
-    @Input private NodeInputList<ValueNode> fields;
+    @Input private NodeInputList<ValueNode> fieldValues;
 
     public VirtualObjectNode object() {
         return object;
     }
 
-    public NodeInputList<ValueNode> fields() {
-        return fields;
+    public NodeInputList<ValueNode> fieldValues() {
+        return fieldValues;
     }
 
-    public VirtualObjectState(VirtualObjectNode object, ValueNode[] fields) {
+    public VirtualObjectState(VirtualObjectNode object, ValueNode[] fieldValues) {
+        assert object.fieldsCount() == fieldValues.length;
         this.object = object;
-        assert object.fields().length == fields.length;
-        this.fields = new NodeInputList<>(this, fields);
+        this.fieldValues = new NodeInputList<>(this, fieldValues);
+    }
+
+    private VirtualObjectState(VirtualObjectNode object, List<ValueNode> fieldValues) {
+        assert object.fieldsCount() == fieldValues.size();
+        this.object = object;
+        this.fieldValues = new NodeInputList<>(this, fieldValues);
     }
 
     @Override
     public void generate(LIRGeneratorTool generator) {
         // Nothing to do, virtual object states are processed as part of the handling of StateSplit nodes.
     }
+
+    @Override
+    public VirtualObjectState duplicateWithVirtualState() {
+        return graph().add(new VirtualObjectState(object, fieldValues));
+    }
+
+    @Override
+    public void applyToNonVirtual(NodeClosure< ? super ValueNode> closure) {
+        for (ValueNode value : fieldValues) {
+            closure.apply(this, value);
+        }
+    }
 }
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.snippets.test/src/com/oracle/graal/snippets/WordTest.java
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/graal/com.oracle.graal.snippets.test/src/com/oracle/graal/snippets/WordTest.java	Tue Jun 26 10:56:03 2012 +0200
@@ -0,0 +1,139 @@
+/*
+ * Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.graal.snippets;
+
+import java.lang.reflect.*;
+
+import org.junit.*;
+
+import com.oracle.graal.api.*;
+import com.oracle.graal.api.code.*;
+import com.oracle.graal.api.meta.*;
+import com.oracle.graal.compiler.*;
+import com.oracle.graal.compiler.tests.*;
+import com.oracle.graal.nodes.*;
+import com.oracle.graal.snippets.Snippet.InliningPolicy;
+
+/**
+ * Tests for the {@link Word} type.
+ */
+public class WordTest extends GraalCompilerTest implements SnippetsInterface {
+
+    private final SnippetInstaller installer;
+
+    public WordTest() {
+        TargetDescription target = Graal.getRequiredCapability(GraalCompiler.class).target;
+        installer = new SnippetInstaller(runtime, target);
+    }
+
+    private static final ThreadLocal<InliningPolicy> inliningPolicy = new ThreadLocal<>();
+
+    @Override
+    protected StructuredGraph parse(Method m) {
+        ResolvedJavaMethod resolvedMethod = runtime.getResolvedJavaMethod(m);
+        return installer.makeGraph(resolvedMethod, inliningPolicy.get());
+    }
+
+    @Test
+    public void test_arithmetic() {
+        long[] words = new long[] {Long.MIN_VALUE, Long.MIN_VALUE + 1, -1L, 0L, 1L, Long.MAX_VALUE - 1, Long.MAX_VALUE};
+        for (long word : words) {
+            for (int addend = -1000; addend < 1000; addend++) {
+                test("plus_int", word, addend);
+                test("plus_int", word, -addend);
+                test("minus_int", word, addend);
+                test("minus_int", word, -addend);
+            }
+            for (long addend : words) {
+                test("plus_int", word, (int) addend);
+                test("minus_int", word, (int) addend);
+                test("plus_int", word, -((int) addend));
+                test("minus_int", word, -((int) addend));
+            }
+        }
+    }
+
+    @Test
+    public void test_compare() {
+        long[] words = new long[] {Long.MIN_VALUE, Long.MIN_VALUE + 1, -1L, 0L, 1L, Long.MAX_VALUE - 1, Long.MAX_VALUE};
+        for (long word1 : words) {
+            for (long word2 : words) {
+                for (String method : new String[] {"aboveOrEqual", "above", "belowOrEqual", "below"}) {
+                    test(method, word1, word2);
+                    test(method, word2, word1);
+                }
+            }
+        }
+    }
+
+    @Test
+    public void test_fromObject() {
+        inliningPolicy.set(new InliningPolicy() {
+            public boolean shouldInline(ResolvedJavaMethod method, ResolvedJavaMethod caller) {
+                return InliningPolicy.Default.shouldInline(method, caller) && !method.name().equals("hashCode");
+            }
+        });
+        test("fromToObject", "object1", "object2");
+        test("fromToObject", "object1", "object1");
+        test("fromToObject", "object", null);
+        test("fromToObject", null, "object");
+        test("fromToObject", null, null);
+        inliningPolicy.set(null);
+    }
+
+    @Snippet
+    public static long plus_int(long word, int addend) {
+        return Word.fromLong(word).plus(addend).toLong();
+    }
+
+    @Snippet
+    public static long minus_int(long word, int addend) {
+        return Word.fromLong(word).plus(addend).toLong();
+    }
+
+    @Snippet
+    public static boolean aboveOrEqual(long word1, long word2) {
+        return Word.fromLong(word1).aboveOrEqual(Word.fromLong(word2));
+    }
+
+    @Snippet
+    public static boolean above(long word1, long word2) {
+        return Word.fromLong(word1).above(Word.fromLong(word2));
+    }
+
+    @Snippet
+    public static boolean belowOrEqual(long word1, long word2) {
+        return Word.fromLong(word1).belowOrEqual(Word.fromLong(word2));
+    }
+
+    @Snippet
+    public static boolean below(long word1, long word2) {
+        return Word.fromLong(word1).below(Word.fromLong(word2));
+    }
+
+    @Snippet
+    public static int fromToObject(Object o1, Object o2) {
+        return Word.fromObject(o1).toObject().hashCode() + Word.fromObject(o2).toObject().hashCode();
+    }
+}
+
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.snippets/src/com/oracle/graal/snippets/Snippet.java
--- a/graal/com.oracle.graal.snippets/src/com/oracle/graal/snippets/Snippet.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.snippets/src/com/oracle/graal/snippets/Snippet.java	Tue Jun 26 10:56:03 2012 +0200
@@ -27,6 +27,7 @@
 
 import com.oracle.graal.api.meta.*;
 import com.oracle.graal.graph.Node.NodeIntrinsic;
+import com.oracle.graal.snippets.Word.Operation;
 import com.oracle.graal.snippets.nodes.*;
 
 /**
@@ -67,6 +68,9 @@
                         return false;
                     }
                 }
+                if (method.getAnnotation(Operation.class) != null) {
+                    return false;
+                }
                 return true;
             }
         };
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.snippets/src/com/oracle/graal/snippets/SnippetInstaller.java
--- a/graal/com.oracle.graal.snippets/src/com/oracle/graal/snippets/SnippetInstaller.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.snippets/src/com/oracle/graal/snippets/SnippetInstaller.java	Tue Jun 26 10:56:03 2012 +0200
@@ -134,10 +134,18 @@
         }
     }
 
-    private StructuredGraph makeGraph(final ResolvedJavaMethod method, final InliningPolicy policy) {
+    public StructuredGraph makeGraph(final ResolvedJavaMethod method, final InliningPolicy policy) {
+        StructuredGraph graph = parseGraph(method, policy);
+
+        Debug.dump(graph, "%s: Final", method.name());
+
+        return graph;
+    }
+
+    private StructuredGraph parseGraph(final ResolvedJavaMethod method, final InliningPolicy policy) {
         StructuredGraph graph = graphCache.get(method);
         if (graph == null) {
-            graph = buildGraph(method, policy);
+            graph = buildGraph(method, policy == null ? inliningPolicy(method) : policy);
             //System.out.println("built " + graph);
             graphCache.put(method, graph);
         }
@@ -155,17 +163,19 @@
 
                 Debug.dump(graph, "%s: %s", method.name(), GraphBuilderPhase.class.getSimpleName());
 
+                new SnippetVerificationPhase().apply(graph);
+
                 new SnippetIntrinsificationPhase(runtime, pool).apply(graph);
 
                 for (Invoke invoke : graph.getInvokes()) {
                     MethodCallTargetNode callTarget = invoke.callTarget();
                     ResolvedJavaMethod callee = callTarget.targetMethod();
                     if (policy.shouldInline(callee, method)) {
-                        StructuredGraph targetGraph = makeGraph(callee, policy);
+                        StructuredGraph targetGraph = parseGraph(callee, policy);
                         InliningUtil.inline(invoke, targetGraph, true);
                         Debug.dump(graph, "after inlining %s", callee);
                         if (GraalOptions.OptCanonicalizer) {
-                            new WordTypeRewriterPhase(target).apply(graph);
+                            new WordTypeRewriterPhase(target.wordKind, runtime.getResolvedJavaType(target.wordKind)).apply(graph);
                             new CanonicalizerPhase(target, runtime, null).apply(graph);
                         }
                     }
@@ -173,7 +183,7 @@
 
                 new SnippetIntrinsificationPhase(runtime, pool).apply(graph);
 
-                new WordTypeRewriterPhase(target).apply(graph);
+                new WordTypeRewriterPhase(target.wordKind, runtime.getResolvedJavaType(target.wordKind)).apply(graph);
 
                 new DeadCodeEliminationPhase().apply(graph);
                 if (GraalOptions.OptCanonicalizer) {
@@ -186,8 +196,10 @@
 
                 new InsertStateAfterPlaceholderPhase().apply(graph);
 
-                Debug.dump(graph, "%s: Final", method.name());
-
+                if (GraalOptions.ProbabilityAnalysis) {
+                    new DeadCodeEliminationPhase().apply(graph);
+                    new ComputeProbabilityPhase().apply(graph);
+                }
                 return graph;
             }
         });
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.snippets/src/com/oracle/graal/snippets/SnippetIntrinsificationPhase.java
--- a/graal/com.oracle.graal.snippets/src/com/oracle/graal/snippets/SnippetIntrinsificationPhase.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.snippets/src/com/oracle/graal/snippets/SnippetIntrinsificationPhase.java	Tue Jun 26 10:56:03 2012 +0200
@@ -119,7 +119,7 @@
             }
             ValueNode argument = tryBoxingElimination(parameterIndex, target, arguments.get(i));
             if (folding || CodeUtil.getParameterAnnotation(ConstantNodeParameter.class, parameterIndex, target) != null) {
-                assert argument instanceof ConstantNode : "parameter " + parameterIndex + " must be a compile time constant for calling " + invoke.callTarget().targetMethod() + ": " + argument;
+                assert argument instanceof ConstantNode : "parameter " + parameterIndex + " must be a compile time constant for calling " + invoke.callTarget().targetMethod() + " at " + sourceLocation(invoke.node()) + ": " + argument;
                 ConstantNode constantNode = (ConstantNode) argument;
                 Constant constant = constantNode.asConstant();
                 Object o = constant.boxedValue();
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.snippets/src/com/oracle/graal/snippets/SnippetTemplate.java
--- a/graal/com.oracle.graal.snippets/src/com/oracle/graal/snippets/SnippetTemplate.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.snippets/src/com/oracle/graal/snippets/SnippetTemplate.java	Tue Jun 26 10:56:03 2012 +0200
@@ -270,7 +270,7 @@
                 if (loopBegin != null) {
                     LoopEx loop = new LoopsData(snippetCopy).loop(loopBegin);
                     int mark = snippetCopy.getMark();
-                    LoopTransformations.fullUnroll(loop);
+                    LoopTransformations.fullUnroll(loop, runtime);
                     new CanonicalizerPhase(null, runtime, null, mark, null).apply(snippetCopy);
                 }
                 FixedNode explodeLoopNext = explodeLoop.next();
@@ -458,7 +458,10 @@
         if (replacee instanceof FixedNode) {
             GraphUtil.killCFG((FixedNode) replacee);
         } else {
-            replacee.safeDelete();
+            GraphUtil.killWithUnusedFloatingInputs(replacee);
+        }
+        if (anchor != replacee) {
+            GraphUtil.killCFG(anchor);
         }
 
         Debug.dump(replaceeGraph, "After lowering %s with %s", replacee, this);
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.snippets/src/com/oracle/graal/snippets/SnippetVerificationPhase.java
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/graal/com.oracle.graal.snippets/src/com/oracle/graal/snippets/SnippetVerificationPhase.java	Tue Jun 26 10:56:03 2012 +0200
@@ -0,0 +1,134 @@
+/*
+ * Copyright (c) 2011, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.graal.snippets;
+
+import static com.oracle.graal.snippets.WordTypeRewriterPhase.*;
+
+import java.lang.reflect.*;
+
+import com.oracle.graal.api.meta.*;
+import com.oracle.graal.compiler.phases.*;
+import com.oracle.graal.graph.*;
+import com.oracle.graal.graph.Node.NodeIntrinsic;
+import com.oracle.graal.nodes.*;
+import com.oracle.graal.nodes.java.*;
+import com.oracle.graal.nodes.util.*;
+import com.oracle.graal.snippets.Word.*;
+
+/**
+ * Verifies invariants that must hold for snippet code above and beyond normal
+ * bytecode verification.
+ */
+public class SnippetVerificationPhase extends Phase {
+
+    @Override
+    protected void run(StructuredGraph graph) {
+        for (ValueNode node : graph.getNodes().filter(ValueNode.class)) {
+            for (Node usage : node.usages()) {
+                if (usage instanceof AccessMonitorNode) {
+                    verify(!isWord(node), node, usage, "word value has no monitor");
+                } else if (usage instanceof LoadFieldNode) {
+                    verify(!isWord(node) || ((LoadFieldNode) usage).object() != node, node, usage, "cannot load from word value");
+                } else if (usage instanceof StoreFieldNode) {
+                    verify(!isWord(node) || ((StoreFieldNode) usage).object() != node, node, usage, "cannot store to word value");
+                } else if (usage instanceof CheckCastNode) {
+                    verify(!isWord(node), node, usage, "word value cannot be cast");
+                    verify(!isWord(((CheckCastNode) usage).targetClass()), node, usage, "cannot cast to word value");
+                } else if (usage instanceof LoadIndexedNode) {
+                    verify(!isWord(node) || ((LoadIndexedNode) usage).array() != node, node, usage, "cannot load from word value");
+                    verify(!isWord(node) || ((LoadIndexedNode) usage).index() != node, node, usage, "cannot use word value as index");
+                } else if (usage instanceof StoreIndexedNode) {
+                    verify(!isWord(node) || ((StoreIndexedNode) usage).array() != node, node, usage, "cannot store to word value");
+                    verify(!isWord(node) || ((StoreIndexedNode) usage).index() != node, node, usage, "cannot use word value as index");
+                    verify(!isWord(node) || ((StoreIndexedNode) usage).value() != node, node, usage, "cannot store word value to array");
+                } else if (usage instanceof MethodCallTargetNode) {
+                    MethodCallTargetNode callTarget = (MethodCallTargetNode) usage;
+                    ResolvedJavaMethod method = callTarget.targetMethod();
+                    if (method.getAnnotation(NodeIntrinsic.class) == null) {
+                        Invoke invoke = (Invoke) callTarget.usages().first();
+                        NodeInputList<ValueNode> arguments = callTarget.arguments();
+                        boolean isStatic = Modifier.isStatic(method.accessFlags());
+                        int argc = 0;
+                        if (!isStatic) {
+                            ValueNode receiver = arguments.get(argc);
+                            if (receiver == node && isWord(node)) {
+                                Operation operation = method.getAnnotation(Word.Operation.class);
+                                verify(operation != null, node, invoke.node(), "cannot dispatch on word value to non @Operation annotated method " + method);
+                            }
+                            argc++;
+                        }
+                        Signature signature = method.signature();
+                        for (int i = 0; i < signature.argumentCount(false); i++) {
+                            ValueNode argument = arguments.get(argc);
+                            if (argument == node) {
+                                ResolvedJavaType type = (ResolvedJavaType) signature.argumentTypeAt(i, method.holder());
+                                verify((type.toJava() == Word.class) == isWord(argument), node, invoke.node(), "cannot pass word value to non-word parameter " + i + " or vice-versa");
+                            }
+                            argc++;
+                        }
+                    }
+                } else if (usage instanceof ArrayLengthNode) {
+                    verify(!isWord(node) || ((ArrayLengthNode) usage).array() != node, node, usage, "cannot get array length from word value");
+                } else if (usage instanceof PhiNode) {
+                    if (!(node instanceof MergeNode)) {
+                        PhiNode phi = (PhiNode) usage;
+                        for (ValueNode input : phi.values()) {
+                            verify(isWord(node) == isWord(input), node, input, "cannot merge word and non-word values");
+                        }
+                    }
+                }
+            }
+        }
+    }
+
+    private static void verify(boolean condition, Node node, Node usage, String message) {
+        if (!condition) {
+            error(node, usage, message);
+        }
+    }
+
+    private static void error(Node node, Node usage, String message) {
+        throw new GraalInternalError(String.format("Snippet verification error: %s" +
+                        "%n   node: %s (%s)" +
+                        "%n  usage: %s (%s)", message, node, sourceLocation(node), usage, sourceLocation(usage)));
+    }
+
+    private static String sourceLocation(Node n) {
+        if (n instanceof PhiNode) {
+            StringBuilder buf = new StringBuilder();
+            for (Node usage : n.usages()) {
+                String loc = sourceLocation(usage);
+                if (!loc.equals("<unknown>")) {
+                    if (buf.length() != 0) {
+                        buf.append(", ");
+                    }
+                    buf.append(loc);
+                }
+            }
+            return buf.toString();
+        } else {
+            String loc = GraphUtil.approxSourceLocation(n);
+            return loc == null ? "<unknown>" : loc;
+        }
+    }
+}
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.snippets/src/com/oracle/graal/snippets/Word.java
--- a/graal/com.oracle.graal.snippets/src/com/oracle/graal/snippets/Word.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.snippets/src/com/oracle/graal/snippets/Word.java	Tue Jun 26 10:56:03 2012 +0200
@@ -26,8 +26,6 @@
 
 import java.lang.annotation.*;
 
-import com.oracle.graal.nodes.calc.*;
-
 /**
  * Special type for use in snippets to represent machine word sized data.
  */
@@ -46,32 +44,137 @@
      * The canonical {@link Operation} represented by a method in the {@link Word} class.
      */
     public enum Opcode {
+        ZERO,
+        W2A,
+        A2W,
+        L2W,
+        I2W,
+        W2L,
+        W2I,
         PLUS,
         MINUS,
-        COMPARE;
+        BELOW,
+        BELOW_EQUAL,
+        ABOVE,
+        ABOVE_EQUAL;
+    }
+
+    private Word(long value, Object object) {
+        assert object == null || value == 0L;
+        this.value = value;
+        this.object = object;
+    }
+
+    private final long value;
+    private final Object object;
+
+    @Operation(ZERO)
+    public static Word zero() {
+        return new Word(0L, null);
+    }
+
+    @Operation(W2A)
+    public Object toObject() {
+        assert value == 0L;
+        return object;
+    }
+
+    @Operation(A2W)
+    public static Word fromObject(Object value) {
+        return new Word(0L, value);
+    }
+
+    @Operation(L2W)
+    public static Word fromLong(long value) {
+        return new Word(value, null);
     }
 
-    private Word() {
+    @Operation(I2W)
+    public static Word fromInt(int value) {
+        return new Word(value, null);
+    }
+
+    @Operation(W2I)
+    public int toInt() {
+        assert object == null;
+        return (int) value;
+    }
+
+    @Operation(W2L)
+    public long toLong() {
+        assert object == null;
+        return value;
+    }
+
+    @Operation(ABOVE)
+    public boolean above(Word other) {
+        assert object == null;
+        assert other.object == null;
+        long a = value;
+        long b = other.value;
+        return (a > b) ^ ((a < 0) != (b < 0));
     }
 
-    @Operation(COMPARE)
-    public native boolean cmp(Condition condition, Word other);
+    @Operation(ABOVE_EQUAL)
+    public boolean aboveOrEqual(Word other) {
+        assert object == null;
+        assert other.object == null;
+        long a = value;
+        long b = other.value;
+        return (a >= b) ^ ((a < 0) != (b < 0));
+    }
 
-    @Operation(PLUS)
-    public native Word plus(int addend);
+    @Operation(BELOW)
+    public boolean below(Word other) {
+        assert object == null;
+        assert other.object == null;
+        long a = value;
+        long b = other.value;
+        return (a < b) ^ ((a < 0) != (b < 0));
+    }
+
+    @Operation(BELOW_EQUAL)
+    public boolean belowOrEqual(Word other) {
+        assert object == null;
+        assert other.object == null;
+        long a = value;
+        long b = other.value;
+        return (a <= b) ^ ((a < 0) != (b < 0));
+    }
 
     @Operation(PLUS)
-    public native Word plus(long addend);
+    public Word plus(int addend) {
+        assert object == null;
+        return new Word(value + addend, null);
+    }
 
     @Operation(PLUS)
-    public native Word plus(Word addend);
+    public Word plus(long addend) {
+        assert object == null;
+        return new Word(value + addend, null);
+    }
+
+    @Operation(PLUS)
+    public Word plus(Word addend) {
+        assert object == null;
+        return new Word(value + addend.value, null);
+    }
 
     @Operation(MINUS)
-    public native Word minus(int addend);
+    public Word minus(int addend) {
+        assert object == null;
+        return new Word(value - addend, null);
+    }
 
     @Operation(MINUS)
-    public native Word minus(long addend);
+    public Word minus(long addend) {
+        assert object == null;
+        return new Word(value - addend, null);
+    }
 
     @Operation(MINUS)
-    public native Word minus(Word addend);
+    public Word minus(Word addend) {
+        assert object == null;
+        return new Word(value - addend.value, null);
+    }
 }
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.snippets/src/com/oracle/graal/snippets/WordTypeRewriterPhase.java
--- a/graal/com.oracle.graal.snippets/src/com/oracle/graal/snippets/WordTypeRewriterPhase.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.snippets/src/com/oracle/graal/snippets/WordTypeRewriterPhase.java	Tue Jun 26 10:56:03 2012 +0200
@@ -22,12 +22,11 @@
  */
 package com.oracle.graal.snippets;
 
-import com.oracle.graal.api.code.*;
 import com.oracle.graal.api.meta.*;
 import com.oracle.graal.compiler.phases.*;
 import com.oracle.graal.graph.*;
 import com.oracle.graal.nodes.*;
-import com.oracle.graal.nodes.PhiNode.*;
+import com.oracle.graal.nodes.PhiNode.PhiType;
 import com.oracle.graal.nodes.calc.*;
 import com.oracle.graal.nodes.calc.ConvertNode.Op;
 import com.oracle.graal.nodes.extended.*;
@@ -44,9 +43,11 @@
 public class WordTypeRewriterPhase extends Phase {
 
     private final Kind wordKind;
+    private final ResolvedJavaType wordType;
 
-    public WordTypeRewriterPhase(TargetDescription target) {
-        this.wordKind = target.wordKind;
+    public WordTypeRewriterPhase(Kind wordKind, ResolvedJavaType wordType) {
+        this.wordKind = wordKind;
+        this.wordType = wordType;
     }
 
     @Override
@@ -60,14 +61,6 @@
             }
         }
 
-        // Remove all checkcasts to Word
-        for (CheckCastNode checkCastNode : graph.getNodes(CheckCastNode.class).snapshot()) {
-            if (!checkCastNode.object().stamp().kind().isObject()) {
-                checkCastNode.replaceAtUsages(checkCastNode.object());
-                graph.removeFixed(checkCastNode);
-            }
-        }
-
         // Remove unnecessary/redundant unsafe casts
         for (UnsafeCastNode unsafeCastNode : graph.getNodes().filter(UnsafeCastNode.class).snapshot()) {
             if (!unsafeCastNode.isDeleted() && unsafeCastNode.object().stamp() == unsafeCastNode.stamp()) {
@@ -81,15 +74,37 @@
             if (operation != null) {
                 NodeInputList<ValueNode> arguments = callTargetNode.arguments();
                 Invoke invoke = (Invoke) callTargetNode.usages().first();
-                assert invoke != null;
+                assert invoke != null : callTargetNode.targetMethod();
 
                 Opcode opcode = operation.value();
                 switch (opcode) {
-                    case COMPARE: {
-                        assert arguments.size() == 3;
-                        assert arguments.get(1) instanceof ConstantNode;
-                        Condition condition = (Condition) arguments.get(1).asConstant().asObject();
-                        invoke.intrinsify(compare(condition, graph, arguments.first(), arguments.last()));
+                    case ZERO: {
+                        assert arguments.size() == 0;
+                        invoke.intrinsify(wordKind.isLong() ? ConstantNode.forLong(0L, graph) : ConstantNode.forInt(0, graph));
+                        break;
+                    }
+
+                    case ABOVE: {
+                        assert arguments.size() == 2;
+                        invoke.intrinsify(compare(Condition.AT, graph, arguments.first(), arguments.last()));
+                        break;
+                    }
+
+                    case ABOVE_EQUAL: {
+                        assert arguments.size() == 2;
+                        invoke.intrinsify(compare(Condition.AE, graph, arguments.first(), arguments.last()));
+                        break;
+                    }
+
+                    case BELOW: {
+                        assert arguments.size() == 2;
+                        invoke.intrinsify(compare(Condition.BT, graph, arguments.first(), arguments.last()));
+                        break;
+                    }
+
+                    case BELOW_EQUAL: {
+                        assert arguments.size() == 2;
+                        invoke.intrinsify(compare(Condition.BE, graph, arguments.first(), arguments.last()));
                         break;
                     }
 
@@ -99,6 +114,58 @@
                         invoke.intrinsify(op);
                         break;
                     }
+
+                    case W2A: {
+                        assert arguments.size() == 1;
+                        ValueNode value = arguments.first();
+                        ResolvedJavaType targetType = (ResolvedJavaType) targetMethod.signature().returnType(targetMethod.holder());
+                        UnsafeCastNode cast = graph.unique(new UnsafeCastNode(value, targetType));
+                        invoke.intrinsify(cast);
+                        break;
+                    }
+
+                    case W2I: {
+                        assert arguments.size() == 1;
+                        ValueNode value = arguments.first();
+                        ValueNode intValue = fromWordKindTo(graph, value, Kind.Int);
+                        invoke.intrinsify(intValue);
+                        break;
+                    }
+
+                    case W2L: {
+                        assert arguments.size() == 1;
+                        ValueNode value = arguments.first();
+                        ValueNode longValue = fromWordKindTo(graph, value, Kind.Long);
+                        invoke.intrinsify(longValue);
+                        break;
+                    }
+
+                    case A2W: {
+                        assert arguments.size() == 1;
+                        ValueNode value = arguments.first();
+                        assert value.kind() == Kind.Object : value + ", " + targetMethod;
+                        UnsafeCastNode cast = graph.unique(new UnsafeCastNode(value, wordType));
+                        invoke.intrinsify(cast);
+                        break;
+                    }
+
+                    case L2W: {
+                        assert arguments.size() == 1;
+                        ValueNode value = arguments.first();
+                        assert value.kind() == Kind.Long;
+                        ValueNode wordValue = asWordKind(graph, value);
+                        invoke.intrinsify(wordValue);
+                        break;
+                    }
+
+                    case I2W: {
+                        assert arguments.size() == 1;
+                        ValueNode value = arguments.first();
+                        assert value.kind() == Kind.Int;
+                        invoke.intrinsify(asWordKind(graph, value));
+                        break;
+                    }
+
                     default: {
                         throw new GraalInternalError("Unknown opcode: %s", opcode);
                     }
@@ -111,7 +178,7 @@
      * Creates comparison node for a given condition and two input values.
      */
     private ValueNode compare(Condition condition, StructuredGraph graph, ValueNode left, ValueNode right) {
-        assert condition.isUnsigned();
+        assert condition.isUnsigned() : condition;
         assert left.kind() == wordKind;
         assert right.kind() == wordKind;
 
@@ -121,12 +188,7 @@
         ValueNode a = mirror ? right : left;
         ValueNode b = mirror ? left : right;
 
-        MaterializeNode materialize;
-        if (condition == Condition.EQ || condition == Condition.NE) {
-            materialize = MaterializeNode.create(graph.unique(new IntegerEqualsNode(a, b)), graph);
-        } else {
-            materialize = MaterializeNode.create(graph.unique(new IntegerBelowThanNode(a, b)), graph);
-        }
+        MaterializeNode materialize = MaterializeNode.create(graph.unique(new IntegerBelowThanNode(a, b)), graph);
 
         ValueNode op;
         if (condition.canonicalNegate()) {
@@ -158,11 +220,29 @@
         return value;
     }
 
-    public boolean isWord(ValueNode node) {
-        return isWord(node.stamp().declaredType());
+    private static ValueNode fromWordKindTo(StructuredGraph graph, ValueNode value, Kind to) {
+        Kind from = value.kind();
+        if (from != to) {
+            Op op;
+            if (from.isLong()) {
+                op = Op.L2I;
+            } else {
+                assert from.isInt();
+                op = Op.I2L;
+            }
+            return graph.unique(new ConvertNode(op, value));
+        }
+        return value;
     }
 
-    public boolean isWord(ResolvedJavaType type) {
+    public static boolean isWord(ValueNode node) {
+        if (node.kind().isObject()) {
+            return isWord(node.objectStamp().type());
+        }
+        return false;
+    }
+
+    public static boolean isWord(ResolvedJavaType type) {
         if (type != null && type.toJava() == Word.class) {
             return true;
         }
@@ -179,10 +259,6 @@
                 changeToWord((ValueNode) n);
                 PhiNode phi = (PhiNode) n;
                 assert phi.type() == PhiType.Value;
-//                    for (ValueNode v : phi.values()) {
-//                        assertTrue(v.kind() == phi.kind(), "all phi values must have same kind");
-//                    }
-
             } else if (n instanceof ReturnNode) {
                 changeToWord((ValueNode) n);
             }
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.tests/src/com/oracle/graal/compiler/tests/CompiledMethodTest.java
--- a/graal/com.oracle.graal.tests/src/com/oracle/graal/compiler/tests/CompiledMethodTest.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.tests/src/com/oracle/graal/compiler/tests/CompiledMethodTest.java	Tue Jun 26 10:56:03 2012 +0200
@@ -67,7 +67,7 @@
         }
 
         final ResolvedJavaMethod riMethod = runtime.getResolvedJavaMethod(method);
-        InstalledCode compiledMethod = compile(riMethod, graph);
+        InstalledCode compiledMethod = getCode(riMethod, graph);
         try {
             Object result = compiledMethod.execute("1", "2", "3");
             Assert.assertEquals("1-2-3", result);
@@ -81,7 +81,7 @@
         Method method = getMethod("testMethod");
         final StructuredGraph graph = parse(method);
         final ResolvedJavaMethod riMethod = runtime.getResolvedJavaMethod(method);
-        InstalledCode compiledMethod = compile(riMethod, graph);
+        InstalledCode compiledMethod = getCode(riMethod, graph);
         try {
             Object result = compiledMethod.executeVarargs("1", "2", "3");
             Assert.assertEquals("1 2 3", result);
@@ -95,7 +95,7 @@
         Method method = getMethod("testMethodVirtual");
         final StructuredGraph graph = parse(method);
         final ResolvedJavaMethod riMethod = runtime.getResolvedJavaMethod(method);
-        InstalledCode compiledMethod = compile(riMethod, graph);
+        InstalledCode compiledMethod = getCode(riMethod, graph);
         try {
             f1 = "0";
             Object result = compiledMethod.executeVarargs(this, "1", "2", "3");
@@ -123,7 +123,7 @@
             }
         }
 
-        InstalledCode compiledMethod = compile(riMethod, graph);
+        InstalledCode compiledMethod = getCode(riMethod, graph);
         final CompilableObject compilableObject = new CompilableObjectImpl(0);
 
         Object result;
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.tests/src/com/oracle/graal/compiler/tests/EscapeAnalysisTest.java
--- a/graal/com.oracle.graal.tests/src/com/oracle/graal/compiler/tests/EscapeAnalysisTest.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.tests/src/com/oracle/graal/compiler/tests/EscapeAnalysisTest.java	Tue Jun 26 10:56:03 2012 +0200
@@ -117,7 +117,7 @@
     }
 
     private void test(final String snippet, final Constant expectedResult) {
-        Debug.scope("ScalarTypeSystemTest", new DebugDumpScope(snippet), new Runnable() {
+        Debug.scope("EscapeAnalysisTest", new DebugDumpScope(snippet), new Runnable() {
             public void run() {
                 StructuredGraph graph = parse(snippet);
                 for (Invoke n : graph.getInvokes()) {
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.tests/src/com/oracle/graal/compiler/tests/GraalCompilerTest.java
--- a/graal/com.oracle.graal.tests/src/com/oracle/graal/compiler/tests/GraalCompilerTest.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.tests/src/com/oracle/graal/compiler/tests/GraalCompilerTest.java	Tue Jun 26 10:56:03 2012 +0200
@@ -23,6 +23,7 @@
 package com.oracle.graal.compiler.tests;
 
 import java.lang.reflect.*;
+import java.util.*;
 import java.util.concurrent.*;
 
 import junit.framework.*;
@@ -84,7 +85,16 @@
         }
     }
 
-    private static String getCanonicalGraphString(StructuredGraph graph) {
+    protected void assertConstantReturn(StructuredGraph graph, int value) {
+        String graphString = getCanonicalGraphString(graph);
+        Assert.assertEquals("unexpected number of ReturnNodes: " + graphString, graph.getNodes(ReturnNode.class).count(), 1);
+        ValueNode result = graph.getNodes(ReturnNode.class).first().result();
+        Assert.assertTrue("unexpected ReturnNode result node: " + graphString, result.isConstant());
+        Assert.assertEquals("unexpected ReturnNode result kind: " + graphString, result.asConstant().kind, Kind.Int);
+        Assert.assertEquals("unexpected ReturnNode result: " + graphString, result.asConstant().asInt(), value);
+    }
+
+    protected static String getCanonicalGraphString(StructuredGraph graph) {
         SchedulePhase schedule = new SchedulePhase();
         schedule.apply(graph);
 
@@ -163,8 +173,7 @@
         } catch (Exception e) {
             throw new RuntimeException(e);
         }
-        InstalledCode compiledMethod = compile(runtime.getResolvedJavaMethod(method), parse(method));
-        compiledMethod.method();
+        InstalledCode compiledMethod = getCode(runtime.getResolvedJavaMethod(method), parse(method));
 
         if (exception != null) {
             try {
@@ -179,13 +188,35 @@
         }
     }
 
-    protected InstalledCode compile(final ResolvedJavaMethod method, final StructuredGraph graph) {
-        return Debug.scope("Compiling", new DebugDumpScope(String.valueOf(compilationId++), true), new Callable<InstalledCode>() {
+    private Map<ResolvedJavaMethod, InstalledCode> cache = new HashMap<>();
+
+    /**
+     * Gets installed code for a given method and graph, compiling it first if necessary.
+     */
+    protected InstalledCode getCode(final ResolvedJavaMethod method, final StructuredGraph graph) {
+        return getCode(method, graph, false);
+    }
+
+    /**
+     * Gets installed code for a given method and graph, compiling it first if necessary.
+     *
+     * @param forceCompile specifies whether to ignore any previous code cached for the (method, key) pair
+     */
+    protected InstalledCode getCode(final ResolvedJavaMethod method, final StructuredGraph graph, boolean forceCompile) {
+        if (!forceCompile) {
+            InstalledCode cached = cache.get(method);
+            if (cached != null && cached.isValid()) {
+                return cached;
+            }
+        }
+        InstalledCode installedCode = Debug.scope("Compiling", new DebugDumpScope(String.valueOf(compilationId++), true), new Callable<InstalledCode>() {
             public InstalledCode call() throws Exception {
                 CompilationResult targetMethod = runtime.compile(method, graph);
                 return addMethod(method, targetMethod);
             }
         });
+        cache.put(method, installedCode);
+        return installedCode;
     }
 
     protected InstalledCode addMethod(final ResolvedJavaMethod method, final CompilationResult tm) {
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.tests/src/com/oracle/graal/compiler/tests/NewInstanceTest.java
--- a/graal/com.oracle.graal.tests/src/com/oracle/graal/compiler/tests/NewInstanceTest.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.tests/src/com/oracle/graal/compiler/tests/NewInstanceTest.java	Tue Jun 26 10:56:03 2012 +0200
@@ -53,6 +53,7 @@
         test("newEmptyString");
         test("newString", "value");
         test("newHashMap", 31);
+        test("newRegression", true);
     }
 
     public static Object newObject() {
@@ -149,4 +150,39 @@
         Object f44;
         Object f45;
     }
+
+    /**
+     * Tests that an earlier bug does not occur. The issue was that the loading of the TLAB
+     * 'top' and 'end' values was being GVN'ed from each branch of the 'if' statement.
+     * This meant that the allocated B object in the true branch overwrote the allocated
+     * array. The cause is that RegisterNode was a floating node and the reads from it
+     * were UnsafeLoads which are also floating. The fix was to make RegisterNode a fixed
+     * node (which it should have been in the first place).
+     */
+    public static Object newRegression(boolean condition) {
+        Object result;
+        if (condition) {
+            Object[] arr = {0, 1, 2, 3, 4, 5};
+            result = new B();
+            for (int i = 0; i < arr.length; ++i) {
+                // If the bug exists, the values of arr will now be deadbeef values
+                // and the virtual dispatch will cause a segfault. This can result in
+                // either a VM crash or a spurious NullPointerException.
+                if (arr[i].equals(Integer.valueOf(i))) {
+                    return false;
+                }
+            }
+        } else {
+            result = new B();
+        }
+        return result;
+    }
+
+    static class B {
+        long f1 = 0xdeadbeefdeadbe01L;
+        long f2 = 0xdeadbeefdeadbe02L;
+        long f3 = 0xdeadbeefdeadbe03L;
+        long f4 = 0xdeadbeefdeadbe04L;
+        long f5 = 0xdeadbeefdeadbe05L;
+    }
 }
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.tests/src/com/oracle/graal/compiler/tests/StampCanonicalizerTest.java
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/graal/com.oracle.graal.tests/src/com/oracle/graal/compiler/tests/StampCanonicalizerTest.java	Tue Jun 26 10:56:03 2012 +0200
@@ -0,0 +1,104 @@
+/*
+ * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.graal.compiler.tests;
+
+import org.junit.*;
+
+import com.oracle.graal.compiler.phases.*;
+import com.oracle.graal.nodes.*;
+import com.oracle.graal.nodes.type.*;
+
+/**
+ * This class tests some specific patterns the stamp system should be able to canonicalize away using
+ * {@link IntegerStamp#mask()}.
+ */
+public class StampCanonicalizerTest extends GraalCompilerTest {
+
+    public static int andStamp(int a, int b) {
+        int v = (a & 0xffff00) & (b & 0xff0000f);
+        return v & 1;
+    }
+
+    @Test
+    public void testAnd() {
+        testZeroReturn("andStamp");
+    }
+
+    public static int shiftLeftStamp1(int a) {
+        int v = a << 1;
+        return v & 1;
+    }
+
+    public static int shiftLeftStamp2(int a) {
+        int v = a << 1;
+        if (a == 17) {
+            v = a * 4;
+        }
+        return v & 1;
+    }
+
+    @Test
+    public void testShift() {
+        testZeroReturn("shiftLeftStamp1");
+        testZeroReturn("shiftLeftStamp2");
+    }
+
+    public static int upperBoundShiftStamp1(int a) {
+        int v = a & 0xffff;
+        return (v << 4) & 0xff00000;
+    }
+
+    public static int upperBoundShiftStamp2(int a) {
+        int v = a & 0xffff;
+        return (v >> 4) & 0xf000;
+    }
+
+    @Test
+    public void testUpperBoundShift() {
+        testZeroReturn("upperBoundShiftStamp1");
+        testZeroReturn("upperBoundShiftStamp2");
+    }
+
+    public static int divStamp1(int[] a) {
+        int v = a.length / 4;
+        return v & 0x80000000;
+    }
+
+    public static int divStamp2(int[] a) {
+        int v = a.length / 5;
+        return v & 0x80000000;
+    }
+
+    @Test
+    public void testDiv() {
+        testZeroReturn("divStamp1");
+        testZeroReturn("divStamp2");
+    }
+
+    private void testZeroReturn(String methodName) {
+        StructuredGraph graph = parse(methodName);
+        new CanonicalizerPhase(null, runtime(), null).apply(graph);
+        new DeadCodeEliminationPhase().apply(graph);
+        assertConstantReturn(graph, 0);
+    }
+}
diff -r bb94f57c822b -r 7ee5a3634003 graal/com.oracle.graal.tests/src/com/oracle/graal/compiler/tests/TypeCheckTest.java
--- a/graal/com.oracle.graal.tests/src/com/oracle/graal/compiler/tests/TypeCheckTest.java	Mon Jun 18 00:29:37 2012 +0200
+++ b/graal/com.oracle.graal.tests/src/com/oracle/graal/compiler/tests/TypeCheckTest.java	Tue Jun 26 10:56:03 2012 +0200
@@ -36,11 +36,13 @@
     protected JavaTypeProfile currentProfile;
 
     @Override
-    protected InstalledCode compile(final ResolvedJavaMethod method, final StructuredGraph graph) {
+    protected InstalledCode getCode(final ResolvedJavaMethod method, final StructuredGraph graph) {
+        boolean forceCompile = false;
         if (currentProfile != null) {
             replaceProfile(graph, currentProfile);
+            forceCompile = true;
         }
-        return super.compile(method, graph);
+        return super.getCode(method, graph, forceCompile);
     }
 
     protected JavaTypeProfile profile(Class... types) {
diff -r bb94f57c822b -r 7ee5a3634003 mx/commands.py
--- a/mx/commands.py	Mon Jun 18 00:29:37 2012 +0200
+++ b/mx/commands.py	Tue Jun 26 10:56:03 2012 +0200
@@ -175,7 +175,7 @@
     """run one or all DaCapo benchmarks
     
     DaCapo options are distinguished from VM options by a '@' prefix.
-    For example, '@--iterations @5' will pass '--iterations 5' to the
+    For example, '@-n @5' will pass '-n 5' to the
     DaCapo harness."""
 
     numTests = {}
@@ -286,13 +286,23 @@
 
 def _vmLibDirInJdk(jdk):
     """
-    Get the directory within a JDK where jvm.cfg file and the server
-    and client subdirectories are located.
+    Get the directory within a JDK where the server and client 
+    subdirectories are located.
     """
     if platform.system() == 'Darwin':
         return join(jdk, 'jre', 'lib')
+    if platform.system() == 'Windows':
+        return join(jdk, 'jre', 'bin')
     return join(jdk, 'jre', 'lib', 'amd64')
 
+def _vmCfgInJdk(jdk):
+    """
+    Get the jvm.cfg file.
+    """
+    if platform.system() == 'Windows':
+        return join(jdk, 'jre', 'lib', 'amd64', 'jvm.cfg')
+    return join(_vmLibDirInJdk(jdk), 'jvm.cfg')
+
 def _jdk(build='product', create=False):
     """
     Get the JDK into which Graal is installed, creating it first if necessary.
@@ -315,7 +325,7 @@
                 
             # Make a copy of the default VM so that this JDK can be
             # reliably used as the bootstrap for a HotSpot build.                
-            jvmCfg = join(_vmLibDirInJdk(jdk), 'jvm.cfg')
+            jvmCfg = _vmCfgInJdk(jdk)
             if not exists(jvmCfg):
                 mx.abort(jvmCfg + ' does not exist')
                 
@@ -364,9 +374,9 @@
         log = open(logFile, 'w')
     ret = False
     while True:
-        line = stdout.readline().decode()
+        line = stdout.readline().decode(sys.stdout.encoding)
         if logFile:
-            log.write(line)
+            log.write(line.encode('utf-8'))
         line = line.strip()
         mx.log(line)
         if line == STARTTOKEN:
@@ -380,7 +390,14 @@
             if match:
                 ret = True
         if line == ENDTOKEN:
-            break
+            if not findInOutput:
+                stdin.write('echo ERR%errorlevel%' + newLine)
+            else:
+                break
+        if line.startswith('ERR'):
+            if line == 'ERR0':
+                ret = True
+            break;
     stdin.write('exit' + newLine)
     if logFile:
         log.close()
@@ -495,8 +512,7 @@
                 mx.log('Error executing create command')
                 return 
             winBuildCmd = 'msbuild ' + _graal_home + r'\build\vs-amd64\jvm.vcxproj /p:Configuration=' + project_config + ' /p:Platform=x64'
-            winBuildSuccess = re.compile('Build succeeded.')
-            if not _runInDebugShell(winBuildCmd, _graal_home, compilelogfile, winBuildSuccess):
+            if not _runInDebugShell(winBuildCmd, _graal_home, compilelogfile):
                 mx.log('Error building project')
                 return 
         else:
@@ -516,7 +532,7 @@
             
             mx.run([mx.gmake_cmd(), build + buildSuffix], cwd=join(_graal_home, 'make'), err=filterXusage)
         
-        jvmCfg = join(_vmLibDirInJdk(jdk), 'jvm.cfg')
+        jvmCfg = _vmCfgInJdk(jdk)
         found = False
         if not exists(jvmCfg):
             mx.abort(jvmCfg + ' does not exist')
@@ -624,26 +640,7 @@
                                 elif e == basename + '.class':
                                     classes.append(pkg + '.' + basename)
 
-
-# Table of unit tests.
-# Keys are project names, values are package name lists.
-# All source files in the given (project,package) pairs are scanned for lines
-# containing '@Test'. These are then determined to be the classes defining
-# unit tests.
-_unittests = {
-    'com.oracle.graal.tests': ['com.oracle.graal.compiler.tests'],
-}
-_jtttests = {
-    'com.oracle.graal.jtt': ['com.oracle.graal.jtt'],
-}
-
-def unittest(args):
-    """run the Graal Compiler Unit Tests in the GraalVM
-    
-    If filters are supplied, only tests whose fully qualified name
-    include a filter as a substring are run. Negative filters are
-    those with a '-' prefix. VM args should have a @ prefix."""
-    
+def _run_tests(args, harnessName, harness):
     pos = [a for a in args if a[0] != '-' and a[0] != '@' ]
     neg = [a[1:] for a in args if a[0] == '-']
     vmArgs = [a[1:] for a in args if a[0] == '@']
@@ -654,18 +651,30 @@
                 return True
         return False
     
-    for proj in _unittests.iterkeys():
-        p = mx.project(proj)
-        classes = []
-        for pkg in _unittests[proj]:
-            _find_classes_with_annotations(classes, p, pkg, ['@Test'])
+    for p in mx.projects():
+        if getattr(p, 'testHarness', None) == harnessName:
+            classes = []
+            _find_classes_with_annotations(classes, p, None, ['@Test'])
+        
+            if len(pos) != 0:
+                classes = [c for c in classes if containsAny(c, pos)]
+            if len(neg) != 0:
+                classes = [c for c in classes if not containsAny(c, neg)]
+            
+            if len(classes) != 0:
+                mx.log('running tests in ' + p.name)
+                harness(p, vmArgs, classes)                
+
+def unittest(args):
+    """run the Graal Compiler Unit Tests in the GraalVM
     
-        if len(pos) != 0:
-            classes = [c for c in classes if containsAny(c, pos)]
-        if len(neg) != 0:
-            classes = [c for c in classes if not containsAny(c, neg)]
-        
-        vm(['-XX:-BootstrapGraal', '-esa'] + vmArgs + ['-cp', mx.classpath(proj), 'org.junit.runner.JUnitCore'] + classes)
+    If filters are supplied, only tests whose fully qualified name
+    include a filter as a substring are run. Negative filters are
+    those with a '-' prefix. VM args should have a @ prefix."""
+    
+    def harness(p, vmArgs, classes):
+        vm(['-XX:-BootstrapGraal', '-esa'] + vmArgs + ['-cp', mx.classpath(p.name), 'org.junit.runner.JUnitCore'] + classes)
+    _run_tests(args, 'unittest', harness)
     
 def jtt(args):
     """run the Java Tester Tests in the GraalVM
@@ -674,28 +683,9 @@
     include a filter as a substring are run. Negative filters are
     those with a '-' prefix. VM args should have a @ prefix."""
     
-    pos = [a for a in args if a[0] != '-' and a[0] != '@' ]
-    neg = [a[1:] for a in args if a[0] == '-']
-    vmArgs = [a[1:] for a in args if a[0] == '@']
-
-    def containsAny(c, substrings):
-        for s in substrings:
-            if s in c:
-                return True
-        return False
-    
-    for proj in _jtttests.iterkeys():
-        p = mx.project(proj)
-        classes = []
-        for pkg in _jtttests[proj]:
-            _find_classes_with_annotations(classes, p, pkg, ['@Test'])
-    
-        if len(pos) != 0:
-            classes = [c for c in classes if containsAny(c, pos)]
-        if len(neg) != 0:
-            classes = [c for c in classes if not containsAny(c, neg)]
-            
-        vm(['-XX:-BootstrapGraal', '-XX:CompileOnly=com/oracle/graal/jtt', '-XX:CompileCommand=compileonly,java/lang/Object::<init>', '-XX:CompileCommand=quiet', '-Xcomp', '-esa'] + vmArgs + ['-cp', mx.classpath(proj), 'org.junit.runner.JUnitCore'] + classes)
+    def harness(p, vmArgs, classes):
+        vm(['-XX:-BootstrapGraal', '-XX:CompileOnly=com/oracle/graal/jtt', '-XX:CompileCommand=compileonly,java/lang/Object::<init>', '-XX:CompileCommand=quiet', '-Xcomp', '-esa'] + vmArgs + ['-cp', mx.classpath(p.name), 'org.junit.runner.JUnitCore'] + classes)
+    _run_tests(args, 'jtt', harness)
     
 def buildvms(args):
     """build one or more VMs in various configurations"""
@@ -798,7 +788,7 @@
                 _jacoco = 'append'
             
             t = Task('JavaTesterTests:' + vmbuild)
-            jtt([])
+            jtt(['@-XX:CompileCommand=exclude,*::run*'] if vmbuild == 'product'  else [])
             tasks.append(t.stop())
             
             if vmbuild == 'product' and args.jacocout is not None:
diff -r bb94f57c822b -r 7ee5a3634003 mx/projects
--- a/mx/projects	Mon Jun 18 00:29:37 2012 +0200
+++ b/mx/projects	Tue Jun 26 10:56:03 2012 +0200
@@ -119,9 +119,10 @@
 # graal.snippets.test
 project@com.oracle.graal.snippets.test@subDir=graal
 project@com.oracle.graal.snippets.test@sourceDirs=src
-project@com.oracle.graal.snippets.test@dependencies=com.oracle.graal.snippets
+project@com.oracle.graal.snippets.test@dependencies=com.oracle.graal.snippets,com.oracle.graal.tests
 project@com.oracle.graal.snippets.test@checkstyle=com.oracle.graal.graph
 project@com.oracle.graal.snippets.test@javaCompliance=1.7
+project@com.oracle.graal.snippets.test@testHarness=unittest
 
 # graal.nodes
 project@com.oracle.graal.nodes@subDir=graal
@@ -184,6 +185,7 @@
 project@com.oracle.graal.tests@dependencies=JUNIT,com.oracle.graal.printer,com.oracle.graal.api
 project@com.oracle.graal.tests@checkstyle=com.oracle.graal.graph
 project@com.oracle.graal.tests@javaCompliance=1.7
+project@com.oracle.graal.tests@testHarness=unittest
 
 # graal.jtt
 project@com.oracle.graal.jtt@subDir=graal
@@ -191,6 +193,7 @@
 project@com.oracle.graal.jtt@dependencies=JUNIT
 project@com.oracle.graal.jtt@checkstyle=com.oracle.graal.graph
 project@com.oracle.graal.jtt@javaCompliance=1.7
+project@com.oracle.graal.jtt@testHarness=jtt
 
 # graal.examples
 project@com.oracle.graal.examples@subDir=graal
diff -r bb94f57c822b -r 7ee5a3634003 src/cpu/x86/vm/c1_Runtime1_x86.cpp
--- a/src/cpu/x86/vm/c1_Runtime1_x86.cpp	Mon Jun 18 00:29:37 2012 +0200
+++ b/src/cpu/x86/vm/c1_Runtime1_x86.cpp	Tue Jun 26 10:56:03 2012 +0200
@@ -1956,7 +1956,10 @@
     }
 
     case graal_verify_oop_id: {
+      // We use enter & leave so that a better stack trace is produced in the hs_err file
+      __ enter();
       __ verify_oop(r13, "graal verify oop");
+      __ leave();
       __ ret(0);
       break;
     }
diff -r bb94f57c822b -r 7ee5a3634003 src/share/vm/graal/graalCodeInstaller.cpp
--- a/src/share/vm/graal/graalCodeInstaller.cpp	Mon Jun 18 00:29:37 2012 +0200
+++ b/src/share/vm/graal/graalCodeInstaller.cpp	Tue Jun 26 10:56:03 2012 +0200
@@ -780,7 +780,9 @@
 
       if (obj->is_a(HotSpotKlassOop::klass())) {
         assert(!obj.is_null(), "");
-        *((jobject*) operand) = JNIHandles::make_local(java_lang_Class::as_klassOop(HotSpotKlassOop::javaMirror(obj)));
+        oop type = HotSpotKlassOop::type(obj);
+        klassOop klass = java_lang_Class::as_klassOop(HotSpotResolvedJavaType::javaMirror(type));
+        *((jobject*) operand) = JNIHandles::make_local(klass);
         _instructions->relocate(instruction, oop_Relocation::spec_for_immediate(), Assembler::imm_operand);
         TRACE_graal_3("relocating (HotSpotJavaType) at %016x/%016x", instruction, operand);
       } else {
diff -r bb94f57c822b -r 7ee5a3634003 src/share/vm/graal/graalCompiler.cpp
--- a/src/share/vm/graal/graalCompiler.cpp	Mon Jun 18 00:29:37 2012 +0200
+++ b/src/share/vm/graal/graalCompiler.cpp	Tue Jun 26 10:56:03 2012 +0200
@@ -273,6 +273,7 @@
   HotSpotResolvedJavaType::set_accessFlags(obj, klass->access_flags().as_int());
   HotSpotResolvedJavaType::set_isInterface(obj, klass->is_interface());
   HotSpotResolvedJavaType::set_superCheckOffset(obj, klass->super_check_offset());
+  HotSpotResolvedJavaType::set_prototypeHeader(obj, (jlong) klass->prototype_header());
   HotSpotResolvedJavaType::set_isInstanceClass(obj, klass->oop_is_instance());
 
   if (klass->oop_is_javaArray()) {
diff -r bb94f57c822b -r 7ee5a3634003 src/share/vm/graal/graalJavaAccess.hpp
--- a/src/share/vm/graal/graalJavaAccess.hpp	Mon Jun 18 00:29:37 2012 +0200
+++ b/src/share/vm/graal/graalJavaAccess.hpp	Tue Jun 26 10:56:03 2012 +0200
@@ -49,6 +49,7 @@
     oop_field(HotSpotResolvedJavaType, javaMirror, "Ljava/lang/Class;")                     \
     oop_field(HotSpotResolvedJavaType, simpleName, "Ljava/lang/String;")                    \
     int_field(HotSpotResolvedJavaType, accessFlags)                                         \
+    long_field(HotSpotResolvedJavaType, prototypeHeader)                                    \
     boolean_field(HotSpotResolvedJavaType, hasFinalizer)                                    \
     boolean_field(HotSpotResolvedJavaType, hasFinalizableSubclass)                          \
     int_field(HotSpotResolvedJavaType, superCheckOffset)                                    \
@@ -56,10 +57,10 @@
     boolean_field(HotSpotResolvedJavaType, isInstanceClass)                                 \
     boolean_field(HotSpotResolvedJavaType, isInterface)                                     \
     int_field(HotSpotResolvedJavaType, instanceSize)                                        \
-  end_class                                                                             \
-  start_class(HotSpotKlassOop)                                                          \
-    oop_field(HotSpotKlassOop, javaMirror, "Ljava/lang/Class;")                         \
-    end_class                                                                           \
+  end_class                                                                                 \
+  start_class(HotSpotKlassOop)                                                              \
+    oop_field(HotSpotKlassOop, type, "Lcom/oracle/graal/api/meta/ResolvedJavaType;")        \
+    end_class                                                                               \
   start_class(HotSpotResolvedJavaMethod)                                                    \
     oop_field(HotSpotResolvedJavaMethod, name, "Ljava/lang/String;")                        \
     oop_field(HotSpotResolvedJavaMethod, holder, "Lcom/oracle/graal/api/meta/ResolvedJavaType;")  \