# HG changeset patch
# User Thomas Wuerthinger <thomas.wuerthinger@oracle.com>
# Date 1373291664 -7200
# Node ID 192a3b3c7292b46319fab496341acf7bb082c359
# Parent  b6e46324233f3f64682b91ae024607fdccddd149# Parent  87c441b324e9152b946e68c869f17cfe34fdef9a
Merge.

diff -r b6e46324233f -r 192a3b3c7292 graal/com.oracle.graal.compiler.test/src/com/oracle/graal/compiler/test/inlining/InliningTest.java
--- a/graal/com.oracle.graal.compiler.test/src/com/oracle/graal/compiler/test/inlining/InliningTest.java	Mon Jul 08 00:05:30 2013 +0200
+++ b/graal/com.oracle.graal.compiler.test/src/com/oracle/graal/compiler/test/inlining/InliningTest.java	Mon Jul 08 15:54:24 2013 +0200
@@ -127,6 +127,45 @@
                         fields.subClassBField.publicOverriddenMethod() + fields.subClassCField.publicOverriddenMethod();
     }
 
+    public interface Attributes {
+
+        int getLength();
+    }
+
+    public class NullAttributes implements Attributes {
+
+        @Override
+        public int getLength() {
+            return 0;
+        }
+
+    }
+
+    public class TenAttributes implements Attributes {
+
+        @Override
+        public int getLength() {
+            return 10;
+        }
+
+    }
+
+    public int getAttributesLength(Attributes a) {
+        return a.getLength();
+    }
+
+    @Test
+    public void testGuardedInline() {
+        NullAttributes nullAttributes = new NullAttributes();
+        for (int i = 0; i < 10000; i++) {
+            getAttributesLength(nullAttributes);
+        }
+        getAttributesLength(new TenAttributes());
+
+        test("getAttributesLength", nullAttributes);
+        test("getAttributesLength", (Object) null);
+    }
+
     @Test
     public void testClassHierarchyAnalysis() {
         assertInlined(getGraph("invokeLeafClassMethodSnippet", false));
diff -r b6e46324233f -r 192a3b3c7292 graal/com.oracle.graal.hotspot.test/src/com/oracle/graal/hotspot/test/WriteBarrierAdditionTest.java
--- a/graal/com.oracle.graal.hotspot.test/src/com/oracle/graal/hotspot/test/WriteBarrierAdditionTest.java	Mon Jul 08 00:05:30 2013 +0200
+++ b/graal/com.oracle.graal.hotspot.test/src/com/oracle/graal/hotspot/test/WriteBarrierAdditionTest.java	Mon Jul 08 15:54:24 2013 +0200
@@ -40,8 +40,6 @@
 import com.oracle.graal.nodes.spi.Lowerable.LoweringType;
 import com.oracle.graal.phases.*;
 import com.oracle.graal.phases.common.*;
-import com.oracle.graal.phases.common.InliningUtil.InlineInfo;
-import com.oracle.graal.phases.common.InliningUtil.InliningPolicy;
 import com.oracle.graal.phases.tiers.*;
 
 /**
@@ -206,7 +204,7 @@
             public void run() {
                 StructuredGraph graph = parse(snippet);
                 HighTierContext context = new HighTierContext(runtime(), new Assumptions(false), replacements);
-                new InliningPhase(runtime(), replacements, context.getAssumptions(), null, getDefaultPhasePlan(), OptimisticOptimizations.ALL, new InlineAllPolicy()).apply(graph);
+                new InliningPhase(runtime(), replacements, context.getAssumptions(), null, getDefaultPhasePlan(), OptimisticOptimizations.ALL, new InliningPhase.InlineEverythingPolicy()).apply(graph);
                 new LoweringPhase(LoweringType.BEFORE_GUARDS).apply(graph, context);
                 new WriteBarrierAdditionPhase().apply(graph);
                 Debug.dump(graph, "After Write Barrier Addition");
@@ -253,17 +251,4 @@
         HotSpotInstalledCode code = getInstalledCode(snippet);
         code.execute(a, b, c);
     }
-
-    final class InlineAllPolicy implements InliningPolicy {
-
-        @Override
-        public boolean continueInlining(StructuredGraph graph) {
-            return true;
-        }
-
-        @Override
-        public boolean isWorthInlining(InlineInfo info, int inliningDepth, double probability, double relevance, boolean fullyProcessed) {
-            return true;
-        }
-    }
 }
diff -r b6e46324233f -r 192a3b3c7292 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/HotSpotRuntime.java
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/HotSpotRuntime.java	Mon Jul 08 00:05:30 2013 +0200
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/meta/HotSpotRuntime.java	Mon Jul 08 15:54:24 2013 +0200
@@ -490,11 +490,13 @@
         } else if (n instanceof Invoke) {
             Invoke invoke = (Invoke) n;
             if (invoke.callTarget() instanceof MethodCallTargetNode) {
+
                 MethodCallTargetNode callTarget = (MethodCallTargetNode) invoke.callTarget();
                 NodeInputList<ValueNode> parameters = callTarget.arguments();
                 ValueNode receiver = parameters.size() <= 0 ? null : parameters.get(0);
+                GuardingNode receiverNullCheck = null;
                 if (!callTarget.isStatic() && receiver.kind() == Kind.Object && !receiver.objectStamp().nonNull()) {
-                    tool.createNullCheckGuard(invoke, receiver);
+                    receiverNullCheck = tool.createNullCheckGuard(invoke, receiver);
                 }
                 JavaType[] signature = MetaUtil.signatureToTypes(callTarget.targetMethod().getSignature(), callTarget.isStatic() ? null : callTarget.targetMethod().getDeclaringClass());
 
@@ -506,7 +508,12 @@
                         if (hsMethod.isInVirtualMethodTable()) {
                             int vtableEntryOffset = hsMethod.vtableEntryOffset();
                             assert vtableEntryOffset > 0;
-                            ReadNode hub = this.createReadHub(tool, graph, wordKind, receiver);
+                            ReadNode hub = createReadHub(graph, wordKind, receiver);
+
+                            if (receiverNullCheck != null) {
+                                hub.setGuard(receiverNullCheck);
+                            }
+
                             ReadNode metaspaceMethod = createReadVirtualMethod(graph, wordKind, hub, hsMethod);
                             // We use LocationNode.ANY_LOCATION for the reads that access the
                             // compiled code entry as HotSpot does not guarantee they are final
@@ -631,7 +638,10 @@
             LoadHubNode loadHub = (LoadHubNode) n;
             assert loadHub.kind() == wordKind;
             ValueNode object = loadHub.object();
-            ReadNode hub = createReadHub(tool, graph, wordKind, object);
+            ReadNode hub = createReadHub(graph, wordKind, object);
+            // A hub read must not float outside its block otherwise
+            // it may float above an explicit null check on its object.
+            hub.setGuard(AbstractBeginNode.prevBegin(loadHub));
             graph.replaceFixed(loadHub, hub);
         } else if (n instanceof LoadMethodNode) {
             LoadMethodNode loadMethodNode = (LoadMethodNode) n;
@@ -822,12 +832,10 @@
         return metaspaceMethod;
     }
 
-    private ReadNode createReadHub(LoweringTool tool, StructuredGraph graph, Kind wordKind, ValueNode object) {
+    private ReadNode createReadHub(StructuredGraph graph, Kind wordKind, ValueNode object) {
         LocationNode location = ConstantLocationNode.create(FINAL_LOCATION, wordKind, config.hubOffset, graph);
         assert !object.isConstant() || object.asConstant().isNull();
-        ReadNode hub = graph.add(new ReadNode(object, location, StampFactory.forKind(wordKind()), WriteBarrierType.NONE, false));
-        tool.createNullCheckGuard(hub, object);
-        return hub;
+        return graph.add(new ReadNode(object, location, StampFactory.forKind(wordKind()), WriteBarrierType.NONE, false));
     }
 
     public static long referentOffset() {
@@ -914,11 +922,11 @@
             ifNodeOffset.replaceAtUsages(phiNode);
         } else {
             IndexedLocationNode location = IndexedLocationNode.create(ANY_LOCATION, load.accessKind(), load.displacement(), load.offset(), graph, 1);
-            // Unsafe Accesses to the metaspace or to any
+            // Unsafe access to a metaspace or to any
             // absolute address do not perform uncompression.
             ReadNode memoryRead = graph.add(new ReadNode(load.object(), location, load.stamp(), WriteBarrierType.NONE, compress));
-            // An unsafe read must not floating outside its block as may float above an explicit
-            // null check on its object.
+            // An unsafe read must not float outside its block otherwise
+            // it may float above an explicit null check on its object.
             memoryRead.setGuard(AbstractBeginNode.prevBegin(load));
             graph.replaceFixedWithFixed(load, memoryRead);
         }
diff -r b6e46324233f -r 192a3b3c7292 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ArrayCopySnippets.java
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ArrayCopySnippets.java	Mon Jul 08 00:05:30 2013 +0200
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ArrayCopySnippets.java	Mon Jul 08 15:54:24 2013 +0200
@@ -24,6 +24,8 @@
 
 import static com.oracle.graal.api.meta.LocationIdentity.*;
 import static com.oracle.graal.hotspot.replacements.HotSpotReplacementsUtil.*;
+import static com.oracle.graal.nodes.GuardingPiNode.*;
+import static com.oracle.graal.nodes.calc.IsNullNode.*;
 import static com.oracle.graal.phases.GraalOptions.*;
 import static com.oracle.graal.replacements.nodes.BranchProbabilityNode.*;
 
@@ -35,10 +37,13 @@
 import com.oracle.graal.graph.*;
 import com.oracle.graal.hotspot.nodes.*;
 import com.oracle.graal.nodes.*;
+import com.oracle.graal.nodes.calc.*;
 import com.oracle.graal.nodes.extended.*;
 import com.oracle.graal.nodes.java.*;
+import com.oracle.graal.nodes.type.*;
 import com.oracle.graal.phases.*;
 import com.oracle.graal.replacements.*;
+import com.oracle.graal.replacements.Snippet.Fold;
 import com.oracle.graal.replacements.nodes.*;
 import com.oracle.graal.word.*;
 
@@ -77,17 +82,19 @@
     private static final long VECTOR_SIZE = arrayIndexScale(Kind.Long);
 
     private static void checkedCopy(Object src, int srcPos, Object dest, int destPos, int length, Kind baseKind) {
-        checkNonNull(src);
-        checkNonNull(dest);
-        checkLimits(src, srcPos, dest, destPos, length);
-        UnsafeArrayCopyNode.arraycopy(src, srcPos, dest, destPos, length, baseKind);
+        Object nonNullSrc = checkNonNull(src);
+        Object nonNullDest = checkNonNull(dest);
+        checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length);
+        UnsafeArrayCopyNode.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length, baseKind);
     }
 
-    public static void checkNonNull(Object obj) {
-        if (obj == null) {
-            checkNPECounter.inc();
-            DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint);
-        }
+    @Fold
+    private static Stamp nonNull() {
+        return StampFactory.objectNonNull();
+    }
+
+    public static Object checkNonNull(Object obj) {
+        return guardingPi(obj, isNull(obj), true, DeoptimizationReason.RuntimeConstraint, DeoptimizationAction.None, nonNull());
     }
 
     public static int checkArrayType(Word hub) {
@@ -184,24 +191,25 @@
 
     @Snippet
     public static void arraycopy(Object src, int srcPos, Object dest, int destPos, int length) {
-        // loading the hubs also checks for nullness
-        Word srcHub = loadHub(src);
-        Word destHub = loadHub(dest);
-        if (probability(FAST_PATH_PROBABILITY, srcHub.equal(destHub)) && probability(FAST_PATH_PROBABILITY, src != dest)) {
+        Object nonNullSrc = checkNonNull(src);
+        Object nonNullDest = checkNonNull(dest);
+        Word srcHub = loadHub(nonNullSrc);
+        Word destHub = loadHub(nonNullDest);
+        if (probability(FAST_PATH_PROBABILITY, srcHub.equal(destHub)) && probability(FAST_PATH_PROBABILITY, nonNullSrc != nonNullDest)) {
             int layoutHelper = checkArrayType(srcHub);
             final boolean isObjectArray = ((layoutHelper & layoutHelperElementTypePrimitiveInPlace()) == 0);
 
-            checkLimits(src, srcPos, dest, destPos, length);
+            checkLimits(nonNullSrc, srcPos, nonNullDest, destPos, length);
             if (probability(FAST_PATH_PROBABILITY, isObjectArray)) {
                 genericObjectExactCallCounter.inc();
-                arrayObjectCopy(src, srcPos, dest, destPos, length);
+                arrayObjectCopy(nonNullSrc, srcPos, nonNullDest, destPos, length);
             } else {
                 genericPrimitiveCallCounter.inc();
-                UnsafeArrayCopyNode.arraycopyPrimitive(src, srcPos, dest, destPos, length, layoutHelper);
+                UnsafeArrayCopyNode.arraycopyPrimitive(nonNullSrc, srcPos, nonNullDest, destPos, length, layoutHelper);
             }
         } else {
             genericObjectCallCounter.inc();
-            System.arraycopy(src, srcPos, dest, destPos, length);
+            System.arraycopy(nonNullSrc, srcPos, nonNullDest, destPos, length);
         }
     }
 
diff -r b6e46324233f -r 192a3b3c7292 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/HotSpotReplacementsUtil.java
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/HotSpotReplacementsUtil.java	Mon Jul 08 00:05:30 2013 +0200
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/HotSpotReplacementsUtil.java	Mon Jul 08 15:54:24 2013 +0200
@@ -437,19 +437,12 @@
     }
 
     /**
-     * Loads the hub from a object, null checking it first.
+     * Loads the hub of an object (without null checking it first).
      */
     public static Word loadHub(Object object) {
         return loadHubIntrinsic(object, getWordKind());
     }
 
-    /**
-     * Loads the hub from a object.
-     */
-    public static Word loadHubNoNullcheck(Object object) {
-        return loadWordFromObject(object, hubOffset());
-    }
-
     public static Object verifyOop(Object object) {
         if (verifyOops()) {
             verifyOopStub(VERIFY_OOP, object);
diff -r b6e46324233f -r 192a3b3c7292 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/MonitorSnippets.java
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/MonitorSnippets.java	Mon Jul 08 00:05:30 2013 +0200
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/MonitorSnippets.java	Mon Jul 08 15:54:24 2013 +0200
@@ -107,7 +107,7 @@
             } else {
                 // The bias pattern is present in the object's mark word. Need to check
                 // whether the bias owner and the epoch are both still current.
-                Word hub = loadHubNoNullcheck(object);
+                Word hub = loadHub(object);
                 final Word prototypeMarkWord = hub.readWord(prototypeMarkWordOffset(), PROTOTYPE_MARK_WORD_LOCATION);
                 final Word thread = thread();
                 final Word tmp = prototypeMarkWord.or(thread).xor(mark).and(~ageMaskInPlace());
diff -r b6e46324233f -r 192a3b3c7292 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/WriteBarrierSnippets.java
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/WriteBarrierSnippets.java	Mon Jul 08 00:05:30 2013 +0200
+++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/WriteBarrierSnippets.java	Mon Jul 08 15:54:24 2013 +0200
@@ -121,10 +121,8 @@
             if (probability(LIKELY_PROBABILITY, doLoad)) {
                 previousOop = (Word) Word.fromObject(field.readObjectCompressed(0));
                 if (trace) {
-                    if (previousOop.notEqual(Word.zero())) {
-                        verifyOop(previousOop.toObject());
-                    }
                     log(trace, "[%d] G1-Pre Thread %p Previous Object %p\n ", gcCycle, thread.rawValue(), previousOop.rawValue());
+                    verifyOop(previousOop.toObject());
                 }
             }
             // If the previous value is null the barrier should not be issued.
@@ -161,7 +159,7 @@
         int gcCycle = 0;
         if (trace) {
             gcCycle = (int) Word.unsigned(HotSpotReplacementsUtil.gcTotalCollectionsAddress()).readLong(0);
-            log(trace, "[%d] G1-Post Thread: %p Object: %p Field: %p\n", gcCycle, thread.rawValue(), Word.fromObject(fixedObject).rawValue());
+            log(trace, "[%d] G1-Post Thread: %p Object: %p\n", gcCycle, thread.rawValue(), Word.fromObject(fixedObject).rawValue());
             log(trace, "[%d] G1-Post Thread: %p Field: %p\n", gcCycle, thread.rawValue(), field.rawValue());
         }
         Word writtenValue = (Word) Word.fromObject(fixedValue);
diff -r b6e46324233f -r 192a3b3c7292 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/GuardingPiNode.java
--- /dev/null	Thu Jan 01 00:00:00 1970 +0000
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/GuardingPiNode.java	Mon Jul 08 15:54:24 2013 +0200
@@ -0,0 +1,93 @@
+/*
+ * Copyright (c) 2012, 2013, Oracle and/or its affiliates. All rights reserved.
+ * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
+ *
+ * This code is free software; you can redistribute it and/or modify it
+ * under the terms of the GNU General Public License version 2 only, as
+ * published by the Free Software Foundation.
+ *
+ * This code is distributed in the hope that it will be useful, but WITHOUT
+ * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
+ * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
+ * version 2 for more details (a copy is included in the LICENSE file that
+ * accompanied this code).
+ *
+ * You should have received a copy of the GNU General Public License version
+ * 2 along with this work; if not, write to the Free Software Foundation,
+ * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA.
+ *
+ * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA
+ * or visit www.oracle.com if you need additional information or have any
+ * questions.
+ */
+package com.oracle.graal.nodes;
+
+import com.oracle.graal.api.code.*;
+import com.oracle.graal.api.meta.*;
+import com.oracle.graal.graph.*;
+import com.oracle.graal.nodes.extended.*;
+import com.oracle.graal.nodes.spi.*;
+import com.oracle.graal.nodes.type.*;
+
+/**
+ * A node that changes the stamp of its input based on some condition being true.
+ */
+@NodeInfo(nameTemplate = "GuardingPi(!={p#negated}) {p#reason/s}")
+public class GuardingPiNode extends FixedWithNextNode implements Lowerable, GuardingNode, Canonicalizable {
+
+    @Input private ValueNode object;
+    @Input private LogicNode condition;
+    private final DeoptimizationReason reason;
+    private final DeoptimizationAction action;
+    private boolean negated;
+
+    public ValueNode object() {
+        return object;
+    }
+
+    public GuardingPiNode(ValueNode object, ValueNode condition, boolean negateCondition, DeoptimizationReason reason, DeoptimizationAction action, Stamp stamp) {
+        super(object.stamp().join(stamp));
+        assert stamp() != null;
+        this.object = object;
+        this.condition = (LogicNode) condition;
+        this.reason = reason;
+        this.action = action;
+        this.negated = negateCondition;
+    }
+
+    @Override
+    public void lower(LoweringTool tool, LoweringType loweringType) {
+        if (loweringType == LoweringType.AFTER_GUARDS) {
+            throw new GraalInternalError("Cannot create guards in after-guard lowering");
+        }
+        FixedGuardNode guard = graph().add(new FixedGuardNode(condition, reason, action, negated));
+        PiNode pi = graph().add(new PiNode(object, stamp(), guard));
+        replaceAtUsages(pi);
+        graph().replaceFixedWithFixed(this, guard);
+    }
+
+    @Override
+    public boolean inferStamp() {
+        return updateStamp(stamp().join(object().stamp()));
+    }
+
+    @Override
+    public ValueNode canonical(CanonicalizerTool tool) {
+        if (condition instanceof LogicConstantNode) {
+            LogicConstantNode c = (LogicConstantNode) condition;
+            if (c.getValue() != negated && stamp().equals(object().stamp())) {
+                return object;
+            }
+        }
+        return this;
+    }
+
+    @NodeIntrinsic
+    public static native Object guardingPi(Object object, LogicNode condition, @ConstantNodeParameter boolean negateCondition, @ConstantNodeParameter DeoptimizationReason reason,
+                    @ConstantNodeParameter DeoptimizationAction action, @ConstantNodeParameter Stamp stamp);
+
+    @Override
+    public ValueNode asNode() {
+        return this;
+    }
+}
diff -r b6e46324233f -r 192a3b3c7292 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/IsNullNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/IsNullNode.java	Mon Jul 08 00:05:30 2013 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/calc/IsNullNode.java	Mon Jul 08 15:54:24 2013 +0200
@@ -92,4 +92,7 @@
         }
         return false;
     }
+
+    @NodeIntrinsic
+    public static native IsNullNode isNull(Object object);
 }
diff -r b6e46324233f -r 192a3b3c7292 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/extended/GuardedNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/extended/GuardedNode.java	Mon Jul 08 00:05:30 2013 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/extended/GuardedNode.java	Mon Jul 08 15:54:24 2013 +0200
@@ -24,6 +24,9 @@
 
 import com.oracle.graal.nodes.*;
 
+/**
+ * A node that may be guarded by a {@linkplain GuardingNode guarding node}.
+ */
 public interface GuardedNode {
 
     GuardingNode getGuard();
diff -r b6e46324233f -r 192a3b3c7292 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/extended/LoadHubNode.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/extended/LoadHubNode.java	Mon Jul 08 00:05:30 2013 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/extended/LoadHubNode.java	Mon Jul 08 15:54:24 2013 +0200
@@ -29,7 +29,8 @@
 import com.oracle.graal.nodes.type.*;
 
 /**
- * Loads an object's {@linkplain Representation#ObjectHub hub}, null-checking the object first.
+ * Loads an object's {@linkplain Representation#ObjectHub hub}. The object is not null-checked by
+ * this operation.
  */
 public final class LoadHubNode extends FixedWithNextNode implements Lowerable, Canonicalizable, Virtualizable {
 
diff -r b6e46324233f -r 192a3b3c7292 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/type/Stamp.java
--- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/type/Stamp.java	Mon Jul 08 00:05:30 2013 +0200
+++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/type/Stamp.java	Mon Jul 08 15:54:24 2013 +0200
@@ -46,6 +46,9 @@
      */
     public abstract ResolvedJavaType javaType(MetaAccessProvider metaAccess);
 
+    /**
+     * Determines if the stamped value is guaranteed to be non-null.
+     */
     public boolean nonNull() {
         return false;
     }
diff -r b6e46324233f -r 192a3b3c7292 graal/com.oracle.graal.phases.common/src/com/oracle/graal/phases/common/InliningPhase.java
--- a/graal/com.oracle.graal.phases.common/src/com/oracle/graal/phases/common/InliningPhase.java	Mon Jul 08 00:05:30 2013 +0200
+++ b/graal/com.oracle.graal.phases.common/src/com/oracle/graal/phases/common/InliningPhase.java	Mon Jul 08 15:54:24 2013 +0200
@@ -431,7 +431,7 @@
         }
     }
 
-    private static class GreedyInliningPolicy extends AbstractInliningPolicy {
+    private static final class GreedyInliningPolicy extends AbstractInliningPolicy {
 
         public GreedyInliningPolicy(Replacements replacements, Map<Invoke, Double> hints) {
             super(replacements, hints);
@@ -488,6 +488,20 @@
         }
     }
 
+    public static final class InlineEverythingPolicy implements InliningPolicy {
+
+        public boolean continueInlining(StructuredGraph graph) {
+            if (graph.getNodeCount() >= MaximumDesiredSize.getValue()) {
+                throw new BailoutException("Inline all calls failed. The resulting graph is too large.");
+            }
+            return true;
+        }
+
+        public boolean isWorthInlining(InlineInfo info, int inliningDepth, double probability, double relevance, boolean fullyProcessed) {
+            return true;
+        }
+    }
+
     private static class InliningIterator {
 
         private final FixedNode start;
diff -r b6e46324233f -r 192a3b3c7292 graal/com.oracle.graal.phases.common/src/com/oracle/graal/phases/common/InliningUtil.java
--- a/graal/com.oracle.graal.phases.common/src/com/oracle/graal/phases/common/InliningUtil.java	Mon Jul 08 00:05:30 2013 +0200
+++ b/graal/com.oracle.graal.phases.common/src/com/oracle/graal/phases/common/InliningUtil.java	Mon Jul 08 15:54:24 2013 +0200
@@ -22,6 +22,9 @@
  */
 package com.oracle.graal.phases.common;
 
+import static com.oracle.graal.api.code.DeoptimizationAction.*;
+import static com.oracle.graal.api.meta.DeoptimizationReason.*;
+import static com.oracle.graal.nodes.type.StampFactory.*;
 import static com.oracle.graal.phases.GraalOptions.*;
 
 import java.lang.reflect.*;
@@ -251,18 +254,26 @@
     }
 
     /**
-     * Represents an opportunity for inlining at the given invoke, with the given weight and level.
+     * Represents an opportunity for inlining at a given invoke, with the given weight and level.
      * The weight is the amortized weight of the additional code - so smaller is better. The level
      * is the number of nested inlinings that lead to this invoke.
      */
     public interface InlineInfo {
 
+        /**
+         * The graph containing the {@link #invoke() invocation} that may be inlined.
+         */
         StructuredGraph graph();
 
+        /**
+         * The invocation that may be inlined.
+         */
         Invoke invoke();
 
         /**
-         * Returns the number of invoked methods.
+         * Returns the number of methods that may be inlined by the {@link #invoke() invocation}.
+         * This may be more than one in the case of a invocation profile showing a number of "hot"
+         * concrete methods dispatched to by the invocation.
          */
         int numberOfMethods();
 
@@ -465,16 +476,16 @@
         }
 
         private void createGuard(StructuredGraph graph, MetaAccessProvider runtime) {
-            InliningUtil.receiverNullCheck(invoke);
-            ValueNode receiver = ((MethodCallTargetNode) invoke.callTarget()).receiver();
+            ValueNode nonNullReceiver = InliningUtil.nonNullReceiver(invoke);
             ConstantNode typeHub = ConstantNode.forConstant(type.getEncoding(Representation.ObjectHub), runtime, graph);
-            LoadHubNode receiverHub = graph.add(new LoadHubNode(receiver, typeHub.kind()));
+            LoadHubNode receiverHub = graph.add(new LoadHubNode(nonNullReceiver, typeHub.kind()));
+
             CompareNode typeCheck = CompareNode.createCompareNode(Condition.EQ, receiverHub, typeHub);
             FixedGuardNode guard = graph.add(new FixedGuardNode(typeCheck, DeoptimizationReason.TypeCheckedInliningViolated, DeoptimizationAction.InvalidateReprofile));
             assert invoke.predecessor() != null;
 
-            ValueNode anchoredReceiver = createAnchoredReceiver(graph, guard, type, receiver, true);
-            invoke.callTarget().replaceFirstInput(receiver, anchoredReceiver);
+            ValueNode anchoredReceiver = createAnchoredReceiver(graph, guard, type, nonNullReceiver, true);
+            invoke.callTarget().replaceFirstInput(nonNullReceiver, anchoredReceiver);
 
             graph.addBeforeFixed(invoke.asNode(), receiverHub);
             graph.addBeforeFixed(invoke.asNode(), guard);
@@ -488,7 +499,7 @@
 
     /**
      * Polymorphic inlining of m methods with n type checks (n >= m) in case that the profiling
-     * information suggests a reasonable amounts of different receiver types and different methods.
+     * information suggests a reasonable amount of different receiver types and different methods.
      * If an unknown type is encountered a deoptimization is triggered.
      */
     private static class MultiTypeGuardInlineInfo extends AbstractInlineInfo {
@@ -572,8 +583,6 @@
 
         @Override
         public void inline(MetaAccessProvider runtime, Assumptions assumptions, Replacements replacements) {
-            // receiver null check must be the first node
-            InliningUtil.receiverNullCheck(invoke);
             if (hasSingleMethod()) {
                 inlineSingleMethod(graph(), runtime, assumptions);
             } else {
@@ -753,9 +762,9 @@
 
         private boolean createDispatchOnTypeBeforeInvoke(StructuredGraph graph, AbstractBeginNode[] successors, boolean invokeIsOnlySuccessor, MetaAccessProvider runtime) {
             assert ptypes.size() >= 1;
-
+            ValueNode nonNullReceiver = nonNullReceiver(invoke);
             Kind hubKind = ((MethodCallTargetNode) invoke.callTarget()).targetMethod().getDeclaringClass().getEncoding(Representation.ObjectHub).getKind();
-            LoadHubNode hub = graph.add(new LoadHubNode(((MethodCallTargetNode) invoke.callTarget()).receiver(), hubKind));
+            LoadHubNode hub = graph.add(new LoadHubNode(nonNullReceiver, hubKind));
             graph.addBeforeFixed(invoke.asNode(), hub);
 
             if (!invokeIsOnlySuccessor && chooseMethodDispatch()) {
@@ -940,8 +949,6 @@
         }
 
         private void devirtualizeWithTypeSwitch(StructuredGraph graph, InvokeKind kind, ResolvedJavaMethod target, MetaAccessProvider runtime) {
-            InliningUtil.receiverNullCheck(invoke);
-
             AbstractBeginNode invocationEntry = graph.add(new BeginNode());
             AbstractBeginNode unknownTypeSux = createUnknownTypeSuccessor(graph);
             AbstractBeginNode[] successors = new AbstractBeginNode[]{invocationEntry, unknownTypeSux};
@@ -1032,6 +1039,10 @@
 
         ResolvedJavaType holder = targetMethod.getDeclaringClass();
         ObjectStamp receiverStamp = callTarget.receiver().objectStamp();
+        if (receiverStamp.alwaysNull()) {
+            // Don't inline if receiver is known to be null
+            return null;
+        }
         if (receiverStamp.type() != null) {
             // the invoke target might be more specific than the holder (happens after inlining:
             // locals lose their declared type...)
@@ -1280,17 +1291,8 @@
 
         FrameState stateAfter = invoke.stateAfter();
         assert stateAfter == null || stateAfter.isAlive();
-        if (receiverNullCheck) {
-            GuardingNode receiverNullCheckNode = receiverNullCheck(invoke);
-            if (receiverNullCheckNode != null) {
-                ValueNode receiver = invoke.callTarget().arguments().get(0);
-                Stamp piStamp = receiver.stamp();
-                if (piStamp instanceof ObjectStamp) {
-                    piStamp = piStamp.join(StampFactory.objectNonNull());
-                }
-                ValueNode anchoredReceiver = createAnchoredReceiver(graph, receiverNullCheckNode, receiver, piStamp);
-                invoke.callTarget().replaceFirstInput(receiver, anchoredReceiver);
-            }
+        if (receiverNullCheck && !((MethodCallTargetNode) invoke.callTarget()).isStatic()) {
+            nonNullReceiver(invoke);
         }
 
         IdentityHashMap<Node, Node> replacements = new IdentityHashMap<>();
@@ -1428,17 +1430,23 @@
         return true;
     }
 
-    public static GuardingNode receiverNullCheck(Invoke invoke) {
+    /**
+     * Gets the receiver for an invoke, adding a guard if necessary to ensure it is non-null.
+     */
+    public static ValueNode nonNullReceiver(Invoke invoke) {
         MethodCallTargetNode callTarget = (MethodCallTargetNode) invoke.callTarget();
+        assert !callTarget.isStatic() : callTarget.targetMethod();
         StructuredGraph graph = callTarget.graph();
-        NodeInputList<ValueNode> parameters = callTarget.arguments();
-        ValueNode firstParam = parameters.size() <= 0 ? null : parameters.get(0);
-        if (!callTarget.isStatic() && firstParam.kind() == Kind.Object && !firstParam.objectStamp().nonNull()) {
-            FixedGuardNode guard = graph.add(new FixedGuardNode(graph.unique(new IsNullNode(firstParam)), DeoptimizationReason.NullCheckException, DeoptimizationAction.InvalidateReprofile, true));
-            graph.addBeforeFixed(invoke.asNode(), guard);
-            return guard;
+        ValueNode firstParam = callTarget.arguments().get(0);
+        if (firstParam.kind() == Kind.Object && !firstParam.objectStamp().nonNull()) {
+            assert !firstParam.objectStamp().alwaysNull();
+            IsNullNode condition = graph.unique(new IsNullNode(firstParam));
+            GuardingPiNode nonNullReceiver = graph.add(new GuardingPiNode(firstParam, condition, true, NullCheckException, InvalidateReprofile, objectNonNull()));
+            graph.addBeforeFixed(invoke.asNode(), nonNullReceiver);
+            callTarget.replaceFirstInput(firstParam, nonNullReceiver);
+            return nonNullReceiver;
         }
-        return null;
+        return firstParam;
     }
 
     public static boolean canIntrinsify(Replacements replacements, ResolvedJavaMethod target) {
diff -r b6e46324233f -r 192a3b3c7292 graal/com.oracle.graal.replacements.test/src/com/oracle/graal/replacements/test/TypeCheckTest.java
--- a/graal/com.oracle.graal.replacements.test/src/com/oracle/graal/replacements/test/TypeCheckTest.java	Mon Jul 08 00:05:30 2013 +0200
+++ b/graal/com.oracle.graal.replacements.test/src/com/oracle/graal/replacements/test/TypeCheckTest.java	Mon Jul 08 15:54:24 2013 +0200
@@ -49,7 +49,7 @@
     }
 
     protected JavaTypeProfile profile(Class... types) {
-        return profile(TriState.UNKNOWN, types);
+        return profile(TriState.FALSE, types);
     }
 
     protected JavaTypeProfile profile(TriState nullSeen, Class... types) {
diff -r b6e46324233f -r 192a3b3c7292 graal/com.oracle.graal.replacements/src/com/oracle/graal/replacements/NodeIntrinsificationPhase.java
--- a/graal/com.oracle.graal.replacements/src/com/oracle/graal/replacements/NodeIntrinsificationPhase.java	Mon Jul 08 00:05:30 2013 +0200
+++ b/graal/com.oracle.graal.replacements/src/com/oracle/graal/replacements/NodeIntrinsificationPhase.java	Mon Jul 08 15:54:24 2013 +0200
@@ -29,6 +29,7 @@
 
 import com.oracle.graal.api.meta.*;
 import com.oracle.graal.debug.*;
+import com.oracle.graal.debug.internal.*;
 import com.oracle.graal.graph.*;
 import com.oracle.graal.graph.Node.ConstantNodeParameter;
 import com.oracle.graal.graph.Node.NodeIntrinsic;
@@ -319,9 +320,13 @@
             usage.replaceFirstInput(input, intrinsifiedNode);
             Debug.log("%s: Checkcast used in a return with forNodeIntrinsic stamp", Debug.contextSnapshot(JavaMethod.class));
         } else if (usage instanceof IsNullNode) {
-            assert usage.usages().count() == 1 && usage.usages().first().predecessor() == input;
-            graph.replaceFloating((FloatingNode) usage, LogicConstantNode.contradiction(graph));
-            Debug.log("%s: Replaced IsNull with false", Debug.contextSnapshot(JavaMethod.class));
+            if (!usage.usages().isEmpty()) {
+                assert usage.usages().count() == 1 && usage.usages().first().predecessor() == input : usage + " " + input;
+                graph.replaceFloating((FloatingNode) usage, LogicConstantNode.contradiction(graph));
+                Debug.log("%s: Replaced IsNull with false", Debug.contextSnapshot(JavaMethod.class));
+            } else {
+                // Removed as usage of a GuardingPiNode
+            }
         } else if (usage instanceof ProxyNode) {
             ProxyNode proxy = (ProxyNode) usage;
             assert proxy.type() == PhiType.Value;
@@ -333,9 +338,15 @@
             for (Node piUsage : usage.usages().snapshot()) {
                 checkCheckCastUsage(graph, intrinsifiedNode, usage, piUsage);
             }
+        } else if (usage instanceof GuardingPiNode) {
+            GuardingPiNode pi = (GuardingPiNode) usage;
+            for (Node piUsage : pi.usages().snapshot()) {
+                checkCheckCastUsage(graph, intrinsifiedNode, usage, piUsage);
+            }
+            graph.removeFixed(pi);
         } else {
-            Debug.dump(graph, "exception");
-            assert false : sourceLocation(usage) + " has unexpected usage " + usage + " of checkcast at " + sourceLocation(input);
+            DebugScope.dump(graph, "exception");
+            assert false : sourceLocation(usage) + " has unexpected usage " + usage + " of checkcast " + input + " at " + sourceLocation(input);
         }
     }
 }
diff -r b6e46324233f -r 192a3b3c7292 src/share/vm/code/nmethod.cpp
--- a/src/share/vm/code/nmethod.cpp	Mon Jul 08 00:05:30 2013 +0200
+++ b/src/share/vm/code/nmethod.cpp	Mon Jul 08 15:54:24 2013 +0200
@@ -518,7 +518,7 @@
                                             code_buffer, frame_size,
                                             basic_lock_owner_sp_offset,
                                             basic_lock_sp_offset, oop_maps);
-    NOT_PRODUCT(if (nm != NULL)  nmethod_stats.note_native_nmethod(nm));
+    if (nm != NULL)  nmethod_stats.note_native_nmethod(nm);
     if (PrintAssembly && nm != NULL) {
       Disassembler::decode(nm);
     }
@@ -554,7 +554,7 @@
     nm = new (nmethod_size) nmethod(method(), nmethod_size,
                                     &offsets, code_buffer, frame_size);
 
-    NOT_PRODUCT(if (nm != NULL)  nmethod_stats.note_nmethod(nm));
+    if (nm != NULL)  nmethod_stats.note_nmethod(nm);
     if (PrintAssembly && nm != NULL) {
       Disassembler::decode(nm);
     }
@@ -639,7 +639,7 @@
         InstanceKlass::cast(klass)->add_dependent_nmethod(nm);
       }
     }
-    NOT_PRODUCT(if (nm != NULL)  nmethod_stats.note_nmethod(nm));
+    if (nm != NULL)  nmethod_stats.note_nmethod(nm);
     if (PrintAssembly && nm != NULL) {
       Disassembler::decode(nm);
     }
diff -r b6e46324233f -r 192a3b3c7292 src/share/vm/runtime/globals.hpp