# HG changeset patch # User Lukas Stadler # Date 1358177348 -3600 # Node ID f04a5e46cdd29b533e4845d3dc76f5d5881058ad # Parent cbcee2e1ce51507debe537fe3ded3682b33bfc42 new facility to add branch probabilities for snippets, probabilities for frequently-used snippets diff -r cbcee2e1ce51 -r f04a5e46cdd2 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/CheckCastSnippets.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/CheckCastSnippets.java Mon Jan 14 15:23:13 2013 +0100 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/CheckCastSnippets.java Mon Jan 14 16:29:08 2013 +0100 @@ -25,6 +25,7 @@ import static com.oracle.graal.hotspot.snippets.HotSpotSnippetUtils.*; import static com.oracle.graal.snippets.SnippetTemplate.*; import static com.oracle.graal.snippets.SnippetTemplate.Arguments.*; +import static com.oracle.graal.snippets.nodes.BranchProbabilityNode.*; import com.oracle.graal.api.code.*; import com.oracle.graal.api.meta.*; @@ -68,11 +69,13 @@ @Parameter("exactHub") Word exactHub, @ConstantParameter("checkNull") boolean checkNull) { if (checkNull && object == null) { + probability(0.1); isNull.inc(); return object; } Word objectHub = loadHub(object); if (objectHub != exactHub) { + probability(0.01); exactMiss.inc(); //bkpt(object, exactHub, objectHub); DeoptimizeNode.deopt(DeoptimizationAction.InvalidateReprofile, DeoptimizationReason.ClassCastException); diff -r cbcee2e1ce51 -r f04a5e46cdd2 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/InstanceOfSnippets.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/InstanceOfSnippets.java Mon Jan 14 15:23:13 2013 +0100 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/InstanceOfSnippets.java Mon Jan 14 16:29:08 2013 +0100 @@ -26,6 +26,7 @@ import static com.oracle.graal.hotspot.snippets.CheckCastSnippets.Templates.*; import static com.oracle.graal.hotspot.snippets.HotSpotSnippetUtils.*; import static com.oracle.graal.snippets.SnippetTemplate.Arguments.*; +import static com.oracle.graal.snippets.nodes.BranchProbabilityNode.*; import com.oracle.graal.api.code.*; import com.oracle.graal.api.meta.*; @@ -66,11 +67,13 @@ @Parameter("falseValue") Object falseValue, @ConstantParameter("checkNull") boolean checkNull) { if (checkNull && object == null) { + probability(0.01); isNull.inc(); return falseValue; } Word objectHub = loadHub(object); if (objectHub != exactHub) { + probability(0.75); exactMiss.inc(); return falseValue; } @@ -90,11 +93,13 @@ @ConstantParameter("checkNull") boolean checkNull, @ConstantParameter("superCheckOffset") int superCheckOffset) { if (checkNull && object == null) { + probability(0.01); isNull.inc(); return falseValue; } Word objectHub = loadHub(object); if (objectHub.readWord(superCheckOffset) != hub) { + probability(0.45); displayMiss.inc(); return falseValue; } @@ -114,6 +119,7 @@ @VarargsParameter("hints") Word[] hints, @ConstantParameter("checkNull") boolean checkNull) { if (checkNull && object == null) { + probability(0.01); isNull.inc(); return falseValue; } @@ -123,6 +129,7 @@ for (int i = 0; i < hints.length; i++) { Word hintHub = hints[i]; if (hintHub == objectHub) { + probability(0.01); hintsHit.inc(); return trueValue; } @@ -151,6 +158,7 @@ int length = secondarySupers.readInt(metaspaceArrayLengthOffset()); for (int i = 0; i < length; i++) { if (t == loadWordElement(secondarySupers, i)) { + probability(0.01); s.writeWord(secondarySuperCacheOffset(), t); secondariesHit.inc(); return true; @@ -171,6 +179,7 @@ @Parameter("falseValue") Object falseValue, @ConstantParameter("checkNull") boolean checkNull) { if (checkNull && object == null) { + probability(0.01); isNull.inc(); return falseValue; } @@ -240,7 +249,7 @@ } } - private static final SnippetCounter.Group counters = GraalOptions.SnippetCounters ? new SnippetCounter.Group("Checkcast") : null; + private static final SnippetCounter.Group counters = GraalOptions.SnippetCounters ? new SnippetCounter.Group("InstanceOf") : null; private static final SnippetCounter hintsHit = new SnippetCounter(counters, "hintsHit", "hit a hint type"); private static final SnippetCounter exactHit = new SnippetCounter(counters, "exactHit", "exact type test succeeded"); private static final SnippetCounter exactMiss = new SnippetCounter(counters, "exactMiss", "exact type test failed"); diff -r cbcee2e1ce51 -r f04a5e46cdd2 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/NewObjectSnippets.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/NewObjectSnippets.java Mon Jan 14 15:23:13 2013 +0100 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/NewObjectSnippets.java Mon Jan 14 16:29:08 2013 +0100 @@ -30,6 +30,7 @@ import static com.oracle.graal.snippets.SnippetTemplate.*; import static com.oracle.graal.snippets.SnippetTemplate.Arguments.*; import static com.oracle.graal.snippets.nodes.ExplodeLoopNode.*; +import static com.oracle.graal.snippets.nodes.BranchProbabilityNode.*; import com.oracle.graal.api.code.*; import com.oracle.graal.api.meta.*; @@ -64,6 +65,7 @@ Word newTop = top.add(size); // this check might lead to problems if the TLAB is within 16GB of the address space end (checked in c++ code) if (newTop.belowOrEqual(end)) { + probability(0.99); thread.writeWord(threadTlabTopOffset(), newTop); return top; } @@ -84,6 +86,7 @@ new_stub.inc(); result = NewInstanceStubCall.call(hub); } else { + probability(0.99); if (locked) { formatObject(hub, size, memory, thread().or(biasedLockPattern()), fillContents); } else { @@ -117,6 +120,7 @@ newarray_stub.inc(); result = NewArrayStubCall.call(hub, length); } else { + probability(0.99); newarray_loopInit.inc(); formatArray(hub, allocationSize, length, headerSize, memory, prototypeMarkWord, fillContents); result = memory.toObject(); @@ -140,6 +144,7 @@ // This handles both negative array sizes and very large array sizes DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); } + probability(0.99); int allocationSize = computeArrayAllocationSize(length, alignment, headerSize, log2ElementSize); Word memory = TLABAllocateNode.allocateVariableSize(allocationSize); return InitializeArrayNode.initialize(memory, length, allocationSize, type, true, false); diff -r cbcee2e1ce51 -r f04a5e46cdd2 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ObjectSubstitutions.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ObjectSubstitutions.java Mon Jan 14 15:23:13 2013 +0100 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/ObjectSubstitutions.java Mon Jan 14 16:29:08 2013 +0100 @@ -24,6 +24,7 @@ import static com.oracle.graal.hotspot.snippets.HotSpotSnippetUtils.*; import static com.oracle.graal.nodes.extended.UnsafeCastNode.*; +import static com.oracle.graal.snippets.nodes.BranchProbabilityNode.*; import com.oracle.graal.hotspot.nodes.*; import com.oracle.graal.snippets.*; @@ -35,8 +36,9 @@ */ @ClassSubstitution(java.lang.Object.class) public class ObjectSubstitutions { + @MethodSubstitution(isStatic = false) - public static Class getClass(final Object thisObj) { + public static Class< ? > getClass(final Object thisObj) { Word hub = loadHub(thisObj); return unsafeCast(hub.readFinalObject(Word.signed(classMirrorOffset())), Class.class, true, true); } @@ -48,8 +50,10 @@ // this code is independent from biased locking (although it does not look that way) final Word biasedLock = mark.and(biasedLockMaskInPlace()); if (biasedLock == Word.unsigned(unlockedMask())) { + probability(0.99); int hash = (int) mark.unsignedShiftRight(identityHashCodeShift()).rawValue(); if (hash != uninitializedIdentityHashCodeValue()) { + probability(0.99); return hash; } } diff -r cbcee2e1ce51 -r f04a5e46cdd2 graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/SystemSubstitutions.java --- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/SystemSubstitutions.java Mon Jan 14 15:23:13 2013 +0100 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/snippets/SystemSubstitutions.java Mon Jan 14 16:29:08 2013 +0100 @@ -23,6 +23,7 @@ package com.oracle.graal.hotspot.snippets; import static com.oracle.graal.hotspot.snippets.HotSpotSnippetUtils.*; +import static com.oracle.graal.snippets.nodes.BranchProbabilityNode.*; import com.oracle.graal.api.code.RuntimeCallTarget.Descriptor; import com.oracle.graal.graph.Node.ConstantNodeParameter; @@ -30,7 +31,7 @@ import com.oracle.graal.hotspot.nodes.*; import com.oracle.graal.nodes.extended.*; import com.oracle.graal.snippets.*; -import com.oracle.graal.snippets.ClassSubstitution.*; +import com.oracle.graal.snippets.ClassSubstitution.MethodSubstitution; import com.oracle.graal.word.*; /** @@ -55,6 +56,7 @@ @MethodSubstitution public static int identityHashCode(Object x) { if (x == null) { + probability(0.01); return 0; } @@ -63,8 +65,10 @@ // this code is independent from biased locking (although it does not look that way) final Word biasedLock = mark.and(biasedLockMaskInPlace()); if (biasedLock == Word.unsigned(unlockedMask())) { + probability(0.99); int hash = (int) mark.unsignedShiftRight(identityHashCodeShift()).rawValue(); if (hash != uninitializedIdentityHashCodeValue()) { + probability(0.99); return hash; } } diff -r cbcee2e1ce51 -r f04a5e46cdd2 graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/IfNode.java --- a/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/IfNode.java Mon Jan 14 15:23:13 2013 +0100 +++ b/graal/com.oracle.graal.nodes/src/com/oracle/graal/nodes/IfNode.java Mon Jan 14 16:29:08 2013 +0100 @@ -121,6 +121,10 @@ return this; } + public void setTakenProbability(double prob) { + takenProbability = prob; + } + @Override public double probability(BeginNode successor) { return successor == trueSuccessor ? takenProbability : 1 - takenProbability; diff -r cbcee2e1ce51 -r f04a5e46cdd2 graal/com.oracle.graal.snippets/src/com/oracle/graal/snippets/nodes/BranchProbabilityNode.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/graal/com.oracle.graal.snippets/src/com/oracle/graal/snippets/nodes/BranchProbabilityNode.java Mon Jan 14 16:29:08 2013 +0100 @@ -0,0 +1,69 @@ +/* + * Copyright (c) 2012, 2013, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + */ +package com.oracle.graal.snippets.nodes; + +import com.oracle.graal.nodes.*; +import com.oracle.graal.nodes.spi.*; +import com.oracle.graal.nodes.type.*; +import com.oracle.graal.nodes.util.*; + +/** + * Instances of this node class will look for a preceding if node and put the given probability into the if node's taken + * probability. Then the branch probability node will be removed. This node is intended primarily for snippets, so that + * they can define their fast and slow paths. + */ +public class BranchProbabilityNode extends FixedWithNextNode implements Simplifiable { + + private final double probability; + + public BranchProbabilityNode(double probability) { + super(StampFactory.forVoid()); + assert probability >= 0 && probability <= 1; + this.probability = probability; + } + + @Override + public void simplify(SimplifierTool tool) { + FixedNode current = this; + while (!(current instanceof BeginNode)) { + current = (FixedNode) current.predecessor(); + } + BeginNode begin = (BeginNode) current; + assert begin.predecessor() instanceof IfNode : "explicit branch probability cannot follow a merge, only if nodes"; + IfNode ifNode = (IfNode) begin.predecessor(); + if (ifNode.trueSuccessor() == begin) { + ifNode.setTakenProbability(probability); + } else { + ifNode.setTakenProbability(1 - probability); + } + + FixedNode next = next(); + setNext(null); + ((FixedWithNextNode) predecessor()).setNext(next); + GraphUtil.killCFG(this); + } + + @NodeIntrinsic + public static native void probability(@ConstantNodeParameter double probability); + +}