Mercurial > hg > truffle
changeset 9316:5e1465ec46d6
Change the way branch probabilities are injected. Update all snippets.
line wrap: on
line diff
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ArrayCopySnippets.java Thu Apr 25 20:10:49 2013 +0200 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ArrayCopySnippets.java Thu Apr 25 21:34:00 2013 +0200 @@ -84,8 +84,8 @@ long nonVectorBytes = byteLength % VECTOR_SIZE; long srcOffset = (long) srcPos * elementSize; long destOffset = (long) destPos * elementSize; - if (src == dest && srcPos < destPos) { // bad aliased case - probability(NOT_FREQUENT_PROBABILITY); + if (probability(NOT_FREQUENT_PROBABILITY, src == dest && srcPos < destPos)) { + // bad aliased case for (long i = byteLength - elementSize; i >= byteLength - nonVectorBytes; i -= elementSize) { UnsafeStoreNode.store(dest, header, i + destOffset, UnsafeLoadNode.load(src, header, i + srcOffset, baseKind), baseKind); } @@ -107,7 +107,6 @@ public static void checkNonNull(Object obj) { if (obj == null) { - probability(DEOPT_PATH_PROBABILITY); checkNPECounter.inc(); DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); } @@ -116,7 +115,6 @@ public static int checkArrayType(Word hub) { int layoutHelper = readLayoutHelper(hub); if (layoutHelper >= 0) { - probability(DEOPT_PATH_PROBABILITY); DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); } return layoutHelper; @@ -124,27 +122,22 @@ public static void checkLimits(Object src, int srcPos, Object dest, int destPos, int length) { if (srcPos < 0) { - probability(DEOPT_PATH_PROBABILITY); checkAIOOBECounter.inc(); DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); } if (destPos < 0) { - probability(DEOPT_PATH_PROBABILITY); checkAIOOBECounter.inc(); DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); } if (length < 0) { - probability(DEOPT_PATH_PROBABILITY); checkAIOOBECounter.inc(); DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); } if (srcPos + length > ArrayLengthNode.arrayLength(src)) { - probability(DEOPT_PATH_PROBABILITY); checkAIOOBECounter.inc(); DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); } if (destPos + length > ArrayLengthNode.arrayLength(dest)) { - probability(DEOPT_PATH_PROBABILITY); checkAIOOBECounter.inc(); DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); } @@ -278,13 +271,10 @@ int log2ElementSize = (layoutHelper >> layoutHelperLog2ElementSizeShift()) & layoutHelperLog2ElementSizeMask(); final boolean isObjectArray = ((layoutHelper & layoutHelperElementTypePrimitiveInPlace()) == 0); - if (srcHub.equal(destHub) && src != dest) { - probability(FAST_PATH_PROBABILITY); - + if (probability(FAST_PATH_PROBABILITY, srcHub.equal(destHub) && src != dest)) { checkLimits(src, srcPos, dest, destPos, length); - if (isObjectArray) { + if (probability(FAST_PATH_PROBABILITY, isObjectArray)) { genericObjectExactCallCounter.inc(); - probability(FAST_PATH_PROBABILITY); arrayObjectCopy(src, srcPos, dest, destPos, length); } else { genericPrimitiveCallCounter.inc();
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/CheckCastSnippets.java Thu Apr 25 20:10:49 2013 +0200 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/CheckCastSnippets.java Thu Apr 25 21:34:00 2013 +0200 @@ -65,13 +65,11 @@ */ @Snippet public static Object checkcastExact(Object object, Word exactHub, @ConstantParameter boolean checkNull) { - if (checkNull && object == null) { - probability(NOT_FREQUENT_PROBABILITY); + if (probability(NOT_FREQUENT_PROBABILITY, checkNull && object == null)) { isNull.inc(); } else { Word objectHub = loadHub(object); if (objectHub.notEqual(exactHub)) { - probability(DEOPT_PATH_PROBABILITY); exactMiss.inc(); DeoptimizeNode.deopt(InvalidateReprofile, ClassCastException); } @@ -92,13 +90,11 @@ */ @Snippet public static Object checkcastPrimary(Word hub, Object object, @ConstantParameter int superCheckOffset, @ConstantParameter boolean checkNull) { - if (checkNull && object == null) { - probability(NOT_FREQUENT_PROBABILITY); + if (probability(NOT_FREQUENT_PROBABILITY, checkNull && object == null)) { isNull.inc(); } else { Word objectHub = loadHub(object); if (objectHub.readWord(superCheckOffset, FINAL_LOCATION).notEqual(hub)) { - probability(DEOPT_PATH_PROBABILITY); displayMiss.inc(); DeoptimizeNode.deopt(InvalidateReprofile, ClassCastException); } @@ -113,8 +109,7 @@ */ @Snippet public static Object checkcastSecondary(Word hub, Object object, @VarargsParameter Word[] hints, @ConstantParameter boolean checkNull) { - if (checkNull && object == null) { - probability(NOT_FREQUENT_PROBABILITY); + if (probability(NOT_FREQUENT_PROBABILITY, checkNull && object == null)) { isNull.inc(); } else { Word objectHub = loadHub(object); @@ -142,8 +137,7 @@ */ @Snippet public static Object checkcastDynamic(Word hub, Object object, @ConstantParameter boolean checkNull) { - if (checkNull && object == null) { - probability(NOT_FREQUENT_PROBABILITY); + if (probability(NOT_FREQUENT_PROBABILITY, checkNull && object == null)) { isNull.inc(); } else { Word objectHub = loadHub(object);
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/HotSpotSnippetUtils.java Thu Apr 25 20:10:49 2013 +0200 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/HotSpotSnippetUtils.java Thu Apr 25 21:34:00 2013 +0200 @@ -608,11 +608,9 @@ // this code is independent from biased locking (although it does not look that way) final Word biasedLock = mark.and(biasedLockMaskInPlace()); - if (biasedLock.equal(Word.unsigned(unlockedMask()))) { - probability(FAST_PATH_PROBABILITY); + if (probability(FAST_PATH_PROBABILITY, biasedLock.equal(Word.unsigned(unlockedMask())))) { int hash = (int) mark.unsignedShiftRight(identityHashCodeShift()).rawValue(); - if (hash != uninitializedIdentityHashCodeValue()) { - probability(FAST_PATH_PROBABILITY); + if (probability(FAST_PATH_PROBABILITY, hash != uninitializedIdentityHashCodeValue())) { return hash; } }
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/InstanceOfSnippets.java Thu Apr 25 20:10:49 2013 +0200 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/InstanceOfSnippets.java Thu Apr 25 21:34:00 2013 +0200 @@ -58,14 +58,12 @@ */ @Snippet public static Object instanceofExact(Object object, Word exactHub, Object trueValue, Object falseValue, @ConstantParameter boolean checkNull) { - if (checkNull && object == null) { - probability(NOT_FREQUENT_PROBABILITY); + if (probability(NOT_FREQUENT_PROBABILITY, checkNull && object == null)) { isNull.inc(); return falseValue; } Word objectHub = loadHub(object); - if (objectHub.notEqual(exactHub)) { - probability(LIKELY_PROBABILITY); + if (probability(LIKELY_PROBABILITY, objectHub.notEqual(exactHub))) { exactMiss.inc(); return falseValue; } @@ -78,14 +76,12 @@ */ @Snippet public static Object instanceofPrimary(Word hub, Object object, @ConstantParameter int superCheckOffset, Object trueValue, Object falseValue, @ConstantParameter boolean checkNull) { - if (checkNull && object == null) { - probability(NOT_FREQUENT_PROBABILITY); + if (probability(NOT_FREQUENT_PROBABILITY, checkNull && object == null)) { isNull.inc(); return falseValue; } Word objectHub = loadHub(object); - if (objectHub.readWord(superCheckOffset, FINAL_LOCATION).notEqual(hub)) { - probability(NOT_LIKELY_PROBABILITY); + if (probability(NOT_LIKELY_PROBABILITY, objectHub.readWord(superCheckOffset, FINAL_LOCATION).notEqual(hub))) { displayMiss.inc(); return falseValue; } @@ -99,8 +95,7 @@ @Snippet public static Object instanceofSecondary(Word hub, Object object, @VarargsParameter Word[] hints, @VarargsParameter boolean[] hintIsPositive, Object trueValue, Object falseValue, @ConstantParameter boolean checkNull) { - if (checkNull && object == null) { - probability(NOT_FREQUENT_PROBABILITY); + if (probability(NOT_FREQUENT_PROBABILITY, checkNull && object == null)) { isNull.inc(); return falseValue; } @@ -110,8 +105,7 @@ for (int i = 0; i < hints.length; i++) { Word hintHub = hints[i]; boolean positive = hintIsPositive[i]; - if (hintHub.equal(objectHub)) { - probability(NOT_FREQUENT_PROBABILITY); + if (probability(NOT_FREQUENT_PROBABILITY, hintHub.equal(objectHub))) { hintsHit.inc(); return positive ? trueValue : falseValue; } @@ -127,8 +121,7 @@ */ @Snippet public static Object instanceofDynamic(Class mirror, Object object, Object trueValue, Object falseValue, @ConstantParameter boolean checkNull) { - if (checkNull && object == null) { - probability(NOT_FREQUENT_PROBABILITY); + if (probability(NOT_FREQUENT_PROBABILITY, checkNull && object == null)) { isNull.inc(); return falseValue; }
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/MonitorSnippets.java Thu Apr 25 20:10:49 2013 +0200 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/MonitorSnippets.java Thu Apr 25 21:34:00 2013 +0200 @@ -102,10 +102,9 @@ final Word biasableLockBits = mark.and(biasedLockMaskInPlace()); // First check to see whether biasing is enabled for this object - if (biasableLockBits.notEqual(Word.unsigned(biasedLockPattern()))) { + if (probability(NOT_FREQUENT_PROBABILITY, biasableLockBits.notEqual(Word.unsigned(biasedLockPattern())))) { // Biasing not enabled -> fall through to lightweight locking } else { - probability(FREQUENT_PROBABILITY); // The bias pattern is present in the object's mark word. Need to check // whether the bias owner and the epoch are both still current. Word hub = loadHub(object); @@ -115,9 +114,8 @@ trace(trace, "prototypeMarkWord: 0x%016lx\n", prototypeMarkWord); trace(trace, " thread: 0x%016lx\n", thread); trace(trace, " tmp: 0x%016lx\n", tmp); - if (tmp.equal(0)) { + if (probability(FREQUENT_PROBABILITY, tmp.equal(0))) { // Object is already biased to current thread -> done - probability(FREQUENT_PROBABILITY); traceObject(trace, "+lock{bias:existing}", object); return; } @@ -131,8 +129,7 @@ // If the low three bits in the xor result aren't clear, that means // the prototype header is no longer biasable and we have to revoke // the bias on this object. - if (tmp.and(biasedLockMaskInPlace()).equal(0)) { - probability(FREQUENT_PROBABILITY); + if (probability(FREQUENT_PROBABILITY, tmp.and(biasedLockMaskInPlace()).equal(0))) { // Biasing is still enabled for object's type. See whether the // epoch of the current bias is still valid, meaning that the epoch // bits of the mark word are equal to the epoch bits of the @@ -142,8 +139,7 @@ // that the current epoch is invalid in order to do this because // otherwise the manipulations it performs on the mark word are // illegal. - if (tmp.and(epochMaskInPlace()).equal(0)) { - probability(FREQUENT_PROBABILITY); + if (probability(FREQUENT_PROBABILITY, tmp.and(epochMaskInPlace()).equal(0))) { // The epoch of the current bias is still valid but we know nothing // about the owner; it might be set or it might be clear. Try to // acquire the bias of the object using an atomic operation. If this @@ -154,7 +150,7 @@ Word biasedMark = unbiasedMark.or(thread); trace(trace, " unbiasedMark: 0x%016lx\n", unbiasedMark); trace(trace, " biasedMark: 0x%016lx\n", biasedMark); - if (compareAndSwap(object, markOffset(), unbiasedMark, biasedMark, MARK_WORD_LOCATION).equal(unbiasedMark)) { + if (probability(VERY_FAST_DEOPT_PATH_PROBABILITY, compareAndSwap(object, markOffset(), unbiasedMark, biasedMark, MARK_WORD_LOCATION).equal(unbiasedMark))) { // Object is now biased to current thread -> done traceObject(trace, "+lock{bias:acquired}", object); return; @@ -162,7 +158,6 @@ // If the biasing toward our thread failed, this means that another thread // owns the bias and we need to revoke that bias. The revocation will occur // in the interpreter runtime. - probability(DEOPT_PATH_PROBABILITY); traceObject(trace, "+lock{stub:revoke}", object); MonitorEnterStubCall.call(object, lock); return; @@ -175,7 +170,7 @@ // the bias from one thread to another directly in this situation. Word biasedMark = prototypeMarkWord.or(thread); trace(trace, " biasedMark: 0x%016lx\n", biasedMark); - if (compareAndSwap(object, markOffset(), mark, biasedMark, MARK_WORD_LOCATION).equal(mark)) { + if (probability(VERY_FAST_DEOPT_PATH_PROBABILITY, compareAndSwap(object, markOffset(), mark, biasedMark, MARK_WORD_LOCATION).equal(mark))) { // Object is now biased to current thread -> done traceObject(trace, "+lock{bias:transfer}", object); return; @@ -183,7 +178,6 @@ // If the biasing toward our thread failed, then another thread // succeeded in biasing it toward itself and we need to revoke that // bias. The revocation will occur in the runtime in the slow case. - probability(DEOPT_PATH_PROBABILITY); traceObject(trace, "+lock{stub:epoch-expired}", object); MonitorEnterStubCall.call(object, lock); return; @@ -239,9 +233,8 @@ // significant 2 bits cleared and page_size is a power of 2 final Word alignedMask = Word.unsigned(wordSize() - 1); final Word stackPointer = stackPointer(); - if (currentMark.subtract(stackPointer).and(alignedMask.subtract(pageSize())).notEqual(0)) { + if (probability(VERY_SLOW_PATH_PROBABILITY, currentMark.subtract(stackPointer).and(alignedMask.subtract(pageSize())).notEqual(0))) { // Most likely not a recursive lock, go into a slow runtime call - probability(DEOPT_PATH_PROBABILITY); traceObject(trace, "+lock{stub:failed-cas}", object); MonitorEnterStubCall.call(object, lock); return; @@ -290,8 +283,7 @@ // the bias bit would be clear. final Word mark = loadWordFromObject(object, markOffset()); trace(trace, " mark: 0x%016lx\n", mark); - if (mark.and(biasedLockMaskInPlace()).equal(Word.unsigned(biasedLockPattern()))) { - probability(FREQUENT_PROBABILITY); + if (probability(FREQUENT_PROBABILITY, mark.and(biasedLockMaskInPlace()).equal(Word.unsigned(biasedLockPattern())))) { endLockScope(); decCounter(); traceObject(trace, "-lock{bias}", object); @@ -313,10 +305,9 @@ // Test if object's mark word is pointing to the displaced mark word, and if so, restore // the displaced mark in the object - if the object's mark word is not pointing to // the displaced mark word, do unlocking via runtime call. - if (DirectCompareAndSwapNode.compareAndSwap(object, markOffset(), lock, displacedMark, MARK_WORD_LOCATION).notEqual(lock)) { + if (probability(VERY_SLOW_PATH_PROBABILITY, DirectCompareAndSwapNode.compareAndSwap(object, markOffset(), lock, displacedMark, MARK_WORD_LOCATION).notEqual(lock))) { // The object's mark word was not pointing to the displaced header, // we do unlocking via runtime call. - probability(DEOPT_PATH_PROBABILITY); traceObject(trace, "-lock{stub}", object); MonitorExitStubCall.call(object); } else {
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/NewObjectSnippets.java Thu Apr 25 20:10:49 2013 +0200 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/NewObjectSnippets.java Thu Apr 25 21:34:00 2013 +0200 @@ -64,8 +64,7 @@ * this check might lead to problems if the TLAB is within 16GB of the address space end * (checked in c++ code) */ - if (newTop.belowOrEqual(end)) { - probability(FAST_PATH_PROBABILITY); + if (probability(FAST_PATH_PROBABILITY, newTop.belowOrEqual(end))) { writeTlabTop(thread, newTop); return top; } @@ -76,11 +75,10 @@ public static Object initializeObject(Word memory, Word hub, Word prototypeMarkWord, @ConstantParameter int size, @ConstantParameter boolean fillContents, @ConstantParameter boolean locked) { Object result; - if (memory.equal(0)) { + if (probability(SLOW_PATH_PROBABILITY, memory.equal(0))) { new_stub.inc(); result = NewInstanceStubCall.call(hub); } else { - probability(FAST_PATH_PROBABILITY); if (locked) { formatObject(hub, size, memory, thread().or(biasedLockPattern()), fillContents); } else { @@ -108,11 +106,10 @@ private static Object initializeArray(Word memory, Word hub, int length, int allocationSize, Word prototypeMarkWord, int headerSize, boolean fillContents) { Object result; - if (memory.equal(0)) { + if (probability(SLOW_PATH_PROBABILITY, memory.equal(0))) { newarray_stub.inc(); result = NewArrayStubCall.call(hub, length); } else { - probability(FAST_PATH_PROBABILITY); newarray_loopInit.inc(); formatArray(hub, allocationSize, length, headerSize, memory, prototypeMarkWord, fillContents); result = memory.toObject(); @@ -130,7 +127,6 @@ public static Object allocateArrayAndInitialize(int length, @ConstantParameter int alignment, @ConstantParameter int headerSize, @ConstantParameter int log2ElementSize, @ConstantParameter boolean fillContents, @ConstantParameter ResolvedJavaType type) { if (!belowThan(length, MAX_ARRAY_FAST_PATH_ALLOCATION_LENGTH)) { - probability(DEOPT_PATH_PROBABILITY); // This handles both negative array sizes and very large array sizes DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); }
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ObjectCloneSnippets.java Thu Apr 25 20:10:49 2013 +0200 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/ObjectCloneSnippets.java Thu Apr 25 21:34:00 2013 +0200 @@ -84,7 +84,6 @@ private static Word getAndCheckHub(Object src) { Word hub = loadHub(src); if (!(src instanceof Cloneable)) { - probability(DEOPT_PATH_PROBABILITY); DeoptimizeNode.deopt(DeoptimizationAction.None, DeoptimizationReason.RuntimeConstraint); } return hub; @@ -110,8 +109,7 @@ genericCloneCounter.inc(); Word hub = getAndCheckHub(src); int layoutHelper = hub.readInt(layoutHelperOffset(), FINAL_LOCATION); - if (layoutHelper < 0) { - probability(LIKELY_PROBABILITY); + if (probability(LIKELY_PROBABILITY, layoutHelper < 0)) { genericArrayCloneCounter.inc(); return arrayClone(src, hub, layoutHelper); } else {
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/SystemSubstitutions.java Thu Apr 25 20:10:49 2013 +0200 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/SystemSubstitutions.java Thu Apr 25 21:34:00 2013 +0200 @@ -56,8 +56,7 @@ @MethodSubstitution public static int identityHashCode(Object x) { - if (x == null) { - probability(NOT_FREQUENT_PROBABILITY); + if (probability(NOT_FREQUENT_PROBABILITY, x == null)) { return 0; }
--- a/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/TypeCheckSnippetUtils.java Thu Apr 25 20:10:49 2013 +0200 +++ b/graal/com.oracle.graal.hotspot/src/com/oracle/graal/hotspot/replacements/TypeCheckSnippetUtils.java Thu Apr 25 21:34:00 2013 +0200 @@ -89,8 +89,7 @@ Word secondarySupers = s.readWord(secondarySupersOffset(), SECONDARY_SUPERS_LOCATION); int length = secondarySupers.readInt(metaspaceArrayLengthOffset(), FINAL_LOCATION); for (int i = 0; i < length; i++) { - if (t.equal(loadSecondarySupersElement(secondarySupers, i))) { - probability(NOT_LIKELY_PROBABILITY); + if (probability(NOT_LIKELY_PROBABILITY, t.equal(loadSecondarySupersElement(secondarySupers, i)))) { s.writeWord(secondarySuperCacheOffset(), t, SECONDARY_SUPER_CACHE_LOCATION); secondariesHit.inc(); return true;
--- a/graal/com.oracle.graal.replacements.amd64/src/com/oracle/graal/replacements/amd64/AMD64ConvertSnippets.java Thu Apr 25 20:10:49 2013 +0200 +++ b/graal/com.oracle.graal.replacements.amd64/src/com/oracle/graal/replacements/amd64/AMD64ConvertSnippets.java Thu Apr 25 21:34:00 2013 +0200 @@ -57,8 +57,7 @@ */ @Snippet public static int f2i(float input, int result) { - if (result == Integer.MIN_VALUE) { - probability(NOT_FREQUENT_PROBABILITY); + if (probability(SLOW_PATH_PROBABILITY, result == Integer.MIN_VALUE)) { if (Float.isNaN(input)) { // input is NaN -> return 0 return 0; @@ -83,8 +82,7 @@ */ @Snippet public static long f2l(float input, long result) { - if (result == Long.MIN_VALUE) { - probability(NOT_FREQUENT_PROBABILITY); + if (probability(SLOW_PATH_PROBABILITY, result == Long.MIN_VALUE)) { if (Float.isNaN(input)) { // input is NaN -> return 0 return 0; @@ -109,8 +107,8 @@ */ @Snippet public static int d2i(double input, int result) { - if (result == Integer.MIN_VALUE) { - probability(NOT_FREQUENT_PROBABILITY); + if (probability(SLOW_PATH_PROBABILITY, result == Integer.MIN_VALUE)) { + if (Double.isNaN(input)) { // input is NaN -> return 0 return 0; @@ -135,8 +133,7 @@ */ @Snippet public static long d2l(double input, long result) { - if (result == Long.MIN_VALUE) { - probability(NOT_FREQUENT_PROBABILITY); + if (probability(SLOW_PATH_PROBABILITY, result == Long.MIN_VALUE)) { if (Double.isNaN(input)) { // input is NaN -> return 0 return 0;
--- a/graal/com.oracle.graal.replacements/src/com/oracle/graal/replacements/nodes/BranchProbabilityNode.java Thu Apr 25 20:10:49 2013 +0200 +++ b/graal/com.oracle.graal.replacements/src/com/oracle/graal/replacements/nodes/BranchProbabilityNode.java Thu Apr 25 21:34:00 2013 +0200 @@ -24,16 +24,15 @@ import com.oracle.graal.graph.*; import com.oracle.graal.nodes.*; +import com.oracle.graal.nodes.calc.*; import com.oracle.graal.nodes.spi.*; -import com.oracle.graal.nodes.type.*; -import com.oracle.graal.nodes.util.*; /** * Instances of this node class will look for a preceding if node and put the given probability into * the if node's taken probability. Then the branch probability node will be removed. This node is * intended primarily for snippets, so that they can define their fast and slow paths. */ -public class BranchProbabilityNode extends FixedWithNextNode implements Simplifiable, Lowerable { +public class BranchProbabilityNode extends FloatingNode implements Canonicalizable, Lowerable { public static final double LIKELY_PROBABILITY = 0.6; public static final double NOT_LIKELY_PROBABILITY = 1 - LIKELY_PROBABILITY; @@ -44,18 +43,28 @@ public static final double FAST_PATH_PROBABILITY = 0.99; public static final double SLOW_PATH_PROBABILITY = 1 - FAST_PATH_PROBABILITY; - public static final double NOT_DEOPT_PATH_PROBABILITY = 0.999; - public static final double DEOPT_PATH_PROBABILITY = 1 - NOT_DEOPT_PATH_PROBABILITY; + public static final double VERY_FAST_DEOPT_PATH_PROBABILITY = 0.999; + public static final double VERY_SLOW_PATH_PROBABILITY = 1 - VERY_FAST_DEOPT_PATH_PROBABILITY; @Input private ValueNode probability; + @Input private ValueNode condition; - public BranchProbabilityNode(ValueNode probability) { - super(StampFactory.forVoid()); + public BranchProbabilityNode(ValueNode probability, ValueNode condition) { + super(condition.stamp()); this.probability = probability; + this.condition = condition; + } + + public ValueNode getProbability() { + return probability; + } + + public ValueNode getCondition() { + return condition; } @Override - public void simplify(SimplifierTool tool) { + public ValueNode canonical(CanonicalizerTool tool) { if (probability.isConstant()) { double probabilityValue = probability.asConstant().asDouble(); if (probabilityValue < 0.0) { @@ -63,40 +72,22 @@ } else if (probabilityValue > 1.0) { throw new GraalInternalError("A probability of more than 1.0 (" + probabilityValue + ") is not allowed!"); } - FixedNode current = this; - while (!(current instanceof BeginNode)) { - current = (FixedNode) current.predecessor(); - } - BeginNode begin = (BeginNode) current; - if (!(begin.predecessor() instanceof IfNode)) { - return; + for (IfNode ifNodeUsages : this.usages().filter(IfNode.class)) { + ifNodeUsages.setTrueSuccessorProbability(probabilityValue); } - IfNode ifNode = (IfNode) begin.predecessor(); - if (ifNode.trueSuccessor() == begin) { - ifNode.setTrueSuccessorProbability(probabilityValue); - } else { - ifNode.setTrueSuccessorProbability(1 - probabilityValue); - } - - FixedNode next = next(); - setNext(null); - ((FixedWithNextNode) predecessor()).setNext(next); - GraphUtil.killCFG(this); + return condition; } + return this; } @SuppressWarnings("unused") @NodeIntrinsic - public static void probability(double probability) { + public static boolean probability(double probability, boolean condition) { + return condition; } @Override public void lower(LoweringTool tool, LoweringType loweringType) { - if (probability.isConstant()) { - throw new GraalInternalError("Injected branch probability must follow an if node."); - } else { - throw new GraalInternalError("Branch probability could not be injected, because the probability value did not reduce to a constant value."); - } + throw new GraalInternalError("Branch probability could not be injected, because the probability value did not reduce to a constant value."); } - }