# HG changeset patch # User kvn # Date 1342127955 25200 # Node ID cc787232c4c59a56a52f143b2f7fee886ea5b164 # Parent 90d5a592ea8f58a0850648c554ce2806fb3c15af# Parent ae9241bbce4a64e67d85727f8af39f6d8c1471e3 Merge diff -r 90d5a592ea8f -r cc787232c4c5 src/share/vm/opto/library_call.cpp --- a/src/share/vm/opto/library_call.cpp Thu Jul 12 14:26:25 2012 -0400 +++ b/src/share/vm/opto/library_call.cpp Thu Jul 12 14:19:15 2012 -0700 @@ -160,6 +160,7 @@ bool inline_trans(vmIntrinsics::ID id); bool inline_abs(vmIntrinsics::ID id); bool inline_sqrt(vmIntrinsics::ID id); + void finish_pow_exp(Node* result, Node* x, Node* y, const TypeFunc* call_type, address funcAddr, const char* funcName); bool inline_pow(vmIntrinsics::ID id); bool inline_exp(vmIntrinsics::ID id); bool inline_min_max(vmIntrinsics::ID id); @@ -1535,40 +1536,79 @@ return true; } +void LibraryCallKit::finish_pow_exp(Node* result, Node* x, Node* y, const TypeFunc* call_type, address funcAddr, const char* funcName) { + //------------------- + //result=(result.isNaN())? funcAddr():result; + // Check: If isNaN() by checking result!=result? then either trap + // or go to runtime + Node* cmpisnan = _gvn.transform(new (C, 3) CmpDNode(result,result)); + // Build the boolean node + Node* bolisnum = _gvn.transform( new (C, 2) BoolNode(cmpisnan, BoolTest::eq) ); + + if (!too_many_traps(Deoptimization::Reason_intrinsic)) { + { + BuildCutout unless(this, bolisnum, PROB_STATIC_FREQUENT); + // End the current control-flow path + push_pair(x); + if (y != NULL) { + push_pair(y); + } + // The pow or exp intrinsic returned a NaN, which requires a call + // to the runtime. Recompile with the runtime call. + uncommon_trap(Deoptimization::Reason_intrinsic, + Deoptimization::Action_make_not_entrant); + } + push_pair(result); + } else { + // If this inlining ever returned NaN in the past, we compile a call + // to the runtime to properly handle corner cases + + IfNode* iff = create_and_xform_if(control(), bolisnum, PROB_STATIC_FREQUENT, COUNT_UNKNOWN); + Node* if_slow = _gvn.transform( new (C, 1) IfFalseNode(iff) ); + Node* if_fast = _gvn.transform( new (C, 1) IfTrueNode(iff) ); + + if (!if_slow->is_top()) { + RegionNode* result_region = new(C, 3) RegionNode(3); + PhiNode* result_val = new (C, 3) PhiNode(result_region, Type::DOUBLE); + + result_region->init_req(1, if_fast); + result_val->init_req(1, result); + + set_control(if_slow); + + const TypePtr* no_memory_effects = NULL; + Node* rt = make_runtime_call(RC_LEAF, call_type, funcAddr, funcName, + no_memory_effects, + x, top(), y, y ? top() : NULL); + Node* value = _gvn.transform(new (C, 1) ProjNode(rt, TypeFunc::Parms+0)); +#ifdef ASSERT + Node* value_top = _gvn.transform(new (C, 1) ProjNode(rt, TypeFunc::Parms+1)); + assert(value_top == top(), "second value must be top"); +#endif + + result_region->init_req(2, control()); + result_val->init_req(2, value); + push_result(result_region, result_val); + } else { + push_pair(result); + } + } +} + //------------------------------inline_exp------------------------------------- // Inline exp instructions, if possible. The Intel hardware only misses // really odd corner cases (+/- Infinity). Just uncommon-trap them. bool LibraryCallKit::inline_exp(vmIntrinsics::ID id) { assert(id == vmIntrinsics::_dexp, "Not exp"); - // If this inlining ever returned NaN in the past, we do not intrinsify it - // every again. NaN results requires StrictMath.exp handling. - if (too_many_traps(Deoptimization::Reason_intrinsic)) return false; - _sp += arg_size(); // restore stack pointer Node *x = pop_math_arg(); Node *result = _gvn.transform(new (C, 2) ExpDNode(0,x)); - //------------------- - //result=(result.isNaN())? StrictMath::exp():result; - // Check: If isNaN() by checking result!=result? then go to Strict Math - Node* cmpisnan = _gvn.transform(new (C, 3) CmpDNode(result,result)); - // Build the boolean node - Node* bolisnum = _gvn.transform( new (C, 2) BoolNode(cmpisnan, BoolTest::eq) ); - - { BuildCutout unless(this, bolisnum, PROB_STATIC_FREQUENT); - // End the current control-flow path - push_pair(x); - // Math.exp intrinsic returned a NaN, which requires StrictMath.exp - // to handle. Recompile without intrinsifying Math.exp - uncommon_trap(Deoptimization::Reason_intrinsic, - Deoptimization::Action_make_not_entrant); - } + finish_pow_exp(result, x, NULL, OptoRuntime::Math_D_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dexp), "EXP"); C->set_has_split_ifs(true); // Has chance for split-if optimization - push_pair(result); - return true; } @@ -1577,17 +1617,12 @@ bool LibraryCallKit::inline_pow(vmIntrinsics::ID id) { assert(id == vmIntrinsics::_dpow, "Not pow"); - // If this inlining ever returned NaN in the past, we do not intrinsify it - // every again. NaN results requires StrictMath.pow handling. - if (too_many_traps(Deoptimization::Reason_intrinsic)) return false; - - // Do not intrinsify on older platforms which lack cmove. - if (ConditionalMoveLimit == 0) return false; - // Pseudocode for pow // if (x <= 0.0) { - // if ((double)((int)y)==y) { // if y is int - // result = ((1&(int)y)==0)?-DPow(abs(x), y):DPow(abs(x), y) + // long longy = (long)y; + // if ((double)longy == y) { // if y is long + // if (y + 1 == y) longy = 0; // huge number: even + // result = ((1&longy) == 0)?-DPow(abs(x), y):DPow(abs(x), y); // } else { // result = NaN; // } @@ -1595,7 +1630,7 @@ // result = DPow(x,y); // } // if (result != result)? { - // uncommon_trap(); + // result = uncommon_trap() or runtime_call(); // } // return result; @@ -1603,15 +1638,14 @@ Node* y = pop_math_arg(); Node* x = pop_math_arg(); - Node *fast_result = _gvn.transform( new (C, 3) PowDNode(0, x, y) ); - - // Short form: if not top-level (i.e., Math.pow but inlining Math.pow - // inside of something) then skip the fancy tests and just check for - // NaN result. - Node *result = NULL; - if( jvms()->depth() >= 1 ) { - result = fast_result; + Node* result = NULL; + + if (!too_many_traps(Deoptimization::Reason_intrinsic)) { + // Short form: skip the fancy tests and just check for NaN result. + result = _gvn.transform( new (C, 3) PowDNode(0, x, y) ); } else { + // If this inlining ever returned NaN in the past, include all + // checks + call to the runtime. // Set the merge point for If node with condition of (x <= 0.0) // There are four possible paths to region node and phi node @@ -1627,55 +1661,95 @@ Node *bol1 = _gvn.transform( new (C, 2) BoolNode( cmp, BoolTest::le ) ); // Branch either way IfNode *if1 = create_and_xform_if(control(),bol1, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN); - Node *opt_test = _gvn.transform(if1); - //assert( opt_test->is_If(), "Expect an IfNode"); - IfNode *opt_if1 = (IfNode*)opt_test; // Fast path taken; set region slot 3 - Node *fast_taken = _gvn.transform( new (C, 1) IfFalseNode(opt_if1) ); + Node *fast_taken = _gvn.transform( new (C, 1) IfFalseNode(if1) ); r->init_req(3,fast_taken); // Capture fast-control // Fast path not-taken, i.e. slow path - Node *complex_path = _gvn.transform( new (C, 1) IfTrueNode(opt_if1) ); + Node *complex_path = _gvn.transform( new (C, 1) IfTrueNode(if1) ); // Set fast path result - Node *fast_result = _gvn.transform( new (C, 3) PowDNode(0, y, x) ); + Node *fast_result = _gvn.transform( new (C, 3) PowDNode(0, x, y) ); phi->init_req(3, fast_result); // Complex path - // Build the second if node (if y is int) - // Node for (int)y - Node *inty = _gvn.transform( new (C, 2) ConvD2INode(y)); - // Node for (double)((int) y) - Node *doubleinty= _gvn.transform( new (C, 2) ConvI2DNode(inty)); - // Check (double)((int) y) : y - Node *cmpinty= _gvn.transform(new (C, 3) CmpDNode(doubleinty, y)); - // Check if (y isn't int) then go to slow path - - Node *bol2 = _gvn.transform( new (C, 2) BoolNode( cmpinty, BoolTest::ne ) ); + // Build the second if node (if y is long) + // Node for (long)y + Node *longy = _gvn.transform( new (C, 2) ConvD2LNode(y)); + // Node for (double)((long) y) + Node *doublelongy= _gvn.transform( new (C, 2) ConvL2DNode(longy)); + // Check (double)((long) y) : y + Node *cmplongy= _gvn.transform(new (C, 3) CmpDNode(doublelongy, y)); + // Check if (y isn't long) then go to slow path + + Node *bol2 = _gvn.transform( new (C, 2) BoolNode( cmplongy, BoolTest::ne ) ); // Branch either way IfNode *if2 = create_and_xform_if(complex_path,bol2, PROB_STATIC_INFREQUENT, COUNT_UNKNOWN); - Node *slow_path = opt_iff(r,if2); // Set region path 2 - - // Calculate DPow(abs(x), y)*(1 & (int)y) + Node* ylong_path = _gvn.transform( new (C, 1) IfFalseNode(if2)); + + Node *slow_path = _gvn.transform( new (C, 1) IfTrueNode(if2) ); + + // Calculate DPow(abs(x), y)*(1 & (long)y) // Node for constant 1 - Node *conone = intcon(1); - // 1& (int)y - Node *signnode= _gvn.transform( new (C, 3) AndINode(conone, inty) ); + Node *conone = longcon(1); + // 1& (long)y + Node *signnode= _gvn.transform( new (C, 3) AndLNode(conone, longy) ); + + // A huge number is always even. Detect a huge number by checking + // if y + 1 == y and set integer to be tested for parity to 0. + // Required for corner case: + // (long)9.223372036854776E18 = max_jlong + // (double)(long)9.223372036854776E18 = 9.223372036854776E18 + // max_jlong is odd but 9.223372036854776E18 is even + Node* yplus1 = _gvn.transform( new (C, 3) AddDNode(y, makecon(TypeD::make(1)))); + Node *cmpyplus1= _gvn.transform(new (C, 3) CmpDNode(yplus1, y)); + Node *bolyplus1 = _gvn.transform( new (C, 2) BoolNode( cmpyplus1, BoolTest::eq ) ); + Node* correctedsign = NULL; + if (ConditionalMoveLimit != 0) { + correctedsign = _gvn.transform( CMoveNode::make(C, NULL, bolyplus1, signnode, longcon(0), TypeLong::LONG)); + } else { + IfNode *ifyplus1 = create_and_xform_if(ylong_path,bolyplus1, PROB_FAIR, COUNT_UNKNOWN); + RegionNode *r = new (C, 3) RegionNode(3); + Node *phi = new (C, 3) PhiNode(r, TypeLong::LONG); + r->init_req(1, _gvn.transform( new (C, 1) IfFalseNode(ifyplus1))); + r->init_req(2, _gvn.transform( new (C, 1) IfTrueNode(ifyplus1))); + phi->init_req(1, signnode); + phi->init_req(2, longcon(0)); + correctedsign = _gvn.transform(phi); + ylong_path = _gvn.transform(r); + record_for_igvn(r); + } + // zero node - Node *conzero = intcon(0); - // Check (1&(int)y)==0? - Node *cmpeq1 = _gvn.transform(new (C, 3) CmpINode(signnode, conzero)); - // Check if (1&(int)y)!=0?, if so the result is negative + Node *conzero = longcon(0); + // Check (1&(long)y)==0? + Node *cmpeq1 = _gvn.transform(new (C, 3) CmpLNode(correctedsign, conzero)); + // Check if (1&(long)y)!=0?, if so the result is negative Node *bol3 = _gvn.transform( new (C, 2) BoolNode( cmpeq1, BoolTest::ne ) ); // abs(x) Node *absx=_gvn.transform( new (C, 2) AbsDNode(x)); // abs(x)^y - Node *absxpowy = _gvn.transform( new (C, 3) PowDNode(0, y, absx) ); + Node *absxpowy = _gvn.transform( new (C, 3) PowDNode(0, absx, y) ); // -abs(x)^y Node *negabsxpowy = _gvn.transform(new (C, 2) NegDNode (absxpowy)); - // (1&(int)y)==1?-DPow(abs(x), y):DPow(abs(x), y) - Node *signresult = _gvn.transform( CMoveNode::make(C, NULL, bol3, absxpowy, negabsxpowy, Type::DOUBLE)); + // (1&(long)y)==1?-DPow(abs(x), y):DPow(abs(x), y) + Node *signresult = NULL; + if (ConditionalMoveLimit != 0) { + signresult = _gvn.transform( CMoveNode::make(C, NULL, bol3, absxpowy, negabsxpowy, Type::DOUBLE)); + } else { + IfNode *ifyeven = create_and_xform_if(ylong_path,bol3, PROB_FAIR, COUNT_UNKNOWN); + RegionNode *r = new (C, 3) RegionNode(3); + Node *phi = new (C, 3) PhiNode(r, Type::DOUBLE); + r->init_req(1, _gvn.transform( new (C, 1) IfFalseNode(ifyeven))); + r->init_req(2, _gvn.transform( new (C, 1) IfTrueNode(ifyeven))); + phi->init_req(1, absxpowy); + phi->init_req(2, negabsxpowy); + signresult = _gvn.transform(phi); + ylong_path = _gvn.transform(r); + record_for_igvn(r); + } // Set complex path fast result + r->init_req(2, ylong_path); phi->init_req(2, signresult); static const jlong nan_bits = CONST64(0x7ff8000000000000); @@ -1689,27 +1763,10 @@ result=_gvn.transform(phi); } - //------------------- - //result=(result.isNaN())? uncommon_trap():result; - // Check: If isNaN() by checking result!=result? then go to Strict Math - Node* cmpisnan = _gvn.transform(new (C, 3) CmpDNode(result,result)); - // Build the boolean node - Node* bolisnum = _gvn.transform( new (C, 2) BoolNode(cmpisnan, BoolTest::eq) ); - - { BuildCutout unless(this, bolisnum, PROB_STATIC_FREQUENT); - // End the current control-flow path - push_pair(x); - push_pair(y); - // Math.pow intrinsic returned a NaN, which requires StrictMath.pow - // to handle. Recompile without intrinsifying Math.pow. - uncommon_trap(Deoptimization::Reason_intrinsic, - Deoptimization::Action_make_not_entrant); - } + finish_pow_exp(result, x, y, OptoRuntime::Math_DD_D_Type(), CAST_FROM_FN_PTR(address, SharedRuntime::dpow), "POW"); C->set_has_split_ifs(true); // Has chance for split-if optimization - push_pair(result); - return true; } diff -r 90d5a592ea8f -r cc787232c4c5 src/share/vm/opto/parse2.cpp --- a/src/share/vm/opto/parse2.cpp Thu Jul 12 14:26:25 2012 -0400 +++ b/src/share/vm/opto/parse2.cpp Thu Jul 12 14:19:15 2012 -0700 @@ -1278,9 +1278,9 @@ // or the narrowOop equivalent. const Type* obj_type = _gvn.type(obj); const TypeOopPtr* tboth = obj_type->join(con_type)->isa_oopptr(); - if (tboth != NULL && tboth != obj_type && tboth->higher_equal(obj_type)) { + if (tboth != NULL && tboth->klass_is_exact() && tboth != obj_type && + tboth->higher_equal(obj_type)) { // obj has to be of the exact type Foo if the CmpP succeeds. - assert(tboth->klass_is_exact(), "klass should be exact"); int obj_in_map = map()->find_edge(obj); JVMState* jvms = this->jvms(); if (obj_in_map >= 0 && diff -r 90d5a592ea8f -r cc787232c4c5 src/share/vm/opto/subnode.cpp --- a/src/share/vm/opto/subnode.cpp Thu Jul 12 14:26:25 2012 -0400 +++ b/src/share/vm/opto/subnode.cpp Thu Jul 12 14:19:15 2012 -0700 @@ -554,9 +554,7 @@ return TypeInt::CC_GE; } else if (hi0 <= lo1) { // Check for special case in Hashtable::get. (See below.) - if ((jint)lo0 >= 0 && (jint)lo1 >= 0 && - in(1)->Opcode() == Op_ModI && - in(1)->in(2) == in(2) ) + if ((jint)lo0 >= 0 && (jint)lo1 >= 0 && is_index_range_check()) return TypeInt::CC_LT; return TypeInt::CC_LE; } @@ -567,13 +565,17 @@ // to be positive. // (This is a gross hack, since the sub method never // looks at the structure of the node in any other case.) - if ((jint)lo0 >= 0 && (jint)lo1 >= 0 && - in(1)->Opcode() == Op_ModI && - in(1)->in(2)->uncast() == in(2)->uncast()) + if ((jint)lo0 >= 0 && (jint)lo1 >= 0 && is_index_range_check()) return TypeInt::CC_LT; return TypeInt::CC; // else use worst case results } +bool CmpUNode::is_index_range_check() const { + // Check for the "(X ModI Y) CmpU Y" shape + return (in(1)->Opcode() == Op_ModI && + in(1)->in(2)->eqv_uncast(in(2))); +} + //------------------------------Idealize--------------------------------------- Node *CmpINode::Ideal( PhaseGVN *phase, bool can_reshape ) { if (phase->type(in(2))->higher_equal(TypeInt::ZERO)) { diff -r 90d5a592ea8f -r cc787232c4c5 src/share/vm/opto/subnode.hpp --- a/src/share/vm/opto/subnode.hpp Thu Jul 12 14:26:25 2012 -0400 +++ b/src/share/vm/opto/subnode.hpp Thu Jul 12 14:19:15 2012 -0700 @@ -158,6 +158,7 @@ CmpUNode( Node *in1, Node *in2 ) : CmpNode(in1,in2) {} virtual int Opcode() const; virtual const Type *sub( const Type *, const Type * ) const; + bool is_index_range_check() const; }; //------------------------------CmpPNode--------------------------------------- diff -r 90d5a592ea8f -r cc787232c4c5 test/compiler/7177917/Test7177917.java --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/test/compiler/7177917/Test7177917.java Thu Jul 12 14:19:15 2012 -0700 @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +/* + * Micro-benchmark for Math.pow() and Math.exp() + */ + +import java.util.*; + +public class Test7177917 { + + static double d; + + static Random r = new Random(0); + + static long m_pow(double[][] values) { + double res = 0; + long start = System.nanoTime(); + for (int i = 0; i < values.length; i++) { + res += Math.pow(values[i][0], values[i][1]); + } + long stop = System.nanoTime(); + d = res; + return (stop - start) / 1000; + } + + static long m_exp(double[] values) { + double res = 0; + long start = System.nanoTime(); + for (int i = 0; i < values.length; i++) { + res += Math.exp(values[i]); + } + long stop = System.nanoTime(); + d = res; + return (stop - start) / 1000; + } + + static double[][] pow_values(int nb) { + double[][] res = new double[nb][2]; + for (int i = 0; i < nb; i++) { + double ylogx = (1 + (r.nextDouble() * 2045)) - 1023; // 2045 rather than 2046 as a safety margin + double x = Math.abs(Double.longBitsToDouble(r.nextLong())); + while (x != x) { + x = Math.abs(Double.longBitsToDouble(r.nextLong())); + } + double logx = Math.log(x) / Math.log(2); + double y = ylogx / logx; + + res[i][0] = x; + res[i][1] = y; + } + return res; + } + + static double[] exp_values(int nb) { + double[] res = new double[nb]; + for (int i = 0; i < nb; i++) { + double ylogx = (1 + (r.nextDouble() * 2045)) - 1023; // 2045 rather than 2046 as a safety margin + double x = Math.E; + double logx = Math.log(x) / Math.log(2); + double y = ylogx / logx; + res[i] = y; + } + return res; + } + + static public void main(String[] args) { + { + // warmup + double[][] warmup_values = pow_values(10); + m_pow(warmup_values); + + for (int i = 0; i < 20000; i++) { + m_pow(warmup_values); + } + // test pow perf + double[][] values = pow_values(1000000); + System.out.println("==> POW " + m_pow(values)); + + // force uncommon trap + double[][] nan_values = new double[1][2]; + nan_values[0][0] = Double.NaN; + nan_values[0][1] = Double.NaN; + m_pow(nan_values); + + // force recompilation + for (int i = 0; i < 20000; i++) { + m_pow(warmup_values); + } + + // test pow perf again + System.out.println("==> POW " + m_pow(values)); + } + { + // warmup + double[] warmup_values = exp_values(10); + m_exp(warmup_values); + + for (int i = 0; i < 20000; i++) { + m_exp(warmup_values); + } + + // test pow perf + double[] values = exp_values(1000000); + System.out.println("==> EXP " + m_exp(values)); + + // force uncommon trap + double[] nan_values = new double[1]; + nan_values[0] = Double.NaN; + m_exp(nan_values); + + // force recompilation + for (int i = 0; i < 20000; i++) { + m_exp(warmup_values); + } + + // test pow perf again + System.out.println("==> EXP " + m_exp(values)); + } + } +}