# HG changeset patch # User kvn # Date 1321463637 28800 # Node ID 1bd45abaa507630264fc739faf8ef0ea91fde873 # Parent 6729bbc1fcd605a2c7c92c74197b1c33bbe240b0 6890673: Eliminate allocations immediately after EA Summary: Try to eliminate allocations and related locks immediately after escape analysis. Reviewed-by: never diff -r 6729bbc1fcd6 -r 1bd45abaa507 src/share/vm/opto/block.cpp --- a/src/share/vm/opto/block.cpp Wed Nov 16 01:39:50 2011 -0800 +++ b/src/share/vm/opto/block.cpp Wed Nov 16 09:13:57 2011 -0800 @@ -898,45 +898,41 @@ void PhaseCFG::verify( ) const { #ifdef ASSERT // Verify sane CFG - for( uint i = 0; i < _num_blocks; i++ ) { + for (uint i = 0; i < _num_blocks; i++) { Block *b = _blocks[i]; uint cnt = b->_nodes.size(); uint j; - for( j = 0; j < cnt; j++ ) { + for (j = 0; j < cnt; j++) { Node *n = b->_nodes[j]; assert( _bbs[n->_idx] == b, "" ); - if( j >= 1 && n->is_Mach() && - n->as_Mach()->ideal_Opcode() == Op_CreateEx ) { - assert( j == 1 || b->_nodes[j-1]->is_Phi(), - "CreateEx must be first instruction in block" ); + if (j >= 1 && n->is_Mach() && + n->as_Mach()->ideal_Opcode() == Op_CreateEx) { + assert(j == 1 || b->_nodes[j-1]->is_Phi(), + "CreateEx must be first instruction in block"); } - for( uint k = 0; k < n->req(); k++ ) { + for (uint k = 0; k < n->req(); k++) { Node *def = n->in(k); - if( def && def != n ) { - assert( _bbs[def->_idx] || def->is_Con(), - "must have block; constants for debug info ok" ); + if (def && def != n) { + assert(_bbs[def->_idx] || def->is_Con(), + "must have block; constants for debug info ok"); // Verify that instructions in the block is in correct order. // Uses must follow their definition if they are at the same block. // Mostly done to check that MachSpillCopy nodes are placed correctly // when CreateEx node is moved in build_ifg_physical(). - if( _bbs[def->_idx] == b && + if (_bbs[def->_idx] == b && !(b->head()->is_Loop() && n->is_Phi()) && // See (+++) comment in reg_split.cpp - !(n->jvms() != NULL && n->jvms()->is_monitor_use(k)) ) { + !(n->jvms() != NULL && n->jvms()->is_monitor_use(k))) { bool is_loop = false; if (n->is_Phi()) { - for( uint l = 1; l < def->req(); l++ ) { + for (uint l = 1; l < def->req(); l++) { if (n == def->in(l)) { is_loop = true; break; // Some kind of loop } } } - assert( is_loop || b->find_node(def) < j, "uses must follow definitions" ); - } - if( def->is_SafePointScalarObject() ) { - assert(_bbs[def->_idx] == b, "SafePointScalarObject Node should be at the same block as its SafePoint node"); - assert(_bbs[def->_idx] == _bbs[def->in(0)->_idx], "SafePointScalarObject Node should be at the same block as its control edge"); + assert(is_loop || b->find_node(def) < j, "uses must follow definitions"); } } } @@ -946,12 +942,11 @@ Node *bp = (Node*)b->_nodes[b->_nodes.size()-1]->is_block_proj(); assert( bp, "last instruction must be a block proj" ); assert( bp == b->_nodes[j], "wrong number of successors for this block" ); - if( bp->is_Catch() ) { - while( b->_nodes[--j]->is_MachProj() ) ; - assert( b->_nodes[j]->is_MachCall(), "CatchProj must follow call" ); - } - else if( bp->is_Mach() && bp->as_Mach()->ideal_Opcode() == Op_If ) { - assert( b->_num_succs == 2, "Conditional branch must have two targets"); + if (bp->is_Catch()) { + while (b->_nodes[--j]->is_MachProj()) ; + assert(b->_nodes[j]->is_MachCall(), "CatchProj must follow call"); + } else if (bp->is_Mach() && bp->as_Mach()->ideal_Opcode() == Op_If) { + assert(b->_num_succs == 2, "Conditional branch must have two targets"); } } #endif diff -r 6729bbc1fcd6 -r 1bd45abaa507 src/share/vm/opto/callnode.cpp --- a/src/share/vm/opto/callnode.cpp Wed Nov 16 01:39:50 2011 -0800 +++ b/src/share/vm/opto/callnode.cpp Wed Nov 16 09:13:57 2011 -0800 @@ -1071,8 +1071,11 @@ init_class_id(Class_SafePointScalarObject); } -bool SafePointScalarObjectNode::pinned() const { return true; } -bool SafePointScalarObjectNode::depends_only_on_test() const { return false; } +// Do not allow value-numbering for SafePointScalarObject node. +uint SafePointScalarObjectNode::hash() const { return NO_HASH; } +uint SafePointScalarObjectNode::cmp( const Node &n ) const { + return (&n == this); // Always fail except on self +} uint SafePointScalarObjectNode::ideal_reg() const { return 0; // No matching to machine instruction @@ -1096,7 +1099,6 @@ if (cached != NULL) { return (SafePointScalarObjectNode*)cached; } - Compile* C = Compile::current(); SafePointScalarObjectNode* res = (SafePointScalarObjectNode*)Node::clone(); res->_first_index += jvms_adj; sosn_map->Insert((void*)this, (void*)res); @@ -1142,6 +1144,8 @@ Node* AllocateArrayNode::Ideal(PhaseGVN *phase, bool can_reshape) { if (remove_dead_region(phase, can_reshape)) return this; + // Don't bother trying to transform a dead node + if (in(0) && in(0)->is_top()) return NULL; const Type* type = phase->type(Ideal_length()); if (type->isa_int() && type->is_int()->_hi < 0) { @@ -1522,13 +1526,16 @@ // perform any generic optimizations first (returns 'this' or NULL) Node *result = SafePointNode::Ideal(phase, can_reshape); + if (result != NULL) return result; + // Don't bother trying to transform a dead node + if (in(0) && in(0)->is_top()) return NULL; // Now see if we can optimize away this lock. We don't actually // remove the locking here, we simply set the _eliminate flag which // prevents macro expansion from expanding the lock. Since we don't // modify the graph, the value returned from this function is the // one computed above. - if (result == NULL && can_reshape && EliminateLocks && !is_eliminated()) { + if (can_reshape && EliminateLocks && (!is_eliminated() || is_coarsened())) { // // If we are locking an unescaped object, the lock/unlock is unnecessary // @@ -1537,8 +1544,16 @@ if (cgr != NULL) es = cgr->escape_state(obj_node()); if (es != PointsToNode::UnknownEscape && es != PointsToNode::GlobalEscape) { - // Mark it eliminated to update any counters - this->set_eliminated(); + if (!is_eliminated()) { + // Mark it eliminated to update any counters + this->set_eliminated(); + } else { + assert(is_coarsened(), "sanity"); + // The lock could be marked eliminated by lock coarsening + // code during first IGVN before EA. Clear coarsened flag + // to eliminate all associated locks/unlocks. + this->clear_coarsened(); + } return result; } @@ -1546,7 +1561,7 @@ // Try lock coarsening // PhaseIterGVN* iter = phase->is_IterGVN(); - if (iter != NULL) { + if (iter != NULL && !is_eliminated()) { GrowableArray lock_ops; @@ -1602,7 +1617,7 @@ lock->set_eliminated(); lock->set_coarsened(); } - } else if (result != NULL && ctrl->is_Region() && + } else if (ctrl->is_Region() && iter->_worklist.member(ctrl)) { // We weren't able to find any opportunities but the region this // lock is control dependent on hasn't been processed yet so put @@ -1623,7 +1638,10 @@ Node *UnlockNode::Ideal(PhaseGVN *phase, bool can_reshape) { // perform any generic optimizations first (returns 'this' or NULL) - Node * result = SafePointNode::Ideal(phase, can_reshape); + Node *result = SafePointNode::Ideal(phase, can_reshape); + if (result != NULL) return result; + // Don't bother trying to transform a dead node + if (in(0) && in(0)->is_top()) return NULL; // Now see if we can optimize away this unlock. We don't actually // remove the unlocking here, we simply set the _eliminate flag which @@ -1631,7 +1649,7 @@ // modify the graph, the value returned from this function is the // one computed above. // Escape state is defined after Parse phase. - if (result == NULL && can_reshape && EliminateLocks && !is_eliminated()) { + if (can_reshape && EliminateLocks && (!is_eliminated() || is_coarsened())) { // // If we are unlocking an unescaped object, the lock/unlock is unnecessary. // @@ -1640,8 +1658,16 @@ if (cgr != NULL) es = cgr->escape_state(obj_node()); if (es != PointsToNode::UnknownEscape && es != PointsToNode::GlobalEscape) { - // Mark it eliminated to update any counters - this->set_eliminated(); + if (!is_eliminated()) { + // Mark it eliminated to update any counters + this->set_eliminated(); + } else { + assert(is_coarsened(), "sanity"); + // The lock could be marked eliminated by lock coarsening + // code during first IGVN before EA. Clear coarsened flag + // to eliminate all associated locks/unlocks. + this->clear_coarsened(); + } } } return result; diff -r 6729bbc1fcd6 -r 1bd45abaa507 src/share/vm/opto/callnode.hpp --- a/src/share/vm/opto/callnode.hpp Wed Nov 16 01:39:50 2011 -0800 +++ b/src/share/vm/opto/callnode.hpp Wed Nov 16 09:13:57 2011 -0800 @@ -440,6 +440,10 @@ // states of the scalarized object fields are collected. uint _n_fields; // Number of non-static fields of the scalarized object. DEBUG_ONLY(AllocateNode* _alloc;) + + virtual uint hash() const ; // { return NO_HASH; } + virtual uint cmp( const Node &n ) const; + public: SafePointScalarObjectNode(const TypeOopPtr* tp, #ifdef ASSERT @@ -454,15 +458,10 @@ uint first_index() const { return _first_index; } uint n_fields() const { return _n_fields; } - DEBUG_ONLY(AllocateNode* alloc() const { return _alloc; }) - // SafePointScalarObject should be always pinned to the control edge - // of the SafePoint node for which it was generated. - virtual bool pinned() const; // { return true; } - - // SafePointScalarObject depends on the SafePoint node - // for which it was generated. - virtual bool depends_only_on_test() const; // { return false; } +#ifdef ASSERT + AllocateNode* alloc() const { return _alloc; } +#endif virtual uint size_of() const { return sizeof(*this); } @@ -880,6 +879,7 @@ bool is_coarsened() { return _coarsened; } void set_coarsened() { _coarsened = true; } + void clear_coarsened() { _coarsened = false; } // locking does not modify its arguments virtual bool may_modify(const TypePtr *addr_t, PhaseTransform *phase){ return false;} diff -r 6729bbc1fcd6 -r 1bd45abaa507 src/share/vm/opto/compile.cpp --- a/src/share/vm/opto/compile.cpp Wed Nov 16 01:39:50 2011 -0800 +++ b/src/share/vm/opto/compile.cpp Wed Nov 16 09:13:57 2011 -0800 @@ -1711,11 +1711,22 @@ if (failing()) return; + // Optimize out fields loads from scalar replaceable allocations. igvn.optimize(); print_method("Iter GVN after EA", 2); if (failing()) return; + if (congraph() != NULL && macro_count() > 0) { + PhaseMacroExpand mexp(igvn); + mexp.eliminate_macro_nodes(); + igvn.set_delay_transform(false); + + igvn.optimize(); + print_method("Iter GVN after eliminating allocations and locks", 2); + + if (failing()) return; + } } // Loop transforms on the ideal graph. Range Check Elimination, diff -r 6729bbc1fcd6 -r 1bd45abaa507 src/share/vm/opto/escape.cpp --- a/src/share/vm/opto/escape.cpp Wed Nov 16 01:39:50 2011 -0800 +++ b/src/share/vm/opto/escape.cpp Wed Nov 16 09:13:57 2011 -0800 @@ -1772,12 +1772,20 @@ Node *n = C->macro_node(i); if (n->is_AbstractLock()) { // Lock and Unlock nodes AbstractLockNode* alock = n->as_AbstractLock(); - if (!alock->is_eliminated()) { + if (!alock->is_eliminated() || alock->is_coarsened()) { PointsToNode::EscapeState es = escape_state(alock->obj_node()); assert(es != PointsToNode::UnknownEscape, "should know"); if (es != PointsToNode::UnknownEscape && es != PointsToNode::GlobalEscape) { - // Mark it eliminated - alock->set_eliminated(); + if (!alock->is_eliminated()) { + // Mark it eliminated to update any counters + alock->set_eliminated(); + } else { + // The lock could be marked eliminated by lock coarsening + // code during first IGVN before EA. Clear coarsened flag + // to eliminate all associated locks/unlocks and relock + // during deoptimization. + alock->clear_coarsened(); + } } } } diff -r 6729bbc1fcd6 -r 1bd45abaa507 src/share/vm/opto/gcm.cpp --- a/src/share/vm/opto/gcm.cpp Wed Nov 16 01:39:50 2011 -0800 +++ b/src/share/vm/opto/gcm.cpp Wed Nov 16 09:13:57 2011 -0800 @@ -95,7 +95,7 @@ assert(in0 != NULL, "Only control-dependent"); const Node *p = in0->is_block_proj(); if (p != NULL && p != n) { // Control from a block projection? - assert(!n->pinned() || n->is_MachConstantBase() || n->is_SafePointScalarObject(), "only pinned MachConstantBase or SafePointScalarObject node is expected here"); + assert(!n->pinned() || n->is_MachConstantBase(), "only pinned MachConstantBase node is expected here"); // Find trailing Region Block *pb = _bbs[in0->_idx]; // Block-projection already has basic block uint j = 0; diff -r 6729bbc1fcd6 -r 1bd45abaa507 src/share/vm/opto/loopnode.cpp --- a/src/share/vm/opto/loopnode.cpp Wed Nov 16 01:39:50 2011 -0800 +++ b/src/share/vm/opto/loopnode.cpp Wed Nov 16 09:13:57 2011 -0800 @@ -1946,7 +1946,7 @@ } // Nothing to do, so get out - if( !C->has_loops() && !do_split_ifs && !_verify_me && !_verify_only ) { + if( !C->has_loops() && !skip_loop_opts && !do_split_ifs && !_verify_me && !_verify_only ) { _igvn.optimize(); // Cleanup NeverBranches return; } diff -r 6729bbc1fcd6 -r 1bd45abaa507 src/share/vm/opto/macro.cpp --- a/src/share/vm/opto/macro.cpp Wed Nov 16 01:39:50 2011 -0800 +++ b/src/share/vm/opto/macro.cpp Wed Nov 16 09:13:57 2011 -0800 @@ -81,7 +81,7 @@ uint old_unique = C->unique(); Node* new_in = old_sosn->clone(jvms_adj, sosn_map); if (old_unique != C->unique()) { - new_in->set_req(0, newcall->in(0)); // reset control edge + new_in->set_req(0, C->root()); // reset control edge new_in = transform_later(new_in); // Register new node. } old_in = new_in; @@ -565,7 +565,6 @@ if (res == NULL) { // All users were eliminated. } else if (!res->is_CheckCastPP()) { - alloc->_is_scalar_replaceable = false; // don't try again NOT_PRODUCT(fail_eliminate = "Allocation does not have unique CheckCastPP";) can_eliminate = false; } else { @@ -719,7 +718,7 @@ alloc, #endif first_ind, nfields); - sobj->init_req(0, sfpt->in(TypeFunc::Control)); + sobj->init_req(0, C->root()); transform_later(sobj); // Scan object's fields adding an input to the safepoint for each field. @@ -762,10 +761,10 @@ Node *field_val = value_from_mem(mem, basic_elem_type, field_type, field_addr_type, alloc); if (field_val == NULL) { - // we weren't able to find a value for this field, - // give up on eliminating this allocation - alloc->_is_scalar_replaceable = false; // don't try again - // remove any extra entries we added to the safepoint + // We weren't able to find a value for this field, + // give up on eliminating this allocation. + + // Remove any extra entries we added to the safepoint. uint last = sfpt->req() - 1; for (int k = 0; k < j; k++) { sfpt->del_req(last--); @@ -1804,9 +1803,9 @@ #ifndef PRODUCT if (PrintEliminateLocks) { if (alock->is_Lock()) { - tty->print_cr("++++ Eliminating: %d Lock", alock->_idx); + tty->print_cr("++++ Eliminated: %d Lock", alock->_idx); } else { - tty->print_cr("++++ Eliminating: %d Unlock", alock->_idx); + tty->print_cr("++++ Eliminated: %d Unlock", alock->_idx); } } #endif @@ -2165,11 +2164,12 @@ _igvn.replace_node(_memproj_fallthrough, mem_phi); } -//------------------------------expand_macro_nodes---------------------- -// Returns true if a failure occurred. -bool PhaseMacroExpand::expand_macro_nodes() { +//---------------------------eliminate_macro_nodes---------------------- +// Eliminate scalar replaced allocations and associated locks. +void PhaseMacroExpand::eliminate_macro_nodes() { if (C->macro_count() == 0) - return false; + return; + // First, attempt to eliminate locks int cnt = C->macro_count(); for (int i=0; i < cnt; i++) { @@ -2189,14 +2189,6 @@ debug_only(int old_macro_count = C->macro_count();); if (n->is_AbstractLock()) { success = eliminate_locking_node(n->as_AbstractLock()); - } else if (n->Opcode() == Op_LoopLimit) { - // Remove it from macro list and put on IGVN worklist to optimize. - C->remove_macro_node(n); - _igvn._worklist.push(n); - success = true; - } else if (n->Opcode() == Op_Opaque1 || n->Opcode() == Op_Opaque2) { - _igvn.replace_node(n, n->in(1)); - success = true; } assert(success == (C->macro_count() < old_macro_count), "elimination reduces macro count"); progress = progress || success; @@ -2220,18 +2212,50 @@ assert(!n->as_AbstractLock()->is_eliminated(), "sanity"); break; default: - assert(false, "unknown node type in macro list"); + assert(n->Opcode() == Op_LoopLimit || + n->Opcode() == Op_Opaque1 || + n->Opcode() == Op_Opaque2, "unknown node type in macro list"); } assert(success == (C->macro_count() < old_macro_count), "elimination reduces macro count"); progress = progress || success; } } +} + +//------------------------------expand_macro_nodes---------------------- +// Returns true if a failure occurred. +bool PhaseMacroExpand::expand_macro_nodes() { + // Last attempt to eliminate macro nodes. + eliminate_macro_nodes(); + // Make sure expansion will not cause node limit to be exceeded. // Worst case is a macro node gets expanded into about 50 nodes. // Allow 50% more for optimization. if (C->check_node_count(C->macro_count() * 75, "out of nodes before macro expansion" ) ) return true; + // Eliminate Opaque and LoopLimit nodes. Do it after all loop optimizations. + bool progress = true; + while (progress) { + progress = false; + for (int i = C->macro_count(); i > 0; i--) { + Node * n = C->macro_node(i-1); + bool success = false; + debug_only(int old_macro_count = C->macro_count();); + if (n->Opcode() == Op_LoopLimit) { + // Remove it from macro list and put on IGVN worklist to optimize. + C->remove_macro_node(n); + _igvn._worklist.push(n); + success = true; + } else if (n->Opcode() == Op_Opaque1 || n->Opcode() == Op_Opaque2) { + _igvn.replace_node(n, n->in(1)); + success = true; + } + assert(success == (C->macro_count() < old_macro_count), "elimination reduces macro count"); + progress = progress || success; + } + } + // expand "macro" nodes // nodes are removed from the macro list as they are processed while (C->macro_count() > 0) { @@ -2265,5 +2289,6 @@ _igvn.set_delay_transform(false); _igvn.optimize(); + if (C->failing()) return true; return false; } diff -r 6729bbc1fcd6 -r 1bd45abaa507 src/share/vm/opto/macro.hpp --- a/src/share/vm/opto/macro.hpp Wed Nov 16 01:39:50 2011 -0800 +++ b/src/share/vm/opto/macro.hpp Wed Nov 16 09:13:57 2011 -0800 @@ -119,6 +119,7 @@ PhaseMacroExpand(PhaseIterGVN &igvn) : Phase(Macro_Expand), _igvn(igvn) { _igvn.set_delay_transform(true); } + void eliminate_macro_nodes(); bool expand_macro_nodes(); }; diff -r 6729bbc1fcd6 -r 1bd45abaa507 src/share/vm/opto/memnode.cpp --- a/src/share/vm/opto/memnode.cpp Wed Nov 16 01:39:50 2011 -0800 +++ b/src/share/vm/opto/memnode.cpp Wed Nov 16 09:13:57 2011 -0800 @@ -2661,6 +2661,8 @@ // control copies Node *StrIntrinsicNode::Ideal(PhaseGVN *phase, bool can_reshape) { if (remove_dead_region(phase, can_reshape)) return this; + // Don't bother trying to transform a dead node + if (in(0) && in(0)->is_top()) return NULL; if (can_reshape) { Node* mem = phase->transform(in(MemNode::Memory)); @@ -2675,6 +2677,12 @@ return NULL; } +//------------------------------Value------------------------------------------ +const Type *StrIntrinsicNode::Value( PhaseTransform *phase ) const { + if (in(0) && phase->type(in(0)) == Type::TOP) return Type::TOP; + return bottom_type(); +} + //============================================================================= MemBarNode::MemBarNode(Compile* C, int alias_idx, Node* precedent) : MultiNode(TypeFunc::Parms + (precedent == NULL? 0: 1)), @@ -2715,6 +2723,8 @@ // control copies Node *MemBarNode::Ideal(PhaseGVN *phase, bool can_reshape) { if (remove_dead_region(phase, can_reshape)) return this; + // Don't bother trying to transform a dead node + if (in(0) && in(0)->is_top()) return NULL; // Eliminate volatile MemBars for scalar replaced objects. if (can_reshape && req() == (Precedent+1) && diff -r 6729bbc1fcd6 -r 1bd45abaa507 src/share/vm/opto/memnode.hpp --- a/src/share/vm/opto/memnode.hpp Wed Nov 16 01:39:50 2011 -0800 +++ b/src/share/vm/opto/memnode.hpp Wed Nov 16 09:13:57 2011 -0800 @@ -800,6 +800,7 @@ virtual uint match_edge(uint idx) const; virtual uint ideal_reg() const { return Op_RegI; } virtual Node *Ideal(PhaseGVN *phase, bool can_reshape); + virtual const Type *Value(PhaseTransform *phase) const; }; //------------------------------StrComp-------------------------------------