Mercurial > hg > truffle
diff src/share/vm/opto/library_call.cpp @ 833:acba6af809c8
6840775: Multiple JVM crashes seen with 1.6.0_10 through 1.6.0_14
Summary: Put missed reference to allocated array in copyOf() intrinsic into OopMap for the call slow_arraycopy().
Reviewed-by: never
author | kvn |
---|---|
date | Wed, 01 Jul 2009 20:22:18 -0700 |
parents | bf3489cc0aa0 |
children | 4325cdaa78ad |
line wrap: on
line diff
--- a/src/share/vm/opto/library_call.cpp Wed Jul 01 15:06:54 2009 -0700 +++ b/src/share/vm/opto/library_call.cpp Wed Jul 01 20:22:18 2009 -0700 @@ -165,6 +165,7 @@ bool inline_native_getLength(); bool inline_array_copyOf(bool is_copyOfRange); bool inline_array_equals(); + void copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark); bool inline_native_clone(bool is_virtual); bool inline_native_Reflection_getCallerClass(); bool inline_native_AtomicLong_get(); @@ -181,7 +182,6 @@ Node* src, Node* src_offset, Node* dest, Node* dest_offset, Node* copy_length, - int nargs, // arguments on stack for debug info bool disjoint_bases = false, bool length_never_negative = false, RegionNode* slow_region = NULL); @@ -202,17 +202,16 @@ void generate_slow_arraycopy(const TypePtr* adr_type, Node* src, Node* src_offset, Node* dest, Node* dest_offset, - Node* copy_length, - int nargs); + Node* copy_length); Node* generate_checkcast_arraycopy(const TypePtr* adr_type, Node* dest_elem_klass, Node* src, Node* src_offset, Node* dest, Node* dest_offset, - Node* copy_length, int nargs); + Node* copy_length); Node* generate_generic_arraycopy(const TypePtr* adr_type, Node* src, Node* src_offset, Node* dest, Node* dest_offset, - Node* copy_length, int nargs); + Node* copy_length); void generate_unchecked_arraycopy(const TypePtr* adr_type, BasicType basic_elem_type, bool disjoint_bases, @@ -3229,7 +3228,8 @@ Node* orig_tail = _gvn.transform( new(C, 3) SubINode(orig_length, start) ); Node* moved = generate_min_max(vmIntrinsics::_min, orig_tail, length); - Node* newcopy = new_array(klass_node, length, nargs); + const bool raw_mem_only = true; + Node* newcopy = new_array(klass_node, length, nargs, raw_mem_only); // Generate a direct call to the right arraycopy function(s). // We know the copy is disjoint but we might not know if the @@ -3240,7 +3240,7 @@ bool length_never_negative = true; generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT, original, start, newcopy, intcon(0), moved, - nargs, disjoint_bases, length_never_negative); + disjoint_bases, length_never_negative); push(newcopy); } @@ -3882,6 +3882,98 @@ return true; } +//------------------------clone_coping----------------------------------- +// Helper function for inline_native_clone. +void LibraryCallKit::copy_to_clone(Node* obj, Node* alloc_obj, Node* obj_size, bool is_array, bool card_mark) { + assert(obj_size != NULL, ""); + Node* raw_obj = alloc_obj->in(1); + assert(alloc_obj->is_CheckCastPP() && raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), ""); + + if (ReduceBulkZeroing) { + // We will be completely responsible for initializing this object - + // mark Initialize node as complete. + AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn); + // The object was just allocated - there should be no any stores! + guarantee(alloc != NULL && alloc->maybe_set_complete(&_gvn), ""); + } + + // Cast to Object for arraycopy. + // We can't use the original CheckCastPP since it should be moved + // after the arraycopy to prevent stores flowing above it. + Node* new_obj = new(C, 2) CheckCastPPNode(alloc_obj->in(0), raw_obj, + TypeInstPtr::NOTNULL); + new_obj = _gvn.transform(new_obj); + // Substitute in the locally valid dest_oop. + replace_in_map(alloc_obj, new_obj); + + // Copy the fastest available way. + // TODO: generate fields copies for small objects instead. + Node* src = obj; + Node* dest = new_obj; + Node* size = _gvn.transform(obj_size); + + // Exclude the header but include array length to copy by 8 bytes words. + // Can't use base_offset_in_bytes(bt) since basic type is unknown. + int base_off = is_array ? arrayOopDesc::length_offset_in_bytes() : + instanceOopDesc::base_offset_in_bytes(); + // base_off: + // 8 - 32-bit VM + // 12 - 64-bit VM, compressed oops + // 16 - 64-bit VM, normal oops + if (base_off % BytesPerLong != 0) { + assert(UseCompressedOops, ""); + if (is_array) { + // Exclude length to copy by 8 bytes words. + base_off += sizeof(int); + } else { + // Include klass to copy by 8 bytes words. + base_off = instanceOopDesc::klass_offset_in_bytes(); + } + assert(base_off % BytesPerLong == 0, "expect 8 bytes alignment"); + } + src = basic_plus_adr(src, base_off); + dest = basic_plus_adr(dest, base_off); + + // Compute the length also, if needed: + Node* countx = size; + countx = _gvn.transform( new (C, 3) SubXNode(countx, MakeConX(base_off)) ); + countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong) )); + + const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM; + bool disjoint_bases = true; + generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases, + src, NULL, dest, NULL, countx); + + // If necessary, emit some card marks afterwards. (Non-arrays only.) + if (card_mark) { + assert(!is_array, ""); + // Put in store barrier for any and all oops we are sticking + // into this object. (We could avoid this if we could prove + // that the object type contains no oop fields at all.) + Node* no_particular_value = NULL; + Node* no_particular_field = NULL; + int raw_adr_idx = Compile::AliasIdxRaw; + post_barrier(control(), + memory(raw_adr_type), + new_obj, + no_particular_field, + raw_adr_idx, + no_particular_value, + T_OBJECT, + false); + } + + // Move the original CheckCastPP after arraycopy. + _gvn.hash_delete(alloc_obj); + alloc_obj->set_req(0, control()); + // Replace raw memory edge with new CheckCastPP to have a live oop + // at safepoints instead of raw value. + assert(new_obj->is_CheckCastPP() && new_obj->in(1) == alloc_obj->in(1), "sanity"); + alloc_obj->set_req(1, new_obj); // cast to the original type + _gvn.hash_find_insert(alloc_obj); // put back into GVN table + // Restore in the locally valid dest_oop. + replace_in_map(new_obj, alloc_obj); +} //------------------------inline_native_clone---------------------------- // Here are the simple edge cases: @@ -3916,8 +4008,9 @@ // paths into result_reg: enum { _slow_path = 1, // out-of-line call to clone method (virtual or not) - _objArray_path, // plain allocation, plus arrayof_oop_arraycopy - _fast_path, // plain allocation, plus a CopyArray operation + _objArray_path, // plain array allocation, plus arrayof_oop_arraycopy + _array_path, // plain array allocation, plus arrayof_long_arraycopy + _instance_path, // plain instance allocation, plus arrayof_long_arraycopy PATH_LIMIT }; RegionNode* result_reg = new(C, PATH_LIMIT) RegionNode(PATH_LIMIT); @@ -3932,18 +4025,6 @@ int raw_adr_idx = Compile::AliasIdxRaw; const bool raw_mem_only = true; - // paths into alloc_reg (on the fast path, just before the CopyArray): - enum { _typeArray_alloc = 1, _instance_alloc, ALLOC_LIMIT }; - RegionNode* alloc_reg = new(C, ALLOC_LIMIT) RegionNode(ALLOC_LIMIT); - PhiNode* alloc_val = new(C, ALLOC_LIMIT) PhiNode(alloc_reg, raw_adr_type); - PhiNode* alloc_siz = new(C, ALLOC_LIMIT) PhiNode(alloc_reg, TypeX_X); - PhiNode* alloc_i_o = new(C, ALLOC_LIMIT) PhiNode(alloc_reg, Type::ABIO); - PhiNode* alloc_mem = new(C, ALLOC_LIMIT) PhiNode(alloc_reg, Type::MEMORY, - raw_adr_type); - record_for_igvn(alloc_reg); - - bool card_mark = false; // (see below) - Node* array_ctl = generate_array_guard(obj_klass, (RegionNode*)NULL); if (array_ctl != NULL) { // It's an array. @@ -3953,16 +4034,6 @@ Node* obj_size = NULL; Node* alloc_obj = new_array(obj_klass, obj_length, nargs, raw_mem_only, &obj_size); - assert(obj_size != NULL, ""); - Node* raw_obj = alloc_obj->in(1); - assert(raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), ""); - if (ReduceBulkZeroing) { - AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn); - if (alloc != NULL) { - // We will be completely responsible for initializing this object. - alloc->maybe_set_complete(&_gvn); - } - } if (!use_ReduceInitialCardMarks()) { // If it is an oop array, it requires very special treatment, @@ -3976,7 +4047,7 @@ bool length_never_negative = true; generate_arraycopy(TypeAryPtr::OOPS, T_OBJECT, obj, intcon(0), alloc_obj, intcon(0), - obj_length, nargs, + obj_length, disjoint_bases, length_never_negative); result_reg->init_req(_objArray_path, control()); result_val->init_req(_objArray_path, alloc_obj); @@ -3991,19 +4062,24 @@ // the object. // Otherwise, there are no card marks to worry about. - alloc_val->init_req(_typeArray_alloc, raw_obj); - alloc_siz->init_req(_typeArray_alloc, obj_size); - alloc_reg->init_req(_typeArray_alloc, control()); - alloc_i_o->init_req(_typeArray_alloc, i_o()); - alloc_mem->init_req(_typeArray_alloc, memory(raw_adr_type)); + + if (!stopped()) { + copy_to_clone(obj, alloc_obj, obj_size, true, false); + + // Present the results of the copy. + result_reg->init_req(_array_path, control()); + result_val->init_req(_array_path, alloc_obj); + result_i_o ->set_req(_array_path, i_o()); + result_mem ->set_req(_array_path, reset_memory()); + } } - // We only go to the fast case code if we pass a number of guards. + // We only go to the instance fast case code if we pass a number of guards. // The paths which do not pass are accumulated in the slow_region. RegionNode* slow_region = new (C, 1) RegionNode(1); record_for_igvn(slow_region); if (!stopped()) { - // It's an instance. Make the slow-path tests. + // It's an instance (we did array above). Make the slow-path tests. // If this is a virtual call, we generate a funny guard. We grab // the vtable entry corresponding to clone() from the target object. // If the target method which we are calling happens to be the @@ -4030,25 +4106,14 @@ PreserveJVMState pjvms(this); Node* obj_size = NULL; Node* alloc_obj = new_instance(obj_klass, NULL, raw_mem_only, &obj_size); - assert(obj_size != NULL, ""); - Node* raw_obj = alloc_obj->in(1); - assert(raw_obj->is_Proj() && raw_obj->in(0)->is_Allocate(), ""); - if (ReduceBulkZeroing) { - AllocateNode* alloc = AllocateNode::Ideal_allocation(alloc_obj, &_gvn); - if (alloc != NULL && !alloc->maybe_set_complete(&_gvn)) - alloc = NULL; - } - if (!use_ReduceInitialCardMarks()) { - // Put in store barrier for any and all oops we are sticking - // into this object. (We could avoid this if we could prove - // that the object type contains no oop fields at all.) - card_mark = true; - } - alloc_val->init_req(_instance_alloc, raw_obj); - alloc_siz->init_req(_instance_alloc, obj_size); - alloc_reg->init_req(_instance_alloc, control()); - alloc_i_o->init_req(_instance_alloc, i_o()); - alloc_mem->init_req(_instance_alloc, memory(raw_adr_type)); + + copy_to_clone(obj, alloc_obj, obj_size, false, !use_ReduceInitialCardMarks()); + + // Present the results of the slow call. + result_reg->init_req(_instance_path, control()); + result_val->init_req(_instance_path, alloc_obj); + result_i_o ->set_req(_instance_path, i_o()); + result_mem ->set_req(_instance_path, reset_memory()); } // Generate code for the slow case. We make a call to clone(). @@ -4064,82 +4129,12 @@ result_mem ->set_req(_slow_path, reset_memory()); } - // The object is allocated, as an array and/or an instance. Now copy it. - set_control( _gvn.transform(alloc_reg) ); - set_i_o( _gvn.transform(alloc_i_o) ); - set_memory( _gvn.transform(alloc_mem), raw_adr_type ); - Node* raw_obj = _gvn.transform(alloc_val); - - if (!stopped()) { - // Copy the fastest available way. - // (No need for PreserveJVMState, since we're using it all up now.) - // TODO: generate fields/elements copies for small objects instead. - Node* src = obj; - Node* dest = raw_obj; - Node* size = _gvn.transform(alloc_siz); - - // Exclude the header. - int base_off = instanceOopDesc::base_offset_in_bytes(); - if (UseCompressedOops) { - assert(base_off % BytesPerLong != 0, "base with compressed oops"); - // With compressed oops base_offset_in_bytes is 12 which creates - // the gap since countx is rounded by 8 bytes below. - // Copy klass and the gap. - base_off = instanceOopDesc::klass_offset_in_bytes(); - } - src = basic_plus_adr(src, base_off); - dest = basic_plus_adr(dest, base_off); - - // Compute the length also, if needed: - Node* countx = size; - countx = _gvn.transform( new (C, 3) SubXNode(countx, MakeConX(base_off)) ); - countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong) )); - - // Select an appropriate instruction to initialize the range. - // The CopyArray instruction (if supported) can be optimized - // into a discrete set of scalar loads and stores. - bool disjoint_bases = true; - generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases, - src, NULL, dest, NULL, countx); - - // Now that the object is properly initialized, type it as an oop. - // Use a secondary InitializeNode memory barrier. - InitializeNode* init = insert_mem_bar_volatile(Op_Initialize, raw_adr_idx, - raw_obj)->as_Initialize(); - init->set_complete(&_gvn); // (there is no corresponding AllocateNode) - Node* new_obj = new(C, 2) CheckCastPPNode(control(), raw_obj, - TypeInstPtr::NOTNULL); - new_obj = _gvn.transform(new_obj); - - // If necessary, emit some card marks afterwards. (Non-arrays only.) - if (card_mark) { - Node* no_particular_value = NULL; - Node* no_particular_field = NULL; - post_barrier(control(), - memory(raw_adr_type), - new_obj, - no_particular_field, - raw_adr_idx, - no_particular_value, - T_OBJECT, - false); - } - // Present the results of the slow call. - result_reg->init_req(_fast_path, control()); - result_val->init_req(_fast_path, new_obj); - result_i_o ->set_req(_fast_path, i_o()); - result_mem ->set_req(_fast_path, reset_memory()); - } - // Return the combined state. set_control( _gvn.transform(result_reg) ); set_i_o( _gvn.transform(result_i_o) ); set_all_memory( _gvn.transform(result_mem) ); - // Cast the result to a sharper type, since we know what clone does. - Node* new_obj = _gvn.transform(result_val); - Node* cast = new (C, 2) CheckCastPPNode(control(), new_obj, toop); - push(_gvn.transform(cast)); + push(_gvn.transform(result_val)); return true; } @@ -4278,8 +4273,7 @@ // Call StubRoutines::generic_arraycopy stub. generate_arraycopy(TypeRawPtr::BOTTOM, T_CONFLICT, - src, src_offset, dest, dest_offset, length, - nargs); + src, src_offset, dest, dest_offset, length); // Do not let reads from the destination float above the arraycopy. // Since we cannot type the arrays, we don't know which slices @@ -4302,8 +4296,7 @@ // The component types are not the same or are not recognized. Punt. // (But, avoid the native method wrapper to JVM_ArrayCopy.) generate_slow_arraycopy(TypePtr::BOTTOM, - src, src_offset, dest, dest_offset, length, - nargs); + src, src_offset, dest, dest_offset, length); return true; } @@ -4360,7 +4353,7 @@ const TypePtr* adr_type = TypeAryPtr::get_array_body_type(dest_elem); generate_arraycopy(adr_type, dest_elem, src, src_offset, dest, dest_offset, length, - nargs, false, false, slow_region); + false, false, slow_region); return true; } @@ -4405,7 +4398,6 @@ Node* src, Node* src_offset, Node* dest, Node* dest_offset, Node* copy_length, - int nargs, bool disjoint_bases, bool length_never_negative, RegionNode* slow_region) { @@ -4417,7 +4409,6 @@ Node* original_dest = dest; AllocateArrayNode* alloc = NULL; // used for zeroing, if needed - Node* raw_dest = NULL; // used before zeroing, if needed bool must_clear_dest = false; // See if this is the initialization of a newly-allocated array. @@ -4436,15 +4427,18 @@ // "You break it, you buy it." InitializeNode* init = alloc->initialization(); assert(init->is_complete(), "we just did this"); - assert(dest->Opcode() == Op_CheckCastPP, "sanity"); + assert(dest->is_CheckCastPP(), "sanity"); assert(dest->in(0)->in(0) == init, "dest pinned"); - raw_dest = dest->in(1); // grab the raw pointer! - original_dest = dest; - dest = raw_dest; + + // Cast to Object for arraycopy. + // We can't use the original CheckCastPP since it should be moved + // after the arraycopy to prevent stores flowing above it. + Node* new_obj = new(C, 2) CheckCastPPNode(dest->in(0), dest->in(1), + TypeInstPtr::NOTNULL); + dest = _gvn.transform(new_obj); + // Substitute in the locally valid dest_oop. + replace_in_map(original_dest, dest); adr_type = TypeRawPtr::BOTTOM; // all initializations are into raw memory - // Decouple the original InitializeNode, turning it into a simple membar. - // We will build a new one at the end of this routine. - init->set_req(InitializeNode::RawAddress, top()); // From this point on, every exit path is responsible for // initializing any non-copied parts of the object to zero. must_clear_dest = true; @@ -4487,7 +4481,7 @@ assert(!must_clear_dest, ""); Node* cv = generate_generic_arraycopy(adr_type, src, src_offset, dest, dest_offset, - copy_length, nargs); + copy_length); if (cv == NULL) cv = intcon(-1); // failure (no stub available) checked_control = control(); checked_i_o = i_o(); @@ -4519,6 +4513,14 @@ } } + // Use a secondary InitializeNode as raw memory barrier. + // Currently it is needed only on this path since other + // paths have stub or runtime calls as raw memory barriers. + InitializeNode* init = insert_mem_bar_volatile(Op_Initialize, + Compile::AliasIdxRaw, + top())->as_Initialize(); + init->set_complete(&_gvn); // (there is no corresponding AllocateNode) + // Present the results of the fast call. result_region->init_req(zero_path, control()); result_i_o ->init_req(zero_path, i_o()); @@ -4637,8 +4639,7 @@ Node* cv = generate_checkcast_arraycopy(adr_type, dest_elem_klass, src, src_offset, dest, dest_offset, - copy_length, - nargs); + copy_length); if (cv == NULL) cv = intcon(-1); // failure (no stub available) checked_control = control(); checked_i_o = i_o(); @@ -4700,8 +4701,8 @@ slow_i_o2 ->init_req(1, slow_i_o); slow_mem2 ->init_req(1, slow_mem); slow_reg2 ->init_req(2, control()); - slow_i_o2 ->init_req(2, i_o()); - slow_mem2 ->init_req(2, memory(adr_type)); + slow_i_o2 ->init_req(2, checked_i_o); + slow_mem2 ->init_req(2, checked_mem); slow_control = _gvn.transform(slow_reg2); slow_i_o = _gvn.transform(slow_i_o2); @@ -4746,21 +4747,9 @@ alloc->in(AllocateNode::AllocSize)); } - if (dest != original_dest) { - // Promote from rawptr to oop, so it looks right in the call's GC map. - dest = _gvn.transform( new(C,2) CheckCastPPNode(control(), dest, - TypeInstPtr::NOTNULL) ); - - // Edit the call's debug-info to avoid referring to original_dest. - // (The problem with original_dest is that it isn't ready until - // after the InitializeNode completes, but this stuff is before.) - // Substitute in the locally valid dest_oop. - replace_in_map(original_dest, dest); - } - generate_slow_arraycopy(adr_type, src, src_offset, dest, dest_offset, - copy_length, nargs); + copy_length); result_region->init_req(slow_call_path, control()); result_i_o ->init_req(slow_call_path, i_o()); @@ -4780,16 +4769,16 @@ if (dest != original_dest) { // Pin the "finished" array node after the arraycopy/zeroing operations. - // Use a secondary InitializeNode memory barrier. - InitializeNode* init = insert_mem_bar_volatile(Op_Initialize, - Compile::AliasIdxRaw, - raw_dest)->as_Initialize(); - init->set_complete(&_gvn); // (there is no corresponding AllocateNode) _gvn.hash_delete(original_dest); original_dest->set_req(0, control()); + // Replace raw memory edge with new CheckCastPP to have a live oop + // at safepoints instead of raw value. + assert(dest->is_CheckCastPP() && dest->in(1) == original_dest->in(1), "sanity"); + original_dest->set_req(1, dest); // cast to the original type _gvn.hash_find_insert(original_dest); // put back into GVN table + // Restore in the locally valid dest_oop. + replace_in_map(dest, original_dest); } - // The memory edges above are precise in order to model effects around // array copies accurately to allow value numbering of field loads around // arraycopy. Such field loads, both before and after, are common in Java @@ -5073,16 +5062,13 @@ LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type, Node* src, Node* src_offset, Node* dest, Node* dest_offset, - Node* copy_length, - int nargs) { - _sp += nargs; // any deopt will start just before call to enclosing method + Node* copy_length) { Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON, OptoRuntime::slow_arraycopy_Type(), OptoRuntime::slow_arraycopy_Java(), "slow_arraycopy", adr_type, src, src_offset, dest, dest_offset, copy_length); - _sp -= nargs; // Handle exceptions thrown by this fellow: make_slow_call_ex(call, env()->Throwable_klass(), false); @@ -5094,8 +5080,7 @@ Node* dest_elem_klass, Node* src, Node* src_offset, Node* dest, Node* dest_offset, - Node* copy_length, - int nargs) { + Node* copy_length) { if (stopped()) return NULL; address copyfunc_addr = StubRoutines::checkcast_arraycopy(); @@ -5136,8 +5121,7 @@ LibraryCallKit::generate_generic_arraycopy(const TypePtr* adr_type, Node* src, Node* src_offset, Node* dest, Node* dest_offset, - Node* copy_length, - int nargs) { + Node* copy_length) { if (stopped()) return NULL; address copyfunc_addr = StubRoutines::generic_arraycopy();