truffle: src/share/vm/opto/library

comparison src/share/vm/opto/library_call.cpp @ 2324:0ac769a57c64

6627983: G1: Bad oop deference during marking Summary: Bulk zeroing reduction didn't work with G1, because arraycopy would call pre-barriers on uninitialized oops. The solution is to have version of arraycopy stubs that don't have pre-barriers. Also refactored arraycopy stubs generation on SPARC to be more readable and reduced the number of stubs necessary in some cases. Reviewed-by: jrose, kvn, never

author	iveresov
date	Tue, 01 Mar 2011 14:56:48 -0800
parents	41d4973cf100
children	b099aaf51bf8 9dc311b8473e

comparison

equal deleted inserted replaced

-:bc6b27fb3568
+:0ac769a57c64
 Node* generate_limit_guard(Node* offset, Node* subseq_length,
 Node* array_length,
 RegionNode* region);
 Node* generate_current_thread(Node* &tls_output);
 address basictype2arraycopy(BasicType t, Node *src_offset, Node *dest_offset,
-bool disjoint_bases, const char* &name);
+bool disjoint_bases, const char* &name, bool dest_uninitialized);
 Node* load_mirror_from_klass(Node* klass);
 Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null,
 int nargs,
 RegionNode* region, int null_path,
 int offset);
 bool generate_block_arraycopy(const TypePtr* adr_type,
 BasicType basic_elem_type,
 AllocateNode* alloc,
 Node* src,  Node* src_offset,
 Node* dest, Node* dest_offset,
-Node* dest_size);
+Node* dest_size, bool dest_uninitialized);
 void generate_slow_arraycopy(const TypePtr* adr_type,
 Node* src,  Node* src_offset,
 Node* dest, Node* dest_offset,
-Node* copy_length);
+Node* copy_length, bool dest_uninitialized);
 Node* generate_checkcast_arraycopy(const TypePtr* adr_type,
 Node* dest_elem_klass,
 Node* src,  Node* src_offset,
 Node* dest, Node* dest_offset,
-Node* copy_length);
+Node* copy_length, bool dest_uninitialized);
 Node* generate_generic_arraycopy(const TypePtr* adr_type,
 Node* src,  Node* src_offset,
 Node* dest, Node* dest_offset,
-Node* copy_length);
+Node* copy_length, bool dest_uninitialized);
 void generate_unchecked_arraycopy(const TypePtr* adr_type,
 BasicType basic_elem_type,
 bool disjoint_bases,
 Node* src,  Node* src_offset,
 Node* dest, Node* dest_offset,
-Node* copy_length);
+Node* copy_length, bool dest_uninitialized);
 bool inline_unsafe_CAS(BasicType type);
 bool inline_unsafe_ordered_store(BasicType type);
 bool inline_fp_conversions(vmIntrinsics::ID id);
 bool inline_numberOfLeadingZeros(vmIntrinsics::ID id);
 bool inline_numberOfTrailingZeros(vmIntrinsics::ID id);
 countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong) ));
 const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM;
 bool disjoint_bases = true;
 generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases,
-src, NULL, dest, NULL, countx);
+src, NULL, dest, NULL, countx,
+/*dest_uninitialized*/true);
 // If necessary, emit some card marks afterwards.  (Non-arrays only.)
 if (card_mark) {
 assert(!is_array, "");
 // Put in store barrier for any and all oops we are sticking
 };
 // Note:  The condition "disjoint" applies also for overlapping copies
 // where an descending copy is permitted (i.e., dest_offset <= src_offset).
 static address
-select_arraycopy_function(BasicType t, bool aligned, bool disjoint, const char* &name) {
+select_arraycopy_function(BasicType t, bool aligned, bool disjoint, const char* &name, bool dest_uninitialized) {
 int selector =
 (aligned  ? COPYFUNC_ALIGNED  : COPYFUNC_UNALIGNED) +
 (disjoint ? COPYFUNC_DISJOINT : COPYFUNC_CONJOINT);
 #define RETURN_STUB(xxx_arraycopy) { \
 name = #xxx_arraycopy; \
 return StubRoutines::xxx_arraycopy(); }
+#define RETURN_STUB_PARM(xxx_arraycopy, parm) {           \
+name = #xxx_arraycopy; \
+return StubRoutines::xxx_arraycopy(parm); }
 switch (t) {
 case T_BYTE:
 case T_BOOLEAN:
 switch (selector) {
 case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED:    RETURN_STUB(arrayof_jlong_disjoint_arraycopy);
 }
 case T_ARRAY:
 case T_OBJECT:
 switch (selector) {
-case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB(oop_arraycopy);
+case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB_PARM(oop_arraycopy, dest_uninitialized);
-case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED:    RETURN_STUB(arrayof_oop_arraycopy);
+case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED:    RETURN_STUB_PARM(arrayof_oop_arraycopy, dest_uninitialized);
-case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB(oop_disjoint_arraycopy);
+case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED:  RETURN_STUB_PARM(oop_disjoint_arraycopy, dest_uninitialized);
-case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED:    RETURN_STUB(arrayof_oop_disjoint_arraycopy);
+case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED:    RETURN_STUB_PARM(arrayof_oop_disjoint_arraycopy, dest_uninitialized);
 }
 default:
 ShouldNotReachHere();
 return NULL;
 }
 #undef RETURN_STUB
+#undef RETURN_STUB_PARM
 }
 //------------------------------basictype2arraycopy----------------------------
 address LibraryCallKit::basictype2arraycopy(BasicType t,
 Node* src_offset,
 Node* dest_offset,
 bool disjoint_bases,
-const char* &name) {
+const char* &name,
+bool dest_uninitialized) {
 const TypeInt* src_offset_inttype  = gvn().find_int_type(src_offset);;
 const TypeInt* dest_offset_inttype = gvn().find_int_type(dest_offset);;
 bool aligned = false;
 bool disjoint = disjoint_bases;
 } else if (src_offset == dest_offset && src_offset != NULL) {
 // This can occur if the offsets are identical non-constants.
 disjoint = true;
 }
-return select_arraycopy_function(t, aligned, disjoint, name);
+return select_arraycopy_function(t, aligned, disjoint, name, dest_uninitialized);
 }
 //------------------------------inline_arraycopy-----------------------
 bool LibraryCallKit::inline_arraycopy() {
 if (src_elem != dest_elem || dest_elem == T_VOID) {
 // The component types are not the same or are not recognized.  Punt.
 // (But, avoid the native method wrapper to JVM_ArrayCopy.)
 generate_slow_arraycopy(TypePtr::BOTTOM,
-src, src_offset, dest, dest_offset, length);
+src, src_offset, dest, dest_offset, length,
+/*dest_uninitialized*/false);
 return true;
 }
 //---------------------------------------------------------------------------
 // We will make a fast path for this call to arraycopy.
 record_for_igvn(slow_region);
 }
 Node* original_dest      = dest;
 AllocateArrayNode* alloc = NULL;  // used for zeroing, if needed
-bool  must_clear_dest    = false;
+bool  dest_uninitialized = false;
 // See if this is the initialization of a newly-allocated array.
 // If so, we will take responsibility here for initializing it to zero.
 // (Note:  Because tightly_coupled_allocation performs checks on the
 // out-edges of the dest, we need to avoid making derived pointers
 assert(dest->is_CheckCastPP(), "sanity");
 assert(dest->in(0)->in(0) == init, "dest pinned");
 adr_type = TypeRawPtr::BOTTOM;  // all initializations are into raw memory
 // From this point on, every exit path is responsible for
 // initializing any non-copied parts of the object to zero.
-must_clear_dest = true;
+// Also, if this flag is set we make sure that arraycopy interacts properly
+// with G1, eliding pre-barriers. See CR 6627983.
+dest_uninitialized = true;
 } else {
 // No zeroing elimination here.
 alloc             = NULL;
 //original_dest   = dest;
-//must_clear_dest = false;
+//dest_uninitialized = false;
 }
 // Results are placed here:
 enum { fast_path        = 1,  // normal void-returning assembly stub
 checked_path     = 2,  // special assembly stub with cleanup
 Node* checked_mem     = NULL;
 Node* checked_i_o     = NULL;
 Node* checked_value   = NULL;
 if (basic_elem_type == T_CONFLICT) {
-assert(!must_clear_dest, "");
+assert(!dest_uninitialized, "");
 Node* cv = generate_generic_arraycopy(adr_type,
 src, src_offset, dest, dest_offset,
-copy_length);
+copy_length, dest_uninitialized);
 if (cv == NULL)  cv = intcon(-1);  // failure (no stub available)
 checked_control = control();
 checked_i_o     = i_o();
 checked_mem     = memory(adr_type);
 checked_value   = cv;
 if (!length_never_negative) {
 generate_negative_guard(copy_length, slow_region);
 }
 // copy_length is 0.
-if (!stopped() && must_clear_dest) {
+if (!stopped() && dest_uninitialized) {
 Node* dest_length = alloc->in(AllocateNode::ALength);
 if (_gvn.eqv_uncast(copy_length, dest_length)
 || _gvn.find_int_con(dest_length, 1) <= 0) {
 // There is no zeroing to do. No need for a secondary raw memory barrier.
 } else {
 result_region->init_req(zero_path, control());
 result_i_o   ->init_req(zero_path, i_o());
 result_memory->init_req(zero_path, memory(adr_type));
 }
-if (!stopped() && must_clear_dest) {
+if (!stopped() && dest_uninitialized) {
 // We have to initialize the *uncopied* part of the array to zero.
 // The copy destination is the slice dest[off..off+len].  The other slices
 // are dest_head = dest[0..off] and dest_tail = dest[off+len..dest.length].
 Node* dest_size   = alloc->in(AllocateNode::AllocSize);
 Node* dest_length = alloc->in(AllocateNode::ALength);
 // There is no tail.  Try an upgrade to a 64-bit copy.
 bool didit = false;
 { PreserveJVMState pjvms(this);
 didit = generate_block_arraycopy(adr_type, basic_elem_type, alloc,
 src, src_offset, dest, dest_offset,
-dest_size);
+dest_size, dest_uninitialized);
 if (didit) {
 // Present the results of the block-copying fast call.
 result_region->init_req(bcopy_path, control());
 result_i_o   ->init_req(bcopy_path, i_o());
 result_memory->init_req(bcopy_path, memory(adr_type));
 Node* n1 = LoadKlassNode::make(_gvn, immutable_memory(), p1, TypeRawPtr::BOTTOM);
 Node* dest_elem_klass = _gvn.transform(n1);
 Node* cv = generate_checkcast_arraycopy(adr_type,
 dest_elem_klass,
 src, src_offset, dest, dest_offset,
-ConvI2X(copy_length));
+ConvI2X(copy_length), dest_uninitialized);
 if (cv == NULL)  cv = intcon(-1);  // failure (no stub available)
 checked_control = control();
 checked_i_o     = i_o();
 checked_mem     = memory(adr_type);
 checked_value   = cv;
 if (!stopped()) {
 // Generate the fast path, if possible.
 PreserveJVMState pjvms(this);
 generate_unchecked_arraycopy(adr_type, copy_type, disjoint_bases,
 src, src_offset, dest, dest_offset,
-ConvI2X(copy_length));
+ConvI2X(copy_length), dest_uninitialized);
 // Present the results of the fast call.
 result_region->init_req(fast_path, control());
 result_i_o   ->init_req(fast_path, i_o());
 result_memory->init_req(fast_path, memory(adr_type));
 PreserveJVMState pjvms(this);   // replace_in_map may trash the map
 set_memory(slow_mem, adr_type);
 set_i_o(slow_i_o);
-if (must_clear_dest) {
+if (dest_uninitialized) {
 generate_clear_array(adr_type, dest, basic_elem_type,
 intcon(0), NULL,
 alloc->in(AllocateNode::AllocSize));
 }
 generate_slow_arraycopy(adr_type,
 src, src_offset, dest, dest_offset,
-copy_length);
+copy_length, /*dest_uninitialized*/false);
 result_region->init_req(slow_call_path, control());
 result_i_o   ->init_req(slow_call_path, i_o());
 result_memory->init_req(slow_call_path, memory(adr_type));
 }
 LibraryCallKit::generate_block_arraycopy(const TypePtr* adr_type,
 BasicType basic_elem_type,
 AllocateNode* alloc,
 Node* src,  Node* src_offset,
 Node* dest, Node* dest_offset,
-Node* dest_size) {
+Node* dest_size, bool dest_uninitialized) {
 // See if there is an advantage from block transfer.
 int scale = exact_log2(type2aelembytes(basic_elem_type));
 if (scale >= LogBytesPerLong)
 return false;               // it is already a block transfer
 countx = _gvn.transform( new (C, 3) SubXNode(countx, MakeConX(dest_off)) );
 countx = _gvn.transform( new (C, 3) URShiftXNode(countx, intcon(LogBytesPerLong)) );
 bool disjoint_bases = true;   // since alloc != NULL
 generate_unchecked_arraycopy(adr_type, T_LONG, disjoint_bases,
-sptr, NULL, dptr, NULL, countx);
+sptr, NULL, dptr, NULL, countx, dest_uninitialized);
 return true;
 }
 // but without the native wrapper overhead.
 void
 LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type,
 Node* src,  Node* src_offset,
 Node* dest, Node* dest_offset,
-Node* copy_length) {
+Node* copy_length, bool dest_uninitialized) {
+assert(!dest_uninitialized, "Invariant");
 Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON,
 OptoRuntime::slow_arraycopy_Type(),
 OptoRuntime::slow_arraycopy_Java(),
 "slow_arraycopy", adr_type,
 src, src_offset, dest, dest_offset,
 Node*
 LibraryCallKit::generate_checkcast_arraycopy(const TypePtr* adr_type,
 Node* dest_elem_klass,
 Node* src,  Node* src_offset,
 Node* dest, Node* dest_offset,
-Node* copy_length) {
+Node* copy_length, bool dest_uninitialized) {
 if (stopped())  return NULL;
-address copyfunc_addr = StubRoutines::checkcast_arraycopy();
+address copyfunc_addr = StubRoutines::checkcast_arraycopy(dest_uninitialized);
 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
 return NULL;
 }
 // Pick out the parameters required to perform a store-check
 // Helper function; generates code for cases requiring runtime checks.
 Node*
 LibraryCallKit::generate_generic_arraycopy(const TypePtr* adr_type,
 Node* src,  Node* src_offset,
 Node* dest, Node* dest_offset,
-Node* copy_length) {
+Node* copy_length, bool dest_uninitialized) {
+assert(!dest_uninitialized, "Invariant");
 if (stopped())  return NULL;
 address copyfunc_addr = StubRoutines::generic_arraycopy();
 if (copyfunc_addr == NULL) { // Stub was not generated, go slow path.
 return NULL;
 }
 LibraryCallKit::generate_unchecked_arraycopy(const TypePtr* adr_type,
 BasicType basic_elem_type,
 bool disjoint_bases,
 Node* src,  Node* src_offset,
 Node* dest, Node* dest_offset,
-Node* copy_length) {
+Node* copy_length, bool dest_uninitialized) {
 if (stopped())  return;               // nothing to do
 Node* src_start  = src;
 Node* dest_start = dest;
 if (src_offset != NULL || dest_offset != NULL) {
 // Figure out which arraycopy runtime method to call.
 const char* copyfunc_name = "arraycopy";
 address     copyfunc_addr =
 basictype2arraycopy(basic_elem_type, src_offset, dest_offset,
-disjoint_bases, copyfunc_name);
+disjoint_bases, copyfunc_name, dest_uninitialized);
 // Call it.  Note that the count_ix value is not scaled to a byte-size.
 make_runtime_call(RC_LEAF|RC_NO_FP,
 OptoRuntime::fast_arraycopy_Type(),
 copyfunc_addr, copyfunc_name, adr_type,

Mercurial > hg > truffle

comparison src/share/vm/opto/library_call.cpp @ 2324:0ac769a57c64