Mercurial > hg > truffle
changeset 2324:0ac769a57c64
6627983: G1: Bad oop deference during marking
Summary: Bulk zeroing reduction didn't work with G1, because arraycopy would call pre-barriers on uninitialized oops. The solution is to have version of arraycopy stubs that don't have pre-barriers. Also refactored arraycopy stubs generation on SPARC to be more readable and reduced the number of stubs necessary in some cases.
Reviewed-by: jrose, kvn, never
| author | iveresov |
|---|---|
| date | Tue, 01 Mar 2011 14:56:48 -0800 |
| parents | bc6b27fb3568 |
| children | 8c9c9ee30d71 |
| files | src/cpu/sparc/vm/stubGenerator_sparc.cpp src/cpu/x86/vm/stubGenerator_x86_32.cpp src/cpu/x86/vm/stubGenerator_x86_64.cpp src/share/vm/gc_implementation/g1/g1SATBCardTableModRefBS.hpp src/share/vm/memory/barrierSet.cpp src/share/vm/memory/barrierSet.hpp src/share/vm/opto/library_call.cpp src/share/vm/runtime/arguments.cpp src/share/vm/runtime/stubRoutines.cpp src/share/vm/runtime/stubRoutines.hpp |
| diffstat | 10 files changed, 356 insertions(+), 208 deletions(-) [+] |
line wrap: on
line diff
--- a/src/cpu/sparc/vm/stubGenerator_sparc.cpp Tue Mar 01 10:27:15 2011 -0800 +++ b/src/cpu/sparc/vm/stubGenerator_sparc.cpp Tue Mar 01 14:56:48 2011 -0800 @@ -1033,31 +1033,40 @@ // // The input registers are overwritten. // - void gen_write_ref_array_pre_barrier(Register addr, Register count) { + void gen_write_ref_array_pre_barrier(Register addr, Register count, bool dest_uninitialized) { BarrierSet* bs = Universe::heap()->barrier_set(); - if (bs->has_write_ref_pre_barrier()) { - assert(bs->has_write_ref_array_pre_opt(), - "Else unsupported barrier set."); - - __ save_frame(0); - // Save the necessary global regs... will be used after. - if (addr->is_global()) { - __ mov(addr, L0); - } - if (count->is_global()) { - __ mov(count, L1); - } - __ mov(addr->after_save(), O0); - // Get the count into O1 - __ call(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre)); - __ delayed()->mov(count->after_save(), O1); - if (addr->is_global()) { - __ mov(L0, addr); - } - if (count->is_global()) { - __ mov(L1, count); - } - __ restore(); + switch (bs->kind()) { + case BarrierSet::G1SATBCT: + case BarrierSet::G1SATBCTLogging: + // With G1, don't generate the call if we statically know that the target in uninitialized + if (!dest_uninitialized) { + __ save_frame(0); + // Save the necessary global regs... will be used after. + if (addr->is_global()) { + __ mov(addr, L0); + } + if (count->is_global()) { + __ mov(count, L1); + } + __ mov(addr->after_save(), O0); + // Get the count into O1 + __ call(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre)); + __ delayed()->mov(count->after_save(), O1); + if (addr->is_global()) { + __ mov(L0, addr); + } + if (count->is_global()) { + __ mov(L1, count); + } + __ restore(); + } + break; + case BarrierSet::CardTableModRef: + case BarrierSet::CardTableExtension: + case BarrierSet::ModRef: + break; + default: + ShouldNotReachHere(); } } // @@ -1071,7 +1080,7 @@ // The input registers are overwritten. // void gen_write_ref_array_post_barrier(Register addr, Register count, - Register tmp) { + Register tmp) { BarrierSet* bs = Universe::heap()->barrier_set(); switch (bs->kind()) { @@ -2406,7 +2415,7 @@ StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); - assert(!aligned, "usage"); + assert(aligned, "Should always be aligned"); assert_clean_int(O2, O3); // Make sure 'count' is clean int. @@ -2435,7 +2444,8 @@ // to: O1 // count: O2 treated as signed // - address generate_disjoint_oop_copy(bool aligned, address *entry, const char *name) { + address generate_disjoint_oop_copy(bool aligned, address *entry, const char *name, + bool dest_uninitialized = false) { const Register from = O0; // source array address const Register to = O1; // destination array address @@ -2456,7 +2466,7 @@ // save arguments for barrier generation __ mov(to, G1); __ mov(count, G5); - gen_write_ref_array_pre_barrier(G1, G5); + gen_write_ref_array_pre_barrier(G1, G5, dest_uninitialized); #ifdef _LP64 assert_clean_int(count, O3); // Make sure 'count' is clean int. if (UseCompressedOops) { @@ -2486,7 +2496,8 @@ // count: O2 treated as signed // address generate_conjoint_oop_copy(bool aligned, address nooverlap_target, - address *entry, const char *name) { + address *entry, const char *name, + bool dest_uninitialized = false) { const Register from = O0; // source array address const Register to = O1; // destination array address @@ -2509,7 +2520,7 @@ // save arguments for barrier generation __ mov(to, G1); __ mov(count, G5); - gen_write_ref_array_pre_barrier(G1, G5); + gen_write_ref_array_pre_barrier(G1, G5, dest_uninitialized); #ifdef _LP64 if (UseCompressedOops) { @@ -2578,7 +2589,7 @@ // ckval: O4 (super_klass) // ret: O0 zero for success; (-1^K) where K is partial transfer count // - address generate_checkcast_copy(const char *name, address *entry) { + address generate_checkcast_copy(const char *name, address *entry, bool dest_uninitialized = false) { const Register O0_from = O0; // source array address const Register O1_to = O1; // destination array address @@ -2624,8 +2635,7 @@ // caller can pass a 64-bit byte count here (from generic stub) BLOCK_COMMENT("Entry:"); } - - gen_write_ref_array_pre_barrier(O1_to, O2_count); + gen_write_ref_array_pre_barrier(O1_to, O2_count, dest_uninitialized); Label load_element, store_element, do_card_marks, fail, done; __ addcc(O2_count, 0, G1_remain); // initialize loop index, and test it @@ -3083,56 +3093,104 @@ address entry_jlong_arraycopy; address entry_checkcast_arraycopy; - StubRoutines::_jbyte_disjoint_arraycopy = generate_disjoint_byte_copy(false, &entry, - "jbyte_disjoint_arraycopy"); - StubRoutines::_jbyte_arraycopy = generate_conjoint_byte_copy(false, entry, &entry_jbyte_arraycopy, - "jbyte_arraycopy"); - StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_short_copy(false, &entry, - "jshort_disjoint_arraycopy"); - StubRoutines::_jshort_arraycopy = generate_conjoint_short_copy(false, entry, &entry_jshort_arraycopy, - "jshort_arraycopy"); - StubRoutines::_jint_disjoint_arraycopy = generate_disjoint_int_copy(false, &entry, - "jint_disjoint_arraycopy"); - StubRoutines::_jint_arraycopy = generate_conjoint_int_copy(false, entry, &entry_jint_arraycopy, - "jint_arraycopy"); - StubRoutines::_jlong_disjoint_arraycopy = generate_disjoint_long_copy(false, &entry, - "jlong_disjoint_arraycopy"); - StubRoutines::_jlong_arraycopy = generate_conjoint_long_copy(false, entry, &entry_jlong_arraycopy, - "jlong_arraycopy"); - StubRoutines::_oop_disjoint_arraycopy = generate_disjoint_oop_copy(false, &entry, - "oop_disjoint_arraycopy"); - StubRoutines::_oop_arraycopy = generate_conjoint_oop_copy(false, entry, &entry_oop_arraycopy, - "oop_arraycopy"); - - - StubRoutines::_arrayof_jbyte_disjoint_arraycopy = generate_disjoint_byte_copy(true, &entry, - "arrayof_jbyte_disjoint_arraycopy"); - StubRoutines::_arrayof_jbyte_arraycopy = generate_conjoint_byte_copy(true, entry, NULL, - "arrayof_jbyte_arraycopy"); - + //*** jbyte + // Always need aligned and unaligned versions + StubRoutines::_jbyte_disjoint_arraycopy = generate_disjoint_byte_copy(false, &entry, + "jbyte_disjoint_arraycopy"); + StubRoutines::_jbyte_arraycopy = generate_conjoint_byte_copy(false, entry, + &entry_jbyte_arraycopy, + "jbyte_arraycopy"); + StubRoutines::_arrayof_jbyte_disjoint_arraycopy = generate_disjoint_byte_copy(true, &entry, + "arrayof_jbyte_disjoint_arraycopy"); + StubRoutines::_arrayof_jbyte_arraycopy = generate_conjoint_byte_copy(true, entry, NULL, + "arrayof_jbyte_arraycopy"); + + //*** jshort + // Always need aligned and unaligned versions + StubRoutines::_jshort_disjoint_arraycopy = generate_disjoint_short_copy(false, &entry, + "jshort_disjoint_arraycopy"); + StubRoutines::_jshort_arraycopy = generate_conjoint_short_copy(false, entry, + &entry_jshort_arraycopy, + "jshort_arraycopy"); StubRoutines::_arrayof_jshort_disjoint_arraycopy = generate_disjoint_short_copy(true, &entry, "arrayof_jshort_disjoint_arraycopy"); StubRoutines::_arrayof_jshort_arraycopy = generate_conjoint_short_copy(true, entry, NULL, "arrayof_jshort_arraycopy"); - StubRoutines::_arrayof_jint_disjoint_arraycopy = generate_disjoint_int_copy(true, &entry, - "arrayof_jint_disjoint_arraycopy"); + //*** jint + // Aligned versions + StubRoutines::_arrayof_jint_disjoint_arraycopy = generate_disjoint_int_copy(true, &entry, + "arrayof_jint_disjoint_arraycopy"); + StubRoutines::_arrayof_jint_arraycopy = generate_conjoint_int_copy(true, entry, &entry_jint_arraycopy, + "arrayof_jint_arraycopy"); #ifdef _LP64 - // since sizeof(jint) < sizeof(HeapWord), there's a different flavor: - StubRoutines::_arrayof_jint_arraycopy = generate_conjoint_int_copy(true, entry, NULL, "arrayof_jint_arraycopy"); - #else - StubRoutines::_arrayof_jint_arraycopy = StubRoutines::_jint_arraycopy; + // In 64 bit we need both aligned and unaligned versions of jint arraycopy. + // entry_jint_arraycopy always points to the unaligned version (notice that we overwrite it). + StubRoutines::_jint_disjoint_arraycopy = generate_disjoint_int_copy(false, &entry, + "jint_disjoint_arraycopy"); + StubRoutines::_jint_arraycopy = generate_conjoint_int_copy(false, entry, + &entry_jint_arraycopy, + "jint_arraycopy"); +#else + // In 32 bit jints are always HeapWordSize aligned, so always use the aligned version + // (in fact in 32bit we always have a pre-loop part even in the aligned version, + // because it uses 64-bit loads/stores, so the aligned flag is actually ignored). + StubRoutines::_jint_disjoint_arraycopy = StubRoutines::_arrayof_jint_disjoint_arraycopy; + StubRoutines::_jint_arraycopy = StubRoutines::_arrayof_jint_arraycopy; #endif - StubRoutines::_arrayof_jlong_disjoint_arraycopy = generate_disjoint_long_copy(true, NULL, - "arrayof_jlong_disjoint_arraycopy"); - StubRoutines::_arrayof_oop_disjoint_arraycopy = generate_disjoint_oop_copy(true, NULL, - "arrayof_oop_disjoint_arraycopy"); - - StubRoutines::_arrayof_jlong_arraycopy = StubRoutines::_jlong_arraycopy; - StubRoutines::_arrayof_oop_arraycopy = StubRoutines::_oop_arraycopy; - - StubRoutines::_checkcast_arraycopy = generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy); + + //*** jlong + // It is always aligned + StubRoutines::_arrayof_jlong_disjoint_arraycopy = generate_disjoint_long_copy(true, &entry, + "arrayof_jlong_disjoint_arraycopy"); + StubRoutines::_arrayof_jlong_arraycopy = generate_conjoint_long_copy(true, entry, &entry_jlong_arraycopy, + "arrayof_jlong_arraycopy"); + StubRoutines::_jlong_disjoint_arraycopy = StubRoutines::_arrayof_jlong_disjoint_arraycopy; + StubRoutines::_jlong_arraycopy = StubRoutines::_arrayof_jlong_arraycopy; + + + //*** oops + // Aligned versions + StubRoutines::_arrayof_oop_disjoint_arraycopy = generate_disjoint_oop_copy(true, &entry, + "arrayof_oop_disjoint_arraycopy"); + StubRoutines::_arrayof_oop_arraycopy = generate_conjoint_oop_copy(true, entry, &entry_oop_arraycopy, + "arrayof_oop_arraycopy"); + // Aligned versions without pre-barriers + StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit = generate_disjoint_oop_copy(true, &entry, + "arrayof_oop_disjoint_arraycopy_uninit", + /*dest_uninitialized*/true); + StubRoutines::_arrayof_oop_arraycopy_uninit = generate_conjoint_oop_copy(true, entry, NULL, + "arrayof_oop_arraycopy_uninit", + /*dest_uninitialized*/true); +#ifdef _LP64 + if (UseCompressedOops) { + // With compressed oops we need unaligned versions, notice that we overwrite entry_oop_arraycopy. + StubRoutines::_oop_disjoint_arraycopy = generate_disjoint_oop_copy(false, &entry, + "oop_disjoint_arraycopy"); + StubRoutines::_oop_arraycopy = generate_conjoint_oop_copy(false, entry, &entry_oop_arraycopy, + "oop_arraycopy"); + // Unaligned versions without pre-barriers + StubRoutines::_oop_disjoint_arraycopy_uninit = generate_disjoint_oop_copy(false, &entry, + "oop_disjoint_arraycopy_uninit", + /*dest_uninitialized*/true); + StubRoutines::_oop_arraycopy_uninit = generate_conjoint_oop_copy(false, entry, NULL, + "oop_arraycopy_uninit", + /*dest_uninitialized*/true); + } else +#endif + { + // oop arraycopy is always aligned on 32bit and 64bit without compressed oops + StubRoutines::_oop_disjoint_arraycopy = StubRoutines::_arrayof_oop_disjoint_arraycopy; + StubRoutines::_oop_arraycopy = StubRoutines::_arrayof_oop_arraycopy; + StubRoutines::_oop_disjoint_arraycopy_uninit = StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit; + StubRoutines::_oop_arraycopy_uninit = StubRoutines::_arrayof_oop_arraycopy_uninit; + } + + StubRoutines::_checkcast_arraycopy = generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy); + StubRoutines::_checkcast_arraycopy_uninit = generate_checkcast_copy("checkcast_arraycopy_uninit", NULL, + /*dest_uninitialized*/true); + StubRoutines::_unsafe_arraycopy = generate_unsafe_copy("unsafe_arraycopy", entry_jbyte_arraycopy, entry_jshort_arraycopy,
--- a/src/cpu/x86/vm/stubGenerator_x86_32.cpp Tue Mar 01 10:27:15 2011 -0800 +++ b/src/cpu/x86/vm/stubGenerator_x86_32.cpp Tue Mar 01 14:56:48 2011 -0800 @@ -729,18 +729,19 @@ // Input: // start - starting address // count - element count - void gen_write_ref_array_pre_barrier(Register start, Register count) { + void gen_write_ref_array_pre_barrier(Register start, Register count, bool uninitialized_target) { assert_different_registers(start, count); BarrierSet* bs = Universe::heap()->barrier_set(); switch (bs->kind()) { case BarrierSet::G1SATBCT: case BarrierSet::G1SATBCTLogging: - { - __ pusha(); // push registers - __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre), - start, count); - __ popa(); - } + // With G1, don't generate the call if we statically know that the target in uninitialized + if (!uninitialized_target) { + __ pusha(); // push registers + __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre), + start, count); + __ popa(); + } break; case BarrierSet::CardTableModRef: case BarrierSet::CardTableExtension: @@ -919,7 +920,8 @@ address generate_disjoint_copy(BasicType t, bool aligned, Address::ScaleFactor sf, - address* entry, const char *name) { + address* entry, const char *name, + bool dest_uninitialized = false) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); @@ -950,7 +952,7 @@ if (t == T_OBJECT) { __ testl(count, count); __ jcc(Assembler::zero, L_0_count); - gen_write_ref_array_pre_barrier(to, count); + gen_write_ref_array_pre_barrier(to, count, dest_uninitialized); __ mov(saved_to, to); // save 'to' } @@ -1084,7 +1086,8 @@ address generate_conjoint_copy(BasicType t, bool aligned, Address::ScaleFactor sf, address nooverlap_target, - address* entry, const char *name) { + address* entry, const char *name, + bool dest_uninitialized = false) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); @@ -1128,7 +1131,7 @@ if (t == T_OBJECT) { __ testl(count, count); __ jcc(Assembler::zero, L_0_count); - gen_write_ref_array_pre_barrier(dst, count); + gen_write_ref_array_pre_barrier(dst, count, dest_uninitialized); } // copy from high to low @@ -1415,7 +1418,7 @@ // rax, == 0 - success // rax, == -1^K - failure, where K is partial transfer count // - address generate_checkcast_copy(const char *name, address* entry) { + address generate_checkcast_copy(const char *name, address* entry, bool dest_uninitialized = false) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); @@ -1476,7 +1479,7 @@ Address elem_klass_addr(elem, oopDesc::klass_offset_in_bytes()); // Copy from low to high addresses, indexed from the end of each array. - gen_write_ref_array_pre_barrier(to, count); + gen_write_ref_array_pre_barrier(to, count, dest_uninitialized); __ lea(end_from, end_from_addr); __ lea(end_to, end_to_addr); assert(length == count, ""); // else fix next line: @@ -2039,6 +2042,15 @@ generate_conjoint_copy(T_OBJECT, true, Address::times_ptr, entry, &entry_oop_arraycopy, "oop_arraycopy"); + StubRoutines::_oop_disjoint_arraycopy_uninit = + generate_disjoint_copy(T_OBJECT, true, Address::times_ptr, &entry, + "oop_disjoint_arraycopy_uninit", + /*dest_uninitialized*/true); + StubRoutines::_oop_arraycopy_uninit = + generate_conjoint_copy(T_OBJECT, true, Address::times_ptr, entry, + NULL, "oop_arraycopy_uninit", + /*dest_uninitialized*/true); + StubRoutines::_jlong_disjoint_arraycopy = generate_disjoint_long_copy(&entry, "jlong_disjoint_arraycopy"); StubRoutines::_jlong_arraycopy = @@ -2052,20 +2064,20 @@ StubRoutines::_arrayof_jshort_fill = generate_fill(T_SHORT, true, "arrayof_jshort_fill"); StubRoutines::_arrayof_jint_fill = generate_fill(T_INT, true, "arrayof_jint_fill"); - StubRoutines::_arrayof_jint_disjoint_arraycopy = - StubRoutines::_jint_disjoint_arraycopy; - StubRoutines::_arrayof_oop_disjoint_arraycopy = - StubRoutines::_oop_disjoint_arraycopy; - StubRoutines::_arrayof_jlong_disjoint_arraycopy = - StubRoutines::_jlong_disjoint_arraycopy; + StubRoutines::_arrayof_jint_disjoint_arraycopy = StubRoutines::_jint_disjoint_arraycopy; + StubRoutines::_arrayof_oop_disjoint_arraycopy = StubRoutines::_oop_disjoint_arraycopy; + StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit = StubRoutines::_oop_disjoint_arraycopy_uninit; + StubRoutines::_arrayof_jlong_disjoint_arraycopy = StubRoutines::_jlong_disjoint_arraycopy; - StubRoutines::_arrayof_jint_arraycopy = StubRoutines::_jint_arraycopy; - StubRoutines::_arrayof_oop_arraycopy = StubRoutines::_oop_arraycopy; - StubRoutines::_arrayof_jlong_arraycopy = StubRoutines::_jlong_arraycopy; + StubRoutines::_arrayof_jint_arraycopy = StubRoutines::_jint_arraycopy; + StubRoutines::_arrayof_oop_arraycopy = StubRoutines::_oop_arraycopy; + StubRoutines::_arrayof_oop_arraycopy_uninit = StubRoutines::_oop_arraycopy_uninit; + StubRoutines::_arrayof_jlong_arraycopy = StubRoutines::_jlong_arraycopy; StubRoutines::_checkcast_arraycopy = - generate_checkcast_copy("checkcast_arraycopy", - &entry_checkcast_arraycopy); + generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy); + StubRoutines::_checkcast_arraycopy_uninit = + generate_checkcast_copy("checkcast_arraycopy_uninit", NULL, /*dest_uninitialized*/true); StubRoutines::_unsafe_arraycopy = generate_unsafe_copy("unsafe_arraycopy",
--- a/src/cpu/x86/vm/stubGenerator_x86_64.cpp Tue Mar 01 10:27:15 2011 -0800 +++ b/src/cpu/x86/vm/stubGenerator_x86_64.cpp Tue Mar 01 14:56:48 2011 -0800 @@ -1159,34 +1159,35 @@ // Generate code for an array write pre barrier // // addr - starting address - // count - element count + // count - element count + // tmp - scratch register // // Destroy no registers! // - void gen_write_ref_array_pre_barrier(Register addr, Register count) { + void gen_write_ref_array_pre_barrier(Register addr, Register count, bool dest_uninitialized) { BarrierSet* bs = Universe::heap()->barrier_set(); switch (bs->kind()) { case BarrierSet::G1SATBCT: case BarrierSet::G1SATBCTLogging: - { - __ pusha(); // push registers - if (count == c_rarg0) { - if (addr == c_rarg1) { - // exactly backwards!! - __ xchgptr(c_rarg1, c_rarg0); - } else { - __ movptr(c_rarg1, count); - __ movptr(c_rarg0, addr); - } - - } else { - __ movptr(c_rarg0, addr); - __ movptr(c_rarg1, count); - } - __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre), 2); - __ popa(); + // With G1, don't generate the call if we statically know that the target in uninitialized + if (!dest_uninitialized) { + __ pusha(); // push registers + if (count == c_rarg0) { + if (addr == c_rarg1) { + // exactly backwards!! + __ xchgptr(c_rarg1, c_rarg0); + } else { + __ movptr(c_rarg1, count); + __ movptr(c_rarg0, addr); + } + } else { + __ movptr(c_rarg0, addr); + __ movptr(c_rarg1, count); + } + __ call_VM_leaf(CAST_FROM_FN_PTR(address, BarrierSet::static_write_ref_array_pre), 2); + __ popa(); } - break; + break; case BarrierSet::CardTableModRef: case BarrierSet::CardTableExtension: case BarrierSet::ModRef: @@ -1769,7 +1770,8 @@ // disjoint_int_copy_entry is set to the no-overlap entry point // used by generate_conjoint_int_oop_copy(). // - address generate_disjoint_int_oop_copy(bool aligned, bool is_oop, address* entry, const char *name) { + address generate_disjoint_int_oop_copy(bool aligned, bool is_oop, address* entry, + const char *name, bool dest_uninitialized = false) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); @@ -1799,7 +1801,7 @@ // r9 and r10 may be used to save non-volatile registers if (is_oop) { __ movq(saved_to, to); - gen_write_ref_array_pre_barrier(to, count); + gen_write_ref_array_pre_barrier(to, count, dest_uninitialized); } // 'from', 'to' and 'count' are now valid @@ -1860,7 +1862,8 @@ // cache line boundaries will still be loaded and stored atomicly. // address generate_conjoint_int_oop_copy(bool aligned, bool is_oop, address nooverlap_target, - address *entry, const char *name) { + address *entry, const char *name, + bool dest_uninitialized = false) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); @@ -1887,7 +1890,7 @@ if (is_oop) { // no registers are destroyed by this call - gen_write_ref_array_pre_barrier(to, count); + gen_write_ref_array_pre_barrier(to, count, dest_uninitialized); } assert_clean_int(count, rax); // Make sure 'count' is clean int. @@ -1953,7 +1956,8 @@ // disjoint_oop_copy_entry or disjoint_long_copy_entry is set to the // no-overlap entry point used by generate_conjoint_long_oop_copy(). // - address generate_disjoint_long_oop_copy(bool aligned, bool is_oop, address *entry, const char *name) { + address generate_disjoint_long_oop_copy(bool aligned, bool is_oop, address *entry, + const char *name, bool dest_uninitialized = false) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); @@ -1983,7 +1987,7 @@ // 'from', 'to' and 'qword_count' are now valid if (is_oop) { // no registers are destroyed by this call - gen_write_ref_array_pre_barrier(to, qword_count); + gen_write_ref_array_pre_barrier(to, qword_count, dest_uninitialized); } // Copy from low to high addresses. Use 'to' as scratch. @@ -2038,8 +2042,9 @@ // c_rarg1 - destination array address // c_rarg2 - element count, treated as ssize_t, can be zero // - address generate_conjoint_long_oop_copy(bool aligned, bool is_oop, address nooverlap_target, - address *entry, const char *name) { + address generate_conjoint_long_oop_copy(bool aligned, bool is_oop, + address nooverlap_target, address *entry, + const char *name, bool dest_uninitialized = false) { __ align(CodeEntryAlignment); StubCodeMark mark(this, "StubRoutines", name); address start = __ pc(); @@ -2067,7 +2072,7 @@ // Save to and count for store barrier __ movptr(saved_count, qword_count); // No registers are destroyed by this call - gen_write_ref_array_pre_barrier(to, saved_count); + gen_write_ref_array_pre_barrier(to, saved_count, dest_uninitialized); } __ jmp(L_copy_32_bytes); @@ -2146,7 +2151,8 @@ // rax == 0 - success // rax == -1^K - failure, where K is partial transfer count // - address generate_checkcast_copy(const char *name, address *entry) { + address generate_checkcast_copy(const char *name, address *entry, + bool dest_uninitialized = false) { Label L_load_element, L_store_element, L_do_card_marks, L_done; @@ -2240,7 +2246,7 @@ Address from_element_addr(end_from, count, TIMES_OOP, 0); Address to_element_addr(end_to, count, TIMES_OOP, 0); - gen_write_ref_array_pre_barrier(to, count); + gen_write_ref_array_pre_barrier(to, count, dest_uninitialized); // Copy from low to high addresses, indexed from the end of each array. __ lea(end_from, end_from_addr); @@ -2750,14 +2756,29 @@ "oop_disjoint_arraycopy"); StubRoutines::_oop_arraycopy = generate_conjoint_int_oop_copy(false, true, entry, &entry_oop_arraycopy, "oop_arraycopy"); + StubRoutines::_oop_disjoint_arraycopy_uninit = generate_disjoint_int_oop_copy(false, true, &entry, + "oop_disjoint_arraycopy_uninit", + /*dest_uninitialized*/true); + StubRoutines::_oop_arraycopy_uninit = generate_conjoint_int_oop_copy(false, true, entry, + NULL, "oop_arraycopy_uninit", + /*dest_uninitialized*/true); } else { StubRoutines::_oop_disjoint_arraycopy = generate_disjoint_long_oop_copy(false, true, &entry, "oop_disjoint_arraycopy"); StubRoutines::_oop_arraycopy = generate_conjoint_long_oop_copy(false, true, entry, &entry_oop_arraycopy, "oop_arraycopy"); + StubRoutines::_oop_disjoint_arraycopy_uninit = generate_disjoint_long_oop_copy(false, true, &entry, + "oop_disjoint_arraycopy_uninit", + /*dest_uninitialized*/true); + StubRoutines::_oop_arraycopy_uninit = generate_conjoint_long_oop_copy(false, true, entry, + NULL, "oop_arraycopy_uninit", + /*dest_uninitialized*/true); } - StubRoutines::_checkcast_arraycopy = generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy); + StubRoutines::_checkcast_arraycopy = generate_checkcast_copy("checkcast_arraycopy", &entry_checkcast_arraycopy); + StubRoutines::_checkcast_arraycopy_uninit = generate_checkcast_copy("checkcast_arraycopy_uninit", NULL, + /*dest_uninitialized*/true); + StubRoutines::_unsafe_arraycopy = generate_unsafe_copy("unsafe_arraycopy", entry_jbyte_arraycopy, entry_jshort_arraycopy, @@ -2794,6 +2815,9 @@ StubRoutines::_arrayof_oop_disjoint_arraycopy = StubRoutines::_oop_disjoint_arraycopy; StubRoutines::_arrayof_oop_arraycopy = StubRoutines::_oop_arraycopy; + + StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit = StubRoutines::_oop_disjoint_arraycopy_uninit; + StubRoutines::_arrayof_oop_arraycopy_uninit = StubRoutines::_oop_arraycopy_uninit; } void generate_math_stubs() {
--- a/src/share/vm/gc_implementation/g1/g1SATBCardTableModRefBS.hpp Tue Mar 01 10:27:15 2011 -0800 +++ b/src/share/vm/gc_implementation/g1/g1SATBCardTableModRefBS.hpp Tue Mar 01 14:56:48 2011 -0800 @@ -83,11 +83,15 @@ } template <class T> void write_ref_array_pre_work(T* dst, int count); - virtual void write_ref_array_pre(oop* dst, int count) { - write_ref_array_pre_work(dst, count); + virtual void write_ref_array_pre(oop* dst, int count, bool dest_uninitialized) { + if (!dest_uninitialized) { + write_ref_array_pre_work(dst, count); + } } - virtual void write_ref_array_pre(narrowOop* dst, int count) { - write_ref_array_pre_work(dst, count); + virtual void write_ref_array_pre(narrowOop* dst, int count, bool dest_uninitialized) { + if (!dest_uninitialized) { + write_ref_array_pre_work(dst, count); + } } };
--- a/src/share/vm/memory/barrierSet.cpp Tue Mar 01 10:27:15 2011 -0800 +++ b/src/share/vm/memory/barrierSet.cpp Tue Mar 01 14:56:48 2011 -0800 @@ -35,9 +35,9 @@ start, count); #endif if (UseCompressedOops) { - Universe::heap()->barrier_set()->write_ref_array_pre((narrowOop*)start, (int)count); + Universe::heap()->barrier_set()->write_ref_array_pre((narrowOop*)start, (int)count, false); } else { - Universe::heap()->barrier_set()->write_ref_array_pre( (oop*)start, (int)count); + Universe::heap()->barrier_set()->write_ref_array_pre( (oop*)start, (int)count, false); } }
--- a/src/share/vm/memory/barrierSet.hpp Tue Mar 01 10:27:15 2011 -0800 +++ b/src/share/vm/memory/barrierSet.hpp Tue Mar 01 14:56:48 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2000, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2000, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -44,6 +44,10 @@ Uninit }; + enum Flags { + None = 0, + TargetUninitialized = 1 + }; protected: int _max_covered_regions; Name _kind; @@ -128,8 +132,10 @@ virtual void read_prim_array(MemRegion mr) = 0; // Below length is the # array elements being written - virtual void write_ref_array_pre( oop* dst, int length) {} - virtual void write_ref_array_pre(narrowOop* dst, int length) {} + virtual void write_ref_array_pre(oop* dst, int length, + bool dest_uninitialized = false) {} + virtual void write_ref_array_pre(narrowOop* dst, int length, + bool dest_uninitialized = false) {} // Below count is the # array elements being written, starting // at the address "start", which may not necessarily be HeapWord-aligned inline void write_ref_array(HeapWord* start, size_t count);
--- a/src/share/vm/opto/library_call.cpp Tue Mar 01 10:27:15 2011 -0800 +++ b/src/share/vm/opto/library_call.cpp Tue Mar 01 14:56:48 2011 -0800 @@ -97,7 +97,7 @@ RegionNode* region); Node* generate_current_thread(Node* &tls_output); address basictype2arraycopy(BasicType t, Node *src_offset, Node *dest_offset, - bool disjoint_bases, const char* &name); + bool disjoint_bases, const char* &name, bool dest_uninitialized); Node* load_mirror_from_klass(Node* klass); Node* load_klass_from_mirror_common(Node* mirror, bool never_see_null, int nargs, @@ -212,26 +212,26 @@ AllocateNode* alloc, Node* src, Node* src_offset, Node* dest, Node* dest_offset, - Node* dest_size); + Node* dest_size, bool dest_uninitialized); void generate_slow_arraycopy(const TypePtr* adr_type, Node* src, Node* src_offset, Node* dest, Node* dest_offset, - Node* copy_length); + Node* copy_length, bool dest_uninitialized); Node* generate_checkcast_arraycopy(const TypePtr* adr_type, Node* dest_elem_klass, Node* src, Node* src_offset, Node* dest, Node* dest_offset, - Node* copy_length); + Node* copy_length, bool dest_uninitialized); Node* generate_generic_arraycopy(const TypePtr* adr_type, Node* src, Node* src_offset, Node* dest, Node* dest_offset, - Node* copy_length); + Node* copy_length, bool dest_uninitialized); void generate_unchecked_arraycopy(const TypePtr* adr_type, BasicType basic_elem_type, bool disjoint_bases, Node* src, Node* src_offset, Node* dest, Node* dest_offset, - Node* copy_length); + Node* copy_length, bool dest_uninitialized); bool inline_unsafe_CAS(BasicType type); bool inline_unsafe_ordered_store(BasicType type); bool inline_fp_conversions(vmIntrinsics::ID id); @@ -4092,7 +4092,8 @@ const TypePtr* raw_adr_type = TypeRawPtr::BOTTOM; bool disjoint_bases = true; generate_unchecked_arraycopy(raw_adr_type, T_LONG, disjoint_bases, - src, NULL, dest, NULL, countx); + src, NULL, dest, NULL, countx, + /*dest_uninitialized*/true); // If necessary, emit some card marks afterwards. (Non-arrays only.) if (card_mark) { @@ -4306,7 +4307,7 @@ // Note: The condition "disjoint" applies also for overlapping copies // where an descending copy is permitted (i.e., dest_offset <= src_offset). static address -select_arraycopy_function(BasicType t, bool aligned, bool disjoint, const char* &name) { +select_arraycopy_function(BasicType t, bool aligned, bool disjoint, const char* &name, bool dest_uninitialized) { int selector = (aligned ? COPYFUNC_ALIGNED : COPYFUNC_UNALIGNED) + (disjoint ? COPYFUNC_DISJOINT : COPYFUNC_CONJOINT); @@ -4315,6 +4316,10 @@ name = #xxx_arraycopy; \ return StubRoutines::xxx_arraycopy(); } +#define RETURN_STUB_PARM(xxx_arraycopy, parm) { \ + name = #xxx_arraycopy; \ + return StubRoutines::xxx_arraycopy(parm); } + switch (t) { case T_BYTE: case T_BOOLEAN: @@ -4351,10 +4356,10 @@ case T_ARRAY: case T_OBJECT: switch (selector) { - case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(oop_arraycopy); - case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_oop_arraycopy); - case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED: RETURN_STUB(oop_disjoint_arraycopy); - case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB(arrayof_oop_disjoint_arraycopy); + case COPYFUNC_CONJOINT | COPYFUNC_UNALIGNED: RETURN_STUB_PARM(oop_arraycopy, dest_uninitialized); + case COPYFUNC_CONJOINT | COPYFUNC_ALIGNED: RETURN_STUB_PARM(arrayof_oop_arraycopy, dest_uninitialized); + case COPYFUNC_DISJOINT | COPYFUNC_UNALIGNED: RETURN_STUB_PARM(oop_disjoint_arraycopy, dest_uninitialized); + case COPYFUNC_DISJOINT | COPYFUNC_ALIGNED: RETURN_STUB_PARM(arrayof_oop_disjoint_arraycopy, dest_uninitialized); } default: ShouldNotReachHere(); @@ -4362,6 +4367,7 @@ } #undef RETURN_STUB +#undef RETURN_STUB_PARM } //------------------------------basictype2arraycopy---------------------------- @@ -4369,7 +4375,8 @@ Node* src_offset, Node* dest_offset, bool disjoint_bases, - const char* &name) { + const char* &name, + bool dest_uninitialized) { const TypeInt* src_offset_inttype = gvn().find_int_type(src_offset);; const TypeInt* dest_offset_inttype = gvn().find_int_type(dest_offset);; @@ -4395,7 +4402,7 @@ disjoint = true; } - return select_arraycopy_function(t, aligned, disjoint, name); + return select_arraycopy_function(t, aligned, disjoint, name, dest_uninitialized); } @@ -4451,7 +4458,8 @@ // The component types are not the same or are not recognized. Punt. // (But, avoid the native method wrapper to JVM_ArrayCopy.) generate_slow_arraycopy(TypePtr::BOTTOM, - src, src_offset, dest, dest_offset, length); + src, src_offset, dest, dest_offset, length, + /*dest_uninitialized*/false); return true; } @@ -4564,7 +4572,7 @@ Node* original_dest = dest; AllocateArrayNode* alloc = NULL; // used for zeroing, if needed - bool must_clear_dest = false; + bool dest_uninitialized = false; // See if this is the initialization of a newly-allocated array. // If so, we will take responsibility here for initializing it to zero. @@ -4587,12 +4595,14 @@ adr_type = TypeRawPtr::BOTTOM; // all initializations are into raw memory // From this point on, every exit path is responsible for // initializing any non-copied parts of the object to zero. - must_clear_dest = true; + // Also, if this flag is set we make sure that arraycopy interacts properly + // with G1, eliding pre-barriers. See CR 6627983. + dest_uninitialized = true; } else { // No zeroing elimination here. alloc = NULL; //original_dest = dest; - //must_clear_dest = false; + //dest_uninitialized = false; } // Results are placed here: @@ -4624,10 +4634,10 @@ Node* checked_value = NULL; if (basic_elem_type == T_CONFLICT) { - assert(!must_clear_dest, ""); + assert(!dest_uninitialized, ""); Node* cv = generate_generic_arraycopy(adr_type, src, src_offset, dest, dest_offset, - copy_length); + copy_length, dest_uninitialized); if (cv == NULL) cv = intcon(-1); // failure (no stub available) checked_control = control(); checked_i_o = i_o(); @@ -4647,7 +4657,7 @@ } // copy_length is 0. - if (!stopped() && must_clear_dest) { + if (!stopped() && dest_uninitialized) { Node* dest_length = alloc->in(AllocateNode::ALength); if (_gvn.eqv_uncast(copy_length, dest_length) || _gvn.find_int_con(dest_length, 1) <= 0) { @@ -4673,7 +4683,7 @@ result_memory->init_req(zero_path, memory(adr_type)); } - if (!stopped() && must_clear_dest) { + if (!stopped() && dest_uninitialized) { // We have to initialize the *uncopied* part of the array to zero. // The copy destination is the slice dest[off..off+len]. The other slices // are dest_head = dest[0..off] and dest_tail = dest[off+len..dest.length]. @@ -4709,7 +4719,7 @@ { PreserveJVMState pjvms(this); didit = generate_block_arraycopy(adr_type, basic_elem_type, alloc, src, src_offset, dest, dest_offset, - dest_size); + dest_size, dest_uninitialized); if (didit) { // Present the results of the block-copying fast call. result_region->init_req(bcopy_path, control()); @@ -4785,7 +4795,7 @@ Node* cv = generate_checkcast_arraycopy(adr_type, dest_elem_klass, src, src_offset, dest, dest_offset, - ConvI2X(copy_length)); + ConvI2X(copy_length), dest_uninitialized); if (cv == NULL) cv = intcon(-1); // failure (no stub available) checked_control = control(); checked_i_o = i_o(); @@ -4808,7 +4818,7 @@ PreserveJVMState pjvms(this); generate_unchecked_arraycopy(adr_type, copy_type, disjoint_bases, src, src_offset, dest, dest_offset, - ConvI2X(copy_length)); + ConvI2X(copy_length), dest_uninitialized); // Present the results of the fast call. result_region->init_req(fast_path, control()); @@ -4887,7 +4897,7 @@ set_memory(slow_mem, adr_type); set_i_o(slow_i_o); - if (must_clear_dest) { + if (dest_uninitialized) { generate_clear_array(adr_type, dest, basic_elem_type, intcon(0), NULL, alloc->in(AllocateNode::AllocSize)); @@ -4895,7 +4905,7 @@ generate_slow_arraycopy(adr_type, src, src_offset, dest, dest_offset, - copy_length); + copy_length, /*dest_uninitialized*/false); result_region->init_req(slow_call_path, control()); result_i_o ->init_req(slow_call_path, i_o()); @@ -5139,7 +5149,7 @@ AllocateNode* alloc, Node* src, Node* src_offset, Node* dest, Node* dest_offset, - Node* dest_size) { + Node* dest_size, bool dest_uninitialized) { // See if there is an advantage from block transfer. int scale = exact_log2(type2aelembytes(basic_elem_type)); if (scale >= LogBytesPerLong) @@ -5184,7 +5194,7 @@ bool disjoint_bases = true; // since alloc != NULL generate_unchecked_arraycopy(adr_type, T_LONG, disjoint_bases, - sptr, NULL, dptr, NULL, countx); + sptr, NULL, dptr, NULL, countx, dest_uninitialized); return true; } @@ -5197,7 +5207,8 @@ LibraryCallKit::generate_slow_arraycopy(const TypePtr* adr_type, Node* src, Node* src_offset, Node* dest, Node* dest_offset, - Node* copy_length) { + Node* copy_length, bool dest_uninitialized) { + assert(!dest_uninitialized, "Invariant"); Node* call = make_runtime_call(RC_NO_LEAF | RC_UNCOMMON, OptoRuntime::slow_arraycopy_Type(), OptoRuntime::slow_arraycopy_Java(), @@ -5215,10 +5226,10 @@ Node* dest_elem_klass, Node* src, Node* src_offset, Node* dest, Node* dest_offset, - Node* copy_length) { + Node* copy_length, bool dest_uninitialized) { if (stopped()) return NULL; - address copyfunc_addr = StubRoutines::checkcast_arraycopy(); + address copyfunc_addr = StubRoutines::checkcast_arraycopy(dest_uninitialized); if (copyfunc_addr == NULL) { // Stub was not generated, go slow path. return NULL; } @@ -5256,9 +5267,9 @@ LibraryCallKit::generate_generic_arraycopy(const TypePtr* adr_type, Node* src, Node* src_offset, Node* dest, Node* dest_offset, - Node* copy_length) { + Node* copy_length, bool dest_uninitialized) { + assert(!dest_uninitialized, "Invariant"); if (stopped()) return NULL; - address copyfunc_addr = StubRoutines::generic_arraycopy(); if (copyfunc_addr == NULL) { // Stub was not generated, go slow path. return NULL; @@ -5279,7 +5290,7 @@ bool disjoint_bases, Node* src, Node* src_offset, Node* dest, Node* dest_offset, - Node* copy_length) { + Node* copy_length, bool dest_uninitialized) { if (stopped()) return; // nothing to do Node* src_start = src; @@ -5294,7 +5305,7 @@ const char* copyfunc_name = "arraycopy"; address copyfunc_addr = basictype2arraycopy(basic_elem_type, src_offset, dest_offset, - disjoint_bases, copyfunc_name); + disjoint_bases, copyfunc_name, dest_uninitialized); // Call it. Note that the count_ix value is not scaled to a byte-size. make_runtime_call(RC_LEAF|RC_NO_FP,
--- a/src/share/vm/runtime/arguments.cpp Tue Mar 01 10:27:15 2011 -0800 +++ b/src/share/vm/runtime/arguments.cpp Tue Mar 01 14:56:48 2011 -0800 @@ -2790,10 +2790,6 @@ if (!FLAG_IS_DEFAULT(OptoLoopAlignment) && FLAG_IS_DEFAULT(MaxLoopPad)) { FLAG_SET_DEFAULT(MaxLoopPad, OptoLoopAlignment-1); } - // Temporary disable bulk zeroing reduction with G1. See CR 6627983. - if (UseG1GC) { - FLAG_SET_DEFAULT(ReduceBulkZeroing, false); - } #endif // If we are running in a headless jre, force java.awt.headless property
--- a/src/share/vm/runtime/stubRoutines.cpp Tue Mar 01 10:27:15 2011 -0800 +++ b/src/share/vm/runtime/stubRoutines.cpp Tue Mar 01 14:56:48 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -80,30 +80,36 @@ jint StubRoutines::_fpu_subnormal_bias2[3] = { 0, 0, 0 }; // Compiled code entry points default values -// The dafault functions don't have separate disjoint versions. +// The default functions don't have separate disjoint versions. address StubRoutines::_jbyte_arraycopy = CAST_FROM_FN_PTR(address, StubRoutines::jbyte_copy); address StubRoutines::_jshort_arraycopy = CAST_FROM_FN_PTR(address, StubRoutines::jshort_copy); address StubRoutines::_jint_arraycopy = CAST_FROM_FN_PTR(address, StubRoutines::jint_copy); address StubRoutines::_jlong_arraycopy = CAST_FROM_FN_PTR(address, StubRoutines::jlong_copy); address StubRoutines::_oop_arraycopy = CAST_FROM_FN_PTR(address, StubRoutines::oop_copy); +address StubRoutines::_oop_arraycopy_uninit = CAST_FROM_FN_PTR(address, StubRoutines::oop_copy_uninit); address StubRoutines::_jbyte_disjoint_arraycopy = CAST_FROM_FN_PTR(address, StubRoutines::jbyte_copy); address StubRoutines::_jshort_disjoint_arraycopy = CAST_FROM_FN_PTR(address, StubRoutines::jshort_copy); address StubRoutines::_jint_disjoint_arraycopy = CAST_FROM_FN_PTR(address, StubRoutines::jint_copy); address StubRoutines::_jlong_disjoint_arraycopy = CAST_FROM_FN_PTR(address, StubRoutines::jlong_copy); address StubRoutines::_oop_disjoint_arraycopy = CAST_FROM_FN_PTR(address, StubRoutines::oop_copy); +address StubRoutines::_oop_disjoint_arraycopy_uninit = CAST_FROM_FN_PTR(address, StubRoutines::oop_copy_uninit); address StubRoutines::_arrayof_jbyte_arraycopy = CAST_FROM_FN_PTR(address, StubRoutines::arrayof_jbyte_copy); address StubRoutines::_arrayof_jshort_arraycopy = CAST_FROM_FN_PTR(address, StubRoutines::arrayof_jshort_copy); address StubRoutines::_arrayof_jint_arraycopy = CAST_FROM_FN_PTR(address, StubRoutines::arrayof_jint_copy); address StubRoutines::_arrayof_jlong_arraycopy = CAST_FROM_FN_PTR(address, StubRoutines::arrayof_jlong_copy); address StubRoutines::_arrayof_oop_arraycopy = CAST_FROM_FN_PTR(address, StubRoutines::arrayof_oop_copy); +address StubRoutines::_arrayof_oop_arraycopy_uninit = CAST_FROM_FN_PTR(address, StubRoutines::arrayof_oop_copy_uninit); address StubRoutines::_arrayof_jbyte_disjoint_arraycopy = CAST_FROM_FN_PTR(address, StubRoutines::arrayof_jbyte_copy); address StubRoutines::_arrayof_jshort_disjoint_arraycopy = CAST_FROM_FN_PTR(address, StubRoutines::arrayof_jshort_copy); address StubRoutines::_arrayof_jint_disjoint_arraycopy = CAST_FROM_FN_PTR(address, StubRoutines::arrayof_jint_copy); address StubRoutines::_arrayof_jlong_disjoint_arraycopy = CAST_FROM_FN_PTR(address, StubRoutines::arrayof_jlong_copy); -address StubRoutines::_arrayof_oop_disjoint_arraycopy = CAST_FROM_FN_PTR(address, StubRoutines::arrayof_oop_copy); +address StubRoutines::_arrayof_oop_disjoint_arraycopy = CAST_FROM_FN_PTR(address, StubRoutines::arrayof_oop_copy); +address StubRoutines::_arrayof_oop_disjoint_arraycopy_uninit = CAST_FROM_FN_PTR(address, StubRoutines::arrayof_oop_copy_uninit); + address StubRoutines::_checkcast_arraycopy = NULL; +address StubRoutines::_checkcast_arraycopy_uninit = NULL; address StubRoutines::_unsafe_arraycopy = NULL; address StubRoutines::_generic_arraycopy = NULL; @@ -282,12 +288,12 @@ // Default versions of arraycopy functions // -static void gen_arraycopy_barrier_pre(oop* dest, size_t count) { +static void gen_arraycopy_barrier_pre(oop* dest, size_t count, bool dest_uninitialized) { assert(count != 0, "count should be non-zero"); assert(count <= (size_t)max_intx, "count too large"); BarrierSet* bs = Universe::heap()->barrier_set(); assert(bs->has_write_ref_array_pre_opt(), "Must have pre-barrier opt"); - bs->write_ref_array_pre(dest, (int)count); + bs->write_ref_array_pre(dest, (int)count, dest_uninitialized); } static void gen_arraycopy_barrier(oop* dest, size_t count) { @@ -330,7 +336,17 @@ SharedRuntime::_oop_array_copy_ctr++; // Slow-path oop array copy #endif // !PRODUCT assert(count != 0, "count should be non-zero"); - gen_arraycopy_barrier_pre(dest, count); + gen_arraycopy_barrier_pre(dest, count, /*dest_uninitialized*/false); + Copy::conjoint_oops_atomic(src, dest, count); + gen_arraycopy_barrier(dest, count); +JRT_END + +JRT_LEAF(void, StubRoutines::oop_copy_uninit(oop* src, oop* dest, size_t count)) +#ifndef PRODUCT + SharedRuntime::_oop_array_copy_ctr++; // Slow-path oop array copy +#endif // !PRODUCT + assert(count != 0, "count should be non-zero"); + gen_arraycopy_barrier_pre(dest, count, /*dest_uninitialized*/true); Copy::conjoint_oops_atomic(src, dest, count); gen_arraycopy_barrier(dest, count); JRT_END @@ -368,11 +384,20 @@ SharedRuntime::_oop_array_copy_ctr++; // Slow-path oop array copy #endif // !PRODUCT assert(count != 0, "count should be non-zero"); - gen_arraycopy_barrier_pre((oop *) dest, count); + gen_arraycopy_barrier_pre((oop *) dest, count, /*dest_uninitialized*/false); Copy::arrayof_conjoint_oops(src, dest, count); gen_arraycopy_barrier((oop *) dest, count); JRT_END +JRT_LEAF(void, StubRoutines::arrayof_oop_copy_uninit(HeapWord* src, HeapWord* dest, size_t count)) +#ifndef PRODUCT + SharedRuntime::_oop_array_copy_ctr++; // Slow-path oop array copy +#endif // !PRODUCT + assert(count != 0, "count should be non-zero"); + gen_arraycopy_barrier_pre((oop *) dest, count, /*dest_uninitialized*/true); + Copy::arrayof_conjoint_oops(src, dest, count); + gen_arraycopy_barrier((oop *) dest, count); +JRT_END address StubRoutines::select_fill_function(BasicType t, bool aligned, const char* &name) { #define RETURN_STUB(xxx_fill) { \
--- a/src/share/vm/runtime/stubRoutines.hpp Tue Mar 01 10:27:15 2011 -0800 +++ b/src/share/vm/runtime/stubRoutines.hpp Tue Mar 01 14:56:48 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 1997, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 1997, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -164,12 +164,12 @@ static address _jshort_arraycopy; static address _jint_arraycopy; static address _jlong_arraycopy; - static address _oop_arraycopy; + static address _oop_arraycopy, _oop_arraycopy_uninit; static address _jbyte_disjoint_arraycopy; static address _jshort_disjoint_arraycopy; static address _jint_disjoint_arraycopy; static address _jlong_disjoint_arraycopy; - static address _oop_disjoint_arraycopy; + static address _oop_disjoint_arraycopy, _oop_disjoint_arraycopy_uninit; // arraycopy operands aligned on zero'th element boundary // These are identical to the ones aligned aligned on an @@ -179,15 +179,15 @@ static address _arrayof_jshort_arraycopy; static address _arrayof_jint_arraycopy; static address _arrayof_jlong_arraycopy; - static address _arrayof_oop_arraycopy; + static address _arrayof_oop_arraycopy, _arrayof_oop_arraycopy_uninit; static address _arrayof_jbyte_disjoint_arraycopy; static address _arrayof_jshort_disjoint_arraycopy; static address _arrayof_jint_disjoint_arraycopy; static address _arrayof_jlong_disjoint_arraycopy; - static address _arrayof_oop_disjoint_arraycopy; + static address _arrayof_oop_disjoint_arraycopy, _arrayof_oop_disjoint_arraycopy_uninit; // these are recommended but optional: - static address _checkcast_arraycopy; + static address _checkcast_arraycopy, _checkcast_arraycopy_uninit; static address _unsafe_arraycopy; static address _generic_arraycopy; @@ -286,26 +286,36 @@ static address jshort_arraycopy() { return _jshort_arraycopy; } static address jint_arraycopy() { return _jint_arraycopy; } static address jlong_arraycopy() { return _jlong_arraycopy; } - static address oop_arraycopy() { return _oop_arraycopy; } + static address oop_arraycopy(bool dest_uninitialized = false) { + return dest_uninitialized ? _oop_arraycopy_uninit : _oop_arraycopy; + } static address jbyte_disjoint_arraycopy() { return _jbyte_disjoint_arraycopy; } static address jshort_disjoint_arraycopy() { return _jshort_disjoint_arraycopy; } static address jint_disjoint_arraycopy() { return _jint_disjoint_arraycopy; } static address jlong_disjoint_arraycopy() { return _jlong_disjoint_arraycopy; } - static address oop_disjoint_arraycopy() { return _oop_disjoint_arraycopy; } + static address oop_disjoint_arraycopy(bool dest_uninitialized = false) { + return dest_uninitialized ? _oop_disjoint_arraycopy_uninit : _oop_disjoint_arraycopy; + } static address arrayof_jbyte_arraycopy() { return _arrayof_jbyte_arraycopy; } static address arrayof_jshort_arraycopy() { return _arrayof_jshort_arraycopy; } static address arrayof_jint_arraycopy() { return _arrayof_jint_arraycopy; } static address arrayof_jlong_arraycopy() { return _arrayof_jlong_arraycopy; } - static address arrayof_oop_arraycopy() { return _arrayof_oop_arraycopy; } + static address arrayof_oop_arraycopy(bool dest_uninitialized = false) { + return dest_uninitialized ? _arrayof_oop_arraycopy_uninit : _arrayof_oop_arraycopy; + } static address arrayof_jbyte_disjoint_arraycopy() { return _arrayof_jbyte_disjoint_arraycopy; } static address arrayof_jshort_disjoint_arraycopy() { return _arrayof_jshort_disjoint_arraycopy; } static address arrayof_jint_disjoint_arraycopy() { return _arrayof_jint_disjoint_arraycopy; } static address arrayof_jlong_disjoint_arraycopy() { return _arrayof_jlong_disjoint_arraycopy; } - static address arrayof_oop_disjoint_arraycopy() { return _arrayof_oop_disjoint_arraycopy; } + static address arrayof_oop_disjoint_arraycopy(bool dest_uninitialized = false) { + return dest_uninitialized ? _arrayof_oop_disjoint_arraycopy_uninit : _arrayof_oop_disjoint_arraycopy; + } - static address checkcast_arraycopy() { return _checkcast_arraycopy; } + static address checkcast_arraycopy(bool dest_uninitialized = false) { + return dest_uninitialized ? _checkcast_arraycopy_uninit : _checkcast_arraycopy; + } static address unsafe_arraycopy() { return _unsafe_arraycopy; } static address generic_arraycopy() { return _generic_arraycopy; } @@ -352,17 +362,19 @@ // Default versions of the above arraycopy functions for platforms which do // not have specialized versions // - static void jbyte_copy (jbyte* src, jbyte* dest, size_t count); - static void jshort_copy(jshort* src, jshort* dest, size_t count); - static void jint_copy (jint* src, jint* dest, size_t count); - static void jlong_copy (jlong* src, jlong* dest, size_t count); - static void oop_copy (oop* src, oop* dest, size_t count); + static void jbyte_copy (jbyte* src, jbyte* dest, size_t count); + static void jshort_copy (jshort* src, jshort* dest, size_t count); + static void jint_copy (jint* src, jint* dest, size_t count); + static void jlong_copy (jlong* src, jlong* dest, size_t count); + static void oop_copy (oop* src, oop* dest, size_t count); + static void oop_copy_uninit(oop* src, oop* dest, size_t count); - static void arrayof_jbyte_copy (HeapWord* src, HeapWord* dest, size_t count); - static void arrayof_jshort_copy(HeapWord* src, HeapWord* dest, size_t count); - static void arrayof_jint_copy (HeapWord* src, HeapWord* dest, size_t count); - static void arrayof_jlong_copy (HeapWord* src, HeapWord* dest, size_t count); - static void arrayof_oop_copy (HeapWord* src, HeapWord* dest, size_t count); + static void arrayof_jbyte_copy (HeapWord* src, HeapWord* dest, size_t count); + static void arrayof_jshort_copy (HeapWord* src, HeapWord* dest, size_t count); + static void arrayof_jint_copy (HeapWord* src, HeapWord* dest, size_t count); + static void arrayof_jlong_copy (HeapWord* src, HeapWord* dest, size_t count); + static void arrayof_oop_copy (HeapWord* src, HeapWord* dest, size_t count); + static void arrayof_oop_copy_uninit(HeapWord* src, HeapWord* dest, size_t count); }; #endif // SHARE_VM_RUNTIME_STUBROUTINES_HPP