truffle: src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp comparison

comparison src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp @ 526:818efdefcc99

6484956: G1: improve evacuation pause efficiency Summary: A bunch of performance optimizations to decrease GC pause times in G1. Reviewed-by: apetrusenko, jmasa, iveresov

author	tonyp
date	Fri, 16 Jan 2009 13:02:20 -0500
parents	569b3b226089
children	58054a18d735

comparison

equal deleted inserted replaced

-:65de26b5ea82
+:818efdefcc99
 _full_collection(false),
 _unclean_region_list(),
 _unclean_regions_coming(false),
 _young_list(new YoungList(this)),
 _gc_time_stamp(0),
-_surviving_young_words(NULL)
+_surviving_young_words(NULL),
+_in_cset_fast_test(NULL),
+_in_cset_fast_test_base(NULL)
 {
 _g1h = this; // To catch bugs.
 if (_process_strong_tasks == NULL || !_process_strong_tasks->valid()) {
 vm_exit_during_initialization("Failed necessary allocation.");
 }
 }
 g1_policy()->record_collection_pause_start(start_time_sec,
 start_used_bytes);
+guarantee(_in_cset_fast_test == NULL, "invariant");
+guarantee(_in_cset_fast_test_base == NULL, "invariant");
+_in_cset_fast_test_length = n_regions();
+_in_cset_fast_test_base =
+NEW_C_HEAP_ARRAY(bool, _in_cset_fast_test_length);
+memset(_in_cset_fast_test_base, false,
+_in_cset_fast_test_length * sizeof(bool));
+// We're biasing _in_cset_fast_test to avoid subtracting the
+// beginning of the heap every time we want to index; basically
+// it's the same with what we do with the card table.
+_in_cset_fast_test = _in_cset_fast_test_base -
+((size_t) _g1_reserved.start() >> HeapRegion::LogOfHRGrainBytes);
 #if SCAN_ONLY_VERBOSE
 _young_list->print();
 #endif // SCAN_ONLY_VERBOSE
 if (g1_policy()->should_initiate_conc_mark()) {
 // Actually do the work...
 evacuate_collection_set();
 free_collection_set(g1_policy()->collection_set());
 g1_policy()->clear_collection_set();
+FREE_C_HEAP_ARRAY(bool, _in_cset_fast_test_base);
+// this is more for peace of mind; we're nulling them here and
+// we're expecting them to be null at the beginning of the next GC
+_in_cset_fast_test = NULL;
+_in_cset_fast_test_base = NULL;
 if (popular_region != NULL) {
 // We have to wait until now, because we don't want the region to
 // be rescheduled for pop-evac during RS update.
 popular_region->set_popular_pending(false);
 size_t alloc_buffer_waste()                    { return _alloc_buffer_waste; }
 size_t undo_waste()                            { return _undo_waste; }
 void push_on_queue(oop* ref) {
+assert(ref != NULL, "invariant");
+assert(has_partial_array_mask(ref) || _g1h->obj_in_cs(*ref), "invariant");
 if (!refs()->push(ref)) {
 overflowed_refs()->push(ref);
 IF_G1_DETAILED_STATS(note_overflow_push());
 } else {
 IF_G1_DETAILED_STATS(note_push());
 void pop_from_queue(oop*& ref) {
 if (!refs()->pop_local(ref)) {
 ref = NULL;
 } else {
+assert(ref != NULL, "invariant");
+assert(has_partial_array_mask(ref) || _g1h->obj_in_cs(*ref),
+"invariant");
 IF_G1_DETAILED_STATS(note_pop());
 }
 }
 void pop_from_overflow_queue(oop*& ref) {
 // Otherwise.
 alloc_buf->set_buf(buf);
 obj = alloc_buf->allocate(word_sz);
 assert(obj != NULL, "buffer was definitely big enough...");
-}
+} else {
-else {
 obj = _g1h->par_allocate_during_gc(purpose, word_sz);
 }
 return obj;
 }
 add_to_alloc_buffer_waste(waste);
 _alloc_buffers[ap].retire(true, false);
 }
 }
+private:
+void deal_with_reference(oop* ref_to_scan) {
+if (has_partial_array_mask(ref_to_scan)) {
+_partial_scan_cl->do_oop_nv(ref_to_scan);
+} else {
+// Note: we can use "raw" versions of "region_containing" because
+// "obj_to_scan" is definitely in the heap, and is not in a
+// humongous region.
+HeapRegion* r = _g1h->heap_region_containing_raw(ref_to_scan);
+_evac_cl->set_region(r);
+_evac_cl->do_oop_nv(ref_to_scan);
+}
+}
+public:
 void trim_queue() {
+// I've replicated the loop twice, first to drain the overflow
+// queue, second to drain the task queue. This is better than
+// having a single loop, which checks both conditions and, inside
+// it, either pops the overflow queue or the task queue, as each
+// loop is tighter. Also, the decision to drain the overflow queue
+// first is not arbitrary, as the overflow queue is not visible
+// to the other workers, whereas the task queue is. So, we want to
+// drain the "invisible" entries first, while allowing the other
+// workers to potentially steal the "visible" entries.
 while (refs_to_scan() > 0 || overflowed_refs_to_scan() > 0) {
-oop *ref_to_scan = NULL;
+while (overflowed_refs_to_scan() > 0) {
-if (overflowed_refs_to_scan() == 0) {
+oop *ref_to_scan = NULL;
+pop_from_overflow_queue(ref_to_scan);
+assert(ref_to_scan != NULL, "invariant");
+// We shouldn't have pushed it on the queue if it was not
+// pointing into the CSet.
+assert(ref_to_scan != NULL, "sanity");
+assert(has_partial_array_mask(ref_to_scan) ||
+_g1h->obj_in_cs(*ref_to_scan), "sanity");
+deal_with_reference(ref_to_scan);
+}
+while (refs_to_scan() > 0) {
+oop *ref_to_scan = NULL;
 pop_from_queue(ref_to_scan);
-} else {
-pop_from_overflow_queue(ref_to_scan);
+if (ref_to_scan != NULL) {
-}
+// We shouldn't have pushed it on the queue if it was not
-if (ref_to_scan != NULL) {
+// pointing into the CSet.
-if ((intptr_t)ref_to_scan & G1_PARTIAL_ARRAY_MASK) {
+assert(has_partial_array_mask(ref_to_scan) ||
-_partial_scan_cl->do_oop_nv(ref_to_scan);
+_g1h->obj_in_cs(*ref_to_scan), "sanity");
-} else {
-// Note: we can use "raw" versions of "region_containing" because
+deal_with_reference(ref_to_scan);
-// "obj_to_scan" is definitely in the heap, and is not in a
-// humongous region.
-HeapRegion* r = _g1h->heap_region_containing_raw(ref_to_scan);
-_evac_cl->set_region(r);
-_evac_cl->do_oop_nv(ref_to_scan);
 }
 }
 }
 }
 };
 // This closure is applied to the fields of the objects that have just been copied.
 // Should probably be made inline and moved in g1OopClosures.inline.hpp.
 void G1ParScanClosure::do_oop_nv(oop* p) {
 oop obj = *p;
 if (obj != NULL) {
-if (_g1->obj_in_cs(obj)) {
+if (_g1->in_cset_fast_test(obj)) {
-if (obj->is_forwarded()) {
+// We're not going to even bother checking whether the object is
-*p = obj->forwardee();
+// already forwarded or not, as this usually causes an immediate
-} else {
+// stall. We'll try to prefetch the object (for write, given that
-_par_scan_state->push_on_queue(p);
+// we might need to install the forwarding reference) and we'll
-return;
+// get back to it when pop it from the queue
-}
+Prefetch::write(obj->mark_addr(), 0);
-}
+Prefetch::read(obj->mark_addr(), (HeapWordSize*2));
-_g1_rem->par_write_ref(_from, p, _par_scan_state->queue_num());
+// slightly paranoid test; I'm trying to catch potential
+// problems before we go into push_on_queue to know where the
+// problem is coming from
+assert(obj == *p, "the value of *p should not have changed");
+_par_scan_state->push_on_queue(p);
+} else {
+_g1_rem->par_write_ref(_from, p, _par_scan_state->queue_num());
+}
 }
 }
 void G1ParCopyHelper::mark_forwardee(oop* p) {
 // This is called _after_ do_oop_work has been called, hence after
 // installed a forwarding pointer.
 OopsInHeapRegionClosure* cl = _par_scan_state->evac_failure_closure();
 return _g1->handle_evacuation_failure_par(cl, old);
 }
+// We're going to allocate linearly, so might as well prefetch ahead.
+Prefetch::write(obj_ptr, PrefetchCopyIntervalInBytes);
 oop forward_ptr = old->forward_to_atomic(obj);
 if (forward_ptr == NULL) {
 Copy::aligned_disjoint_words((HeapWord*) old, obj_ptr, word_sz);
+if (g1p->track_object_age(alloc_purpose)) {
+// We could simply do obj->incr_age(). However, this causes a
+// performance issue. obj->incr_age() will first check whether
+// the object has a displaced mark by checking its mark word;
+// getting the mark word from the new location of the object
+// stalls. So, given that we already have the mark word and we
+// are about to install it anyway, it's better to increase the
+// age on the mark word, when the object does not have a
+// displaced mark word. We're not expecting many objects to have
+// a displaced marked word, so that case is not optimized
+// further (it could be...) and we simply call obj->incr_age().
+if (m->has_displaced_mark_helper()) {
+// in this case, we have to install the mark word first,
+// otherwise obj looks to be forwarded (the old mark word,
+// which contains the forward pointer, was copied)
+obj->set_mark(m);
+obj->incr_age();
+} else {
+m = m->incr_age();
+}
+}
 obj->set_mark(m);
-if (g1p->track_object_age(alloc_purpose)) {
-obj->incr_age();
-}
 // preserve "next" mark bit
 if (_g1->mark_in_progress() && !_g1->is_obj_ill(old)) {
 if (!use_local_bitmaps ||
 !_par_scan_state->alloc_buffer(alloc_purpose)->mark(obj_ptr)) {
 // if we couldn't mark it on the local bitmap (this happens when
 size_t* surv_young_words = _par_scan_state->surviving_young_words();
 surv_young_words[young_index] += word_sz;
 if (obj->is_objArray() && arrayOop(obj)->length() >= ParGCArrayScanChunk) {
 arrayOop(old)->set_length(0);
-_par_scan_state->push_on_queue((oop*) ((intptr_t)old | G1_PARTIAL_ARRAY_MASK));
+_par_scan_state->push_on_queue(set_partial_array_mask(old));
 } else {
-_scanner->set_region(_g1->heap_region_containing(obj));
+// No point in using the slower heap_region_containing() method,
+// given that we know obj is in the heap.
+_scanner->set_region(_g1->heap_region_containing_raw(obj));
 obj->oop_iterate_backwards(_scanner);
 }
 } else {
 _par_scan_state->undo_allocation(alloc_purpose, obj_ptr, word_sz);
 obj = forward_ptr;
 }
 return obj;
 }
-template<bool do_gen_barrier, G1Barrier barrier, bool do_mark_forwardee>
+template<bool do_gen_barrier, G1Barrier barrier,
-void G1ParCopyClosure<do_gen_barrier, barrier, do_mark_forwardee>::do_oop_work(oop* p) {
+bool do_mark_forwardee, bool skip_cset_test>
+void G1ParCopyClosure<do_gen_barrier, barrier,
+do_mark_forwardee, skip_cset_test>::do_oop_work(oop* p) {
 oop obj = *p;
 assert(barrier != G1BarrierRS || obj != NULL,
 "Precondition: G1BarrierRS implies obj is nonNull");
-if (obj != NULL) {
+// The only time we skip the cset test is when we're scanning
-if (_g1->obj_in_cs(obj)) {
+// references popped from the queue. And we only push on the queue
+// references that we know point into the cset, so no point in
+// checking again. But we'll leave an assert here for peace of mind.
+assert(!skip_cset_test || _g1->obj_in_cs(obj), "invariant");
+// here the null check is implicit in the cset_fast_test() test
+if (skip_cset_test || _g1->in_cset_fast_test(obj)) {
 #if G1_REM_SET_LOGGING
-gclog_or_tty->print_cr("Loc "PTR_FORMAT" contains pointer "PTR_FORMAT" into CS.",
+gclog_or_tty->print_cr("Loc "PTR_FORMAT" contains pointer "PTR_FORMAT" "
-p, (void*) obj);
+"into CS.", p, (void*) obj);
 #endif
 if (obj->is_forwarded()) {
 *p = obj->forwardee();
 } else {
 *p = copy_to_survivor_space(obj);
 }
 // When scanning the RS, we only care about objs in CS.
 if (barrier == G1BarrierRS) {
-_g1_rem->par_write_ref(_from, p, _par_scan_state->queue_num());
-}
-}
-// When scanning moved objs, must look at all oops.
-if (barrier == G1BarrierEvac) {
 _g1_rem->par_write_ref(_from, p, _par_scan_state->queue_num());
 }
+}
-if (do_gen_barrier) {
-par_do_barrier(p);
+// When scanning moved objs, must look at all oops.
-}
+if (barrier == G1BarrierEvac && obj != NULL) {
-}
+_g1_rem->par_write_ref(_from, p, _par_scan_state->queue_num());
 }
-template void G1ParCopyClosure<false, G1BarrierEvac, false>::do_oop_work(oop* p);
+if (do_gen_barrier && obj != NULL) {
+par_do_barrier(p);
-template <class T> void G1ParScanPartialArrayClosure::process_array_chunk(
+}
+}
+template void G1ParCopyClosure<false, G1BarrierEvac, false, true>::do_oop_work(oop* p);
+template<class T> void G1ParScanPartialArrayClosure::process_array_chunk(
 oop obj, int start, int end) {
 // process our set of indices (include header in first chunk)
 assert(start < end, "invariant");
 T* const base      = (T*)objArrayOop(obj)->base();
-T* const start_addr = base + start;
+T* const start_addr = (start == 0) ? (T*) obj : base + start;
 T* const end_addr   = base + end;
 MemRegion mr((HeapWord*)start_addr, (HeapWord*)end_addr);
 _scanner.set_region(_g1->heap_region_containing(obj));
 obj->oop_iterate(&_scanner, mr);
 }
 void G1ParScanPartialArrayClosure::do_oop_nv(oop* p) {
 assert(!UseCompressedOops, "Needs to be fixed to work with compressed oops");
-oop old = oop((intptr_t)p & ~G1_PARTIAL_ARRAY_MASK);
+assert(has_partial_array_mask(p), "invariant");
+oop old = clear_partial_array_mask(p);
 assert(old->is_objArray(), "must be obj array");
 assert(old->is_forwarded(), "must be forwarded");
 assert(Universe::heap()->is_in_reserved(old), "must be in heap.");
 objArrayOop obj = objArrayOop(old->forwardee());
 if (remainder > 2 * ParGCArrayScanChunk) {
 // Test above combines last partial chunk with a full chunk
 end = start + ParGCArrayScanChunk;
 arrayOop(old)->set_length(end);
 // Push remainder.
-_par_scan_state->push_on_queue((oop*) ((intptr_t) old | G1_PARTIAL_ARRAY_MASK));
+_par_scan_state->push_on_queue(set_partial_array_mask(old));
 } else {
 // Restore length so that the heap remains parsable in
 // case of evacuation failure.
 arrayOop(old)->set_length(end);
 }
 // process our set of indices (include header in first chunk)
 process_array_chunk<oop>(obj, start, end);
-oop* start_addr = start == 0 ? (oop*)obj : obj->obj_at_addr<oop>(start);
-oop* end_addr   = (oop*)(obj->base()) + end; // obj_at_addr(end) asserts end < length
-MemRegion mr((HeapWord*)start_addr, (HeapWord*)end_addr);
-_scanner.set_region(_g1->heap_region_containing(obj));
-obj->oop_iterate(&_scanner, mr);
 }
 int G1ScanAndBalanceClosure::_nq = 0;
 class G1ParEvacuateFollowersClosure : public VoidClosure {
 IF_G1_DETAILED_STATS(pss->note_steal_attempt());
 if (queues()->steal(pss->queue_num(),
 pss->hash_seed(),
 ref_to_scan)) {
 IF_G1_DETAILED_STATS(pss->note_steal());
+// slightly paranoid tests; I'm trying to catch potential
+// problems before we go into push_on_queue to know where the
+// problem is coming from
+assert(ref_to_scan != NULL, "invariant");
+assert(has_partial_array_mask(ref_to_scan) ||
+_g1h->obj_in_cs(*ref_to_scan), "invariant");
 pss->push_on_queue(ref_to_scan);
 continue;
 }
 pss->start_term_time();
 if (terminator()->offer_termination()) break;
 void work(int i) {
 ResourceMark rm;
 HandleMark   hm;
-G1ParScanThreadState pss(_g1h, i);
+G1ParScanThreadState            pss(_g1h, i);
-G1ParScanHeapEvacClosure     scan_evac_cl(_g1h, &pss);
+G1ParScanHeapEvacClosure        scan_evac_cl(_g1h, &pss);
-G1ParScanHeapEvacClosure     evac_failure_cl(_g1h, &pss);
+G1ParScanHeapEvacFailureClosure evac_failure_cl(_g1h, &pss);
-G1ParScanPartialArrayClosure partial_scan_cl(_g1h, &pss);
+G1ParScanPartialArrayClosure    partial_scan_cl(_g1h, &pss);
 pss.set_evac_closure(&scan_evac_cl);
 pss.set_evac_failure_closure(&evac_failure_cl);
 pss.set_partial_scan_closure(&partial_scan_cl);

Mercurial > hg > truffle

comparison src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp @ 526:818efdefcc99