# HG changeset patch # User apetrusenko # Date 1242731131 25200 # Node ID 29e7d79232b9a277bdefbb162a8d8e52efc2bbca # Parent 215f81b4d9b35c3e0db2e225488995898506dc82 6819065: G1: eliminate high serial card table clearing time Reviewed-by: iveresov, tonyp diff -r 215f81b4d9b3 -r 29e7d79232b9 src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Mon May 18 11:52:46 2009 -0700 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Tue May 19 04:05:31 2009 -0700 @@ -446,6 +446,59 @@ gclog_or_tty->print_cr(""); } +void G1CollectedHeap::push_dirty_cards_region(HeapRegion* hr) +{ + // Claim the right to put the region on the dirty cards region list + // by installing a self pointer. + HeapRegion* next = hr->get_next_dirty_cards_region(); + if (next == NULL) { + HeapRegion* res = (HeapRegion*) + Atomic::cmpxchg_ptr(hr, hr->next_dirty_cards_region_addr(), + NULL); + if (res == NULL) { + HeapRegion* head; + do { + // Put the region to the dirty cards region list. + head = _dirty_cards_region_list; + next = (HeapRegion*) + Atomic::cmpxchg_ptr(hr, &_dirty_cards_region_list, head); + if (next == head) { + assert(hr->get_next_dirty_cards_region() == hr, + "hr->get_next_dirty_cards_region() != hr"); + if (next == NULL) { + // The last region in the list points to itself. + hr->set_next_dirty_cards_region(hr); + } else { + hr->set_next_dirty_cards_region(next); + } + } + } while (next != head); + } + } +} + +HeapRegion* G1CollectedHeap::pop_dirty_cards_region() +{ + HeapRegion* head; + HeapRegion* hr; + do { + head = _dirty_cards_region_list; + if (head == NULL) { + return NULL; + } + HeapRegion* new_head = head->get_next_dirty_cards_region(); + if (head == new_head) { + // The last region. + new_head = NULL; + } + hr = (HeapRegion*)Atomic::cmpxchg_ptr(new_head, &_dirty_cards_region_list, + head); + } while (hr != head); + assert(hr != NULL, "invariant"); + hr->set_next_dirty_cards_region(NULL); + return hr; +} + void G1CollectedHeap::stop_conc_gc_threads() { _cg1r->stop(); _czft->stop(); @@ -1329,7 +1382,8 @@ _gc_time_stamp(0), _surviving_young_words(NULL), _in_cset_fast_test(NULL), - _in_cset_fast_test_base(NULL) { + _in_cset_fast_test_base(NULL), + _dirty_cards_region_list(NULL) { _g1h = this; // To catch bugs. if (_process_strong_tasks == NULL || !_process_strong_tasks->valid()) { vm_exit_during_initialization("Failed necessary allocation."); @@ -4691,15 +4745,58 @@ } } + +class G1ParCleanupCTTask : public AbstractGangTask { + CardTableModRefBS* _ct_bs; + G1CollectedHeap* _g1h; +public: + G1ParCleanupCTTask(CardTableModRefBS* ct_bs, + G1CollectedHeap* g1h) : + AbstractGangTask("G1 Par Cleanup CT Task"), + _ct_bs(ct_bs), + _g1h(g1h) + { } + + void work(int i) { + HeapRegion* r; + while (r = _g1h->pop_dirty_cards_region()) { + clear_cards(r); + } + } + void clear_cards(HeapRegion* r) { + // Cards for Survivor and Scan-Only regions will be dirtied later. + if (!r->is_scan_only() && !r->is_survivor()) { + _ct_bs->clear(MemRegion(r->bottom(), r->end())); + } + } +}; + + void G1CollectedHeap::cleanUpCardTable() { CardTableModRefBS* ct_bs = (CardTableModRefBS*) (barrier_set()); double start = os::elapsedTime(); - ct_bs->clear(_g1_committed); - + // Iterate over the dirty cards region list. + G1ParCleanupCTTask cleanup_task(ct_bs, this); + if (ParallelGCThreads > 0) { + set_par_threads(workers()->total_workers()); + workers()->run_task(&cleanup_task); + set_par_threads(0); + } else { + while (_dirty_cards_region_list) { + HeapRegion* r = _dirty_cards_region_list; + cleanup_task.clear_cards(r); + _dirty_cards_region_list = r->get_next_dirty_cards_region(); + if (_dirty_cards_region_list == r) { + // The last region. + _dirty_cards_region_list = NULL; + } + r->set_next_dirty_cards_region(NULL); + } + } // now, redirty the cards of the scan-only and survivor regions // (it seemed faster to do it this way, instead of iterating over - // all regions and then clearing / dirtying as approprite) + // all regions and then clearing / dirtying as appropriate) dirtyCardsForYoungRegions(ct_bs, _young_list->first_scan_only_region()); dirtyCardsForYoungRegions(ct_bs, _young_list->first_survivor_region()); diff -r 215f81b4d9b3 -r 29e7d79232b9 src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp Mon May 18 11:52:46 2009 -0700 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp Tue May 19 04:05:31 2009 -0700 @@ -158,6 +158,7 @@ friend class RegionSorter; friend class CountRCClosure; friend class EvacPopObjClosure; + friend class G1ParCleanupCTTask; // Other related classes. friend class G1MarkSweep; @@ -1191,6 +1192,16 @@ ConcurrentMark* concurrent_mark() const { return _cm; } ConcurrentG1Refine* concurrent_g1_refine() const { return _cg1r; } + // The dirty cards region list is used to record a subset of regions + // whose cards need clearing. The list if populated during the + // remembered set scanning and drained during the card table + // cleanup. Although the methods are reentrant, population/draining + // phases must not overlap. For synchronization purposes the last + // element on the list points to itself. + HeapRegion* _dirty_cards_region_list; + void push_dirty_cards_region(HeapRegion* hr); + HeapRegion* pop_dirty_cards_region(); + public: void stop_conc_gc_threads(); diff -r 215f81b4d9b3 -r 29e7d79232b9 src/share/vm/gc_implementation/g1/g1RemSet.cpp --- a/src/share/vm/gc_implementation/g1/g1RemSet.cpp Mon May 18 11:52:46 2009 -0700 +++ b/src/share/vm/gc_implementation/g1/g1RemSet.cpp Tue May 19 04:05:31 2009 -0700 @@ -219,6 +219,7 @@ HeapRegionRemSet* hrrs = r->rem_set(); if (hrrs->iter_is_complete()) return false; // All done. if (!_try_claimed && !hrrs->claim_iter()) return false; + _g1h->push_dirty_cards_region(r); // If we didn't return above, then // _try_claimed || r->claim_iter() // is true: either we're supposed to work on claimed-but-not-complete @@ -242,6 +243,10 @@ assert(card_region != NULL, "Yielding cards not in the heap?"); _cards++; + if (!card_region->is_on_dirty_cards_region_list()) { + _g1h->push_dirty_cards_region(card_region); + } + // If the card is dirty, then we will scan it during updateRS. if (!card_region->in_collection_set() && !_ct_bs->is_card_dirty(card_index)) { if (!_ct_bs->is_card_claimed(card_index) && _ct_bs->claim_card(card_index)) { diff -r 215f81b4d9b3 -r 29e7d79232b9 src/share/vm/gc_implementation/g1/heapRegion.cpp --- a/src/share/vm/gc_implementation/g1/heapRegion.cpp Mon May 18 11:52:46 2009 -0700 +++ b/src/share/vm/gc_implementation/g1/heapRegion.cpp Tue May 19 04:05:31 2009 -0700 @@ -351,6 +351,7 @@ _claimed(InitialClaimValue), _evacuation_failed(false), _prev_marked_bytes(0), _next_marked_bytes(0), _sort_index(-1), _young_type(NotYoung), _next_young_region(NULL), + _next_dirty_cards_region(NULL), _young_index_in_cset(-1), _surv_rate_group(NULL), _age_index(-1), _rem_set(NULL), _zfs(NotZeroFilled) { diff -r 215f81b4d9b3 -r 29e7d79232b9 src/share/vm/gc_implementation/g1/heapRegion.hpp --- a/src/share/vm/gc_implementation/g1/heapRegion.hpp Mon May 18 11:52:46 2009 -0700 +++ b/src/share/vm/gc_implementation/g1/heapRegion.hpp Tue May 19 04:05:31 2009 -0700 @@ -227,6 +227,9 @@ // next region in the young "generation" region set HeapRegion* _next_young_region; + // Next region whose cards need cleaning + HeapRegion* _next_dirty_cards_region; + // For parallel heapRegion traversal. jint _claimed; @@ -468,6 +471,11 @@ _next_young_region = hr; } + HeapRegion* get_next_dirty_cards_region() const { return _next_dirty_cards_region; } + HeapRegion** next_dirty_cards_region_addr() { return &_next_dirty_cards_region; } + void set_next_dirty_cards_region(HeapRegion* hr) { _next_dirty_cards_region = hr; } + bool is_on_dirty_cards_region_list() const { return get_next_dirty_cards_region() != NULL; } + // Allows logical separation between objects allocated before and after. void save_marks();