# HG changeset patch # User apetrusenko # Date 1251721649 25200 # Node ID 8624da129f0b2e19bd87c7726285e02551a6b9dc # Parent 9eebd3ac74cfb3867d5cb5a32bb735e1ddf35866 6841313: G1: dirty cards of survivor regions in parallel Reviewed-by: tonyp, iveresov diff -r 9eebd3ac74cf -r 8624da129f0b src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Thu Aug 13 16:22:45 2009 -0700 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Mon Aug 31 05:27:29 2009 -0700 @@ -2739,8 +2739,6 @@ _in_cset_fast_test = NULL; _in_cset_fast_test_base = NULL; - release_gc_alloc_regions(false /* totally */); - cleanup_surviving_young_words(); if (g1_policy()->in_young_gc_mode()) { @@ -4132,6 +4130,7 @@ G1KeepAliveClosure keep_alive(this); JNIHandles::weak_oops_do(&is_alive, &keep_alive); } + release_gc_alloc_regions(false /* totally */); g1_rem_set()->cleanup_after_oops_into_collection_set_do(); concurrent_g1_refine()->clear_hot_cache(); @@ -4265,12 +4264,18 @@ class G1ParCleanupCTTask : public AbstractGangTask { CardTableModRefBS* _ct_bs; G1CollectedHeap* _g1h; + HeapRegion* volatile _so_head; + HeapRegion* volatile _su_head; public: G1ParCleanupCTTask(CardTableModRefBS* ct_bs, - G1CollectedHeap* g1h) : + G1CollectedHeap* g1h, + HeapRegion* scan_only_list, + HeapRegion* survivor_list) : AbstractGangTask("G1 Par Cleanup CT Task"), _ct_bs(ct_bs), - _g1h(g1h) + _g1h(g1h), + _so_head(scan_only_list), + _su_head(survivor_list) { } void work(int i) { @@ -4278,22 +4283,64 @@ while (r = _g1h->pop_dirty_cards_region()) { clear_cards(r); } - } + // Redirty the cards of the scan-only and survivor regions. + dirty_list(&this->_so_head); + dirty_list(&this->_su_head); + } + void clear_cards(HeapRegion* r) { // Cards for Survivor and Scan-Only regions will be dirtied later. if (!r->is_scan_only() && !r->is_survivor()) { _ct_bs->clear(MemRegion(r->bottom(), r->end())); } } + + void dirty_list(HeapRegion* volatile * head_ptr) { + HeapRegion* head; + do { + // Pop region off the list. + head = *head_ptr; + if (head != NULL) { + HeapRegion* r = (HeapRegion*) + Atomic::cmpxchg_ptr(head->get_next_young_region(), head_ptr, head); + if (r == head) { + assert(!r->isHumongous(), "Humongous regions shouldn't be on survivor list"); + _ct_bs->dirty(MemRegion(r->bottom(), r->end())); + } + } + } while (*head_ptr != NULL); + } }; +#ifndef PRODUCT +class G1VerifyCardTableCleanup: public HeapRegionClosure { + CardTableModRefBS* _ct_bs; +public: + G1VerifyCardTableCleanup(CardTableModRefBS* ct_bs) + : _ct_bs(ct_bs) + { } + virtual bool doHeapRegion(HeapRegion* r) + { + MemRegion mr(r->bottom(), r->end()); + if (r->is_scan_only() || r->is_survivor()) { + _ct_bs->verify_dirty_region(mr); + } else { + _ct_bs->verify_clean_region(mr); + } + return false; + } +}; +#endif + void G1CollectedHeap::cleanUpCardTable() { CardTableModRefBS* ct_bs = (CardTableModRefBS*) (barrier_set()); double start = os::elapsedTime(); // Iterate over the dirty cards region list. - G1ParCleanupCTTask cleanup_task(ct_bs, this); + G1ParCleanupCTTask cleanup_task(ct_bs, this, + _young_list->first_scan_only_region(), + _young_list->first_survivor_region()); if (ParallelGCThreads > 0) { set_par_threads(workers()->total_workers()); workers()->run_task(&cleanup_task); @@ -4309,18 +4356,22 @@ } r->set_next_dirty_cards_region(NULL); } - } - // now, redirty the cards of the scan-only and survivor regions - // (it seemed faster to do it this way, instead of iterating over - // all regions and then clearing / dirtying as appropriate) - dirtyCardsForYoungRegions(ct_bs, _young_list->first_scan_only_region()); - dirtyCardsForYoungRegions(ct_bs, _young_list->first_survivor_region()); - + // now, redirty the cards of the scan-only and survivor regions + // (it seemed faster to do it this way, instead of iterating over + // all regions and then clearing / dirtying as appropriate) + dirtyCardsForYoungRegions(ct_bs, _young_list->first_scan_only_region()); + dirtyCardsForYoungRegions(ct_bs, _young_list->first_survivor_region()); + } double elapsed = os::elapsedTime() - start; g1_policy()->record_clear_ct_time( elapsed * 1000.0); +#ifndef PRODUCT + if (G1VerifyCTCleanup || VerifyAfterGC) { + G1VerifyCardTableCleanup cleanup_verifier(ct_bs); + heap_region_iterate(&cleanup_verifier); + } +#endif } - void G1CollectedHeap::do_collection_pause_if_appropriate(size_t word_size) { if (g1_policy()->should_do_collection_pause(word_size)) { do_collection_pause(); diff -r 9eebd3ac74cf -r 8624da129f0b src/share/vm/gc_implementation/g1/g1_globals.hpp --- a/src/share/vm/gc_implementation/g1/g1_globals.hpp Thu Aug 13 16:22:45 2009 -0700 +++ b/src/share/vm/gc_implementation/g1/g1_globals.hpp Mon Aug 31 05:27:29 2009 -0700 @@ -260,6 +260,9 @@ \ develop(intx, G1CardCountCacheExpandThreshold, 16, \ "Expand the card count cache if the number of collisions for " \ - "a particular entry exceeds this value.") + "a particular entry exceeds this value.") \ + \ + develop(bool, G1VerifyCTCleanup, false, \ + "Verify card table cleanup.") G1_FLAGS(DECLARE_DEVELOPER_FLAG, DECLARE_PD_DEVELOPER_FLAG, DECLARE_PRODUCT_FLAG, DECLARE_PD_PRODUCT_FLAG, DECLARE_DIAGNOSTIC_FLAG, DECLARE_EXPERIMENTAL_FLAG, DECLARE_NOTPRODUCT_FLAG, DECLARE_MANAGEABLE_FLAG, DECLARE_PRODUCT_RW_FLAG) diff -r 9eebd3ac74cf -r 8624da129f0b src/share/vm/memory/cardTableModRefBS.cpp --- a/src/share/vm/memory/cardTableModRefBS.cpp Thu Aug 13 16:22:45 2009 -0700 +++ b/src/share/vm/memory/cardTableModRefBS.cpp Mon Aug 31 05:27:29 2009 -0700 @@ -660,6 +660,29 @@ GuaranteeNotModClosure blk(this); non_clean_card_iterate_work(mr, &blk, false); } + +// To verify a MemRegion is entirely dirty this closure is passed to +// dirty_card_iterate. If the region is dirty do_MemRegion will be +// invoked only once with a MemRegion equal to the one being +// verified. +class GuaranteeDirtyClosure: public MemRegionClosure { + CardTableModRefBS* _ct; + MemRegion _mr; + bool _result; +public: + GuaranteeDirtyClosure(CardTableModRefBS* ct, MemRegion mr) + : _ct(ct), _mr(mr), _result(false) {} + void do_MemRegion(MemRegion mr) { + _result = _mr.equals(mr); + } + bool result() const { return _result; } +}; + +void CardTableModRefBS::verify_dirty_region(MemRegion mr) { + GuaranteeDirtyClosure blk(this, mr); + dirty_card_iterate(mr, &blk); + guarantee(blk.result(), "Non-dirty cards in region that should be dirty"); +} #endif bool CardTableModRefBSForCTRS::card_will_be_scanned(jbyte cv) { diff -r 9eebd3ac74cf -r 8624da129f0b src/share/vm/memory/cardTableModRefBS.hpp --- a/src/share/vm/memory/cardTableModRefBS.hpp Thu Aug 13 16:22:45 2009 -0700 +++ b/src/share/vm/memory/cardTableModRefBS.hpp Mon Aug 31 05:27:29 2009 -0700 @@ -456,6 +456,7 @@ void verify_guard(); void verify_clean_region(MemRegion mr) PRODUCT_RETURN; + void verify_dirty_region(MemRegion mr) PRODUCT_RETURN; static size_t par_chunk_heapword_alignment() { return CardsPerStrideChunk * card_size_in_words;