# HG changeset patch # User johnc # Date 1324667658 28800 # Node ID 023652e49ac0bce237cf0e40b8adbc8c925f7af3 # Parent 5fd354a959c5e431e14c68632fecf3c906b4c09c 7121496: G1: do the per-region evacuation failure handling work in parallel Summary: Parallelize the removal of self forwarding pointers etc. by wrapping in a HeapRegion closure, which is then wrapped inside an AbstractGangTask. Reviewed-by: tonyp, iveresov diff -r 5fd354a959c5 -r 023652e49ac0 src/share/vm/gc_implementation/g1/concurrentMark.cpp --- a/src/share/vm/gc_implementation/g1/concurrentMark.cpp Thu Jan 05 21:21:55 2012 -0800 +++ b/src/share/vm/gc_implementation/g1/concurrentMark.cpp Fri Dec 23 11:14:18 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -3080,19 +3080,6 @@ } }; -class SetClaimValuesInCSetHRClosure: public HeapRegionClosure { - jint _claim_value; - -public: - SetClaimValuesInCSetHRClosure(jint claim_value) : - _claim_value(claim_value) { } - - bool doHeapRegion(HeapRegion* hr) { - hr->set_claim_value(_claim_value); - return false; - } -}; - class G1ParCompleteMarkInCSetTask: public AbstractGangTask { protected: G1CollectedHeap* _g1h; @@ -3135,9 +3122,8 @@ assert(g1h->check_cset_heap_region_claim_values(HeapRegion::CompleteMarkCSetClaimValue), "sanity"); - // Now reset the claim values in the regions in the collection set. - SetClaimValuesInCSetHRClosure set_cv_cl(HeapRegion::InitialClaimValue); - g1h->collection_set_iterate(&set_cv_cl); + // Reset the claim values in the regions in the collection set. + g1h->reset_cset_heap_region_claim_values(); assert(g1h->check_cset_heap_region_claim_values(HeapRegion::InitialClaimValue), "sanity"); diff -r 5fd354a959c5 -r 023652e49ac0 src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Thu Jan 05 21:21:55 2012 -0800 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Fri Dec 23 11:14:18 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -32,6 +32,7 @@ #include "gc_implementation/g1/g1CollectedHeap.inline.hpp" #include "gc_implementation/g1/g1CollectorPolicy.hpp" #include "gc_implementation/g1/g1ErgoVerbose.hpp" +#include "gc_implementation/g1/g1EvacFailure.hpp" #include "gc_implementation/g1/g1MarkSweep.hpp" #include "gc_implementation/g1/g1OopClosures.inline.hpp" #include "gc_implementation/g1/g1RemSet.inline.hpp" @@ -2618,12 +2619,16 @@ } }; -void -G1CollectedHeap::reset_heap_region_claim_values() { +void G1CollectedHeap::reset_heap_region_claim_values() { ResetClaimValuesClosure blk; heap_region_iterate(&blk); } +void G1CollectedHeap::reset_cset_heap_region_claim_values() { + ResetClaimValuesClosure blk; + collection_set_iterate(&blk); +} + #ifdef ASSERT // This checks whether all regions in the heap have the correct claim // value. I also piggy-backed on this a check to ensure that the @@ -3998,157 +4003,26 @@ _evac_failure_scan_stack = NULL; } -class UpdateRSetDeferred : public OopsInHeapRegionClosure { -private: - G1CollectedHeap* _g1; - DirtyCardQueue *_dcq; - CardTableModRefBS* _ct_bs; - -public: - UpdateRSetDeferred(G1CollectedHeap* g1, DirtyCardQueue* dcq) : - _g1(g1), _ct_bs((CardTableModRefBS*)_g1->barrier_set()), _dcq(dcq) {} - - virtual void do_oop(narrowOop* p) { do_oop_work(p); } - virtual void do_oop( oop* p) { do_oop_work(p); } - template void do_oop_work(T* p) { - assert(_from->is_in_reserved(p), "paranoia"); - if (!_from->is_in_reserved(oopDesc::load_decode_heap_oop(p)) && - !_from->is_survivor()) { - size_t card_index = _ct_bs->index_for(p); - if (_ct_bs->mark_card_deferred(card_index)) { - _dcq->enqueue((jbyte*)_ct_bs->byte_for_index(card_index)); - } - } - } -}; - -class RemoveSelfPointerClosure: public ObjectClosure { -private: - G1CollectedHeap* _g1; - ConcurrentMark* _cm; - HeapRegion* _hr; - size_t _prev_marked_bytes; - size_t _next_marked_bytes; - OopsInHeapRegionClosure *_cl; -public: - RemoveSelfPointerClosure(G1CollectedHeap* g1, HeapRegion* hr, - OopsInHeapRegionClosure* cl) : - _g1(g1), _hr(hr), _cm(_g1->concurrent_mark()), _prev_marked_bytes(0), - _next_marked_bytes(0), _cl(cl) {} - - size_t prev_marked_bytes() { return _prev_marked_bytes; } - size_t next_marked_bytes() { return _next_marked_bytes; } - - // - // The original idea here was to coalesce evacuated and dead objects. - // However that caused complications with the block offset table (BOT). - // In particular if there were two TLABs, one of them partially refined. - // |----- TLAB_1--------|----TLAB_2-~~~(partially refined part)~~~| - // The BOT entries of the unrefined part of TLAB_2 point to the start - // of TLAB_2. If the last object of the TLAB_1 and the first object - // of TLAB_2 are coalesced, then the cards of the unrefined part - // would point into middle of the filler object. - // The current approach is to not coalesce and leave the BOT contents intact. - // - // - // We now reset the BOT when we start the object iteration over the - // region and refine its entries for every object we come across. So - // the above comment is not really relevant and we should be able - // to coalesce dead objects if we want to. - void do_object(oop obj) { - HeapWord* obj_addr = (HeapWord*) obj; - assert(_hr->is_in(obj_addr), "sanity"); - size_t obj_size = obj->size(); - _hr->update_bot_for_object(obj_addr, obj_size); - if (obj->is_forwarded() && obj->forwardee() == obj) { - // The object failed to move. - assert(!_g1->is_obj_dead(obj), "We should not be preserving dead objs."); - _cm->markPrev(obj); - assert(_cm->isPrevMarked(obj), "Should be marked!"); - _prev_marked_bytes += (obj_size * HeapWordSize); - if (_g1->mark_in_progress() && !_g1->is_obj_ill(obj)) { - _cm->markAndGrayObjectIfNecessary(obj); - } - obj->set_mark(markOopDesc::prototype()); - // While we were processing RSet buffers during the - // collection, we actually didn't scan any cards on the - // collection set, since we didn't want to update remebered - // sets with entries that point into the collection set, given - // that live objects fromthe collection set are about to move - // and such entries will be stale very soon. This change also - // dealt with a reliability issue which involved scanning a - // card in the collection set and coming across an array that - // was being chunked and looking malformed. The problem is - // that, if evacuation fails, we might have remembered set - // entries missing given that we skipped cards on the - // collection set. So, we'll recreate such entries now. - obj->oop_iterate(_cl); - assert(_cm->isPrevMarked(obj), "Should be marked!"); - } else { - // The object has been either evacuated or is dead. Fill it with a - // dummy object. - MemRegion mr((HeapWord*)obj, obj_size); - CollectedHeap::fill_with_object(mr); - _cm->clearRangeBothMaps(mr); - } - } -}; - void G1CollectedHeap::remove_self_forwarding_pointers() { - UpdateRSetImmediate immediate_update(_g1h->g1_rem_set()); - DirtyCardQueue dcq(&_g1h->dirty_card_queue_set()); - UpdateRSetDeferred deferred_update(_g1h, &dcq); - OopsInHeapRegionClosure *cl; - if (G1DeferredRSUpdate) { - cl = &deferred_update; + assert(check_cset_heap_region_claim_values(HeapRegion::InitialClaimValue), "sanity"); + assert(g1_policy()->assertMarkedBytesDataOK(), "Should be!"); + + G1ParRemoveSelfForwardPtrsTask rsfp_task(this); + + if (G1CollectedHeap::use_parallel_gc_threads()) { + set_par_threads(); + workers()->run_task(&rsfp_task); + set_par_threads(0); } else { - cl = &immediate_update; - } - HeapRegion* cur = g1_policy()->collection_set(); - while (cur != NULL) { - assert(g1_policy()->assertMarkedBytesDataOK(), "Should be!"); - assert(!cur->isHumongous(), "sanity"); - - if (cur->evacuation_failed()) { - assert(cur->in_collection_set(), "bad CS"); - RemoveSelfPointerClosure rspc(_g1h, cur, cl); - - // In the common case we make sure that this is done when the - // region is freed so that it is "ready-to-go" when it's - // re-allocated. However, when evacuation failure happens, a - // region will remain in the heap and might ultimately be added - // to a CSet in the future. So we have to be careful here and - // make sure the region's RSet is ready for parallel iteration - // whenever this might be required in the future. - cur->rem_set()->reset_for_par_iteration(); - cur->reset_bot(); - cl->set_region(cur); - cur->object_iterate(&rspc); - - // A number of manipulations to make the TAMS be the current top, - // and the marked bytes be the ones observed in the iteration. - if (_g1h->concurrent_mark()->at_least_one_mark_complete()) { - // The comments below are the postconditions achieved by the - // calls. Note especially the last such condition, which says that - // the count of marked bytes has been properly restored. - cur->note_start_of_marking(false); - // _next_top_at_mark_start == top, _next_marked_bytes == 0 - cur->add_to_marked_bytes(rspc.prev_marked_bytes()); - // _next_marked_bytes == prev_marked_bytes. - cur->note_end_of_marking(); - // _prev_top_at_mark_start == top(), - // _prev_marked_bytes == prev_marked_bytes - } - // If there is no mark in progress, we modified the _next variables - // above needlessly, but harmlessly. - if (_g1h->mark_in_progress()) { - cur->note_start_of_marking(false); - // _next_top_at_mark_start == top, _next_marked_bytes == 0 - // _next_marked_bytes == next_marked_bytes. - } - } - cur = cur->next_in_collection_set(); - } + rsfp_task.work(0); + } + + assert(check_cset_heap_region_claim_values(HeapRegion::ParEvacFailureClaimValue), "sanity"); + + // Reset the claim values in the regions in the collection set. + reset_cset_heap_region_claim_values(); + + assert(check_cset_heap_region_claim_values(HeapRegion::InitialClaimValue), "sanity"); assert(g1_policy()->assertMarkedBytesDataOK(), "Should be!"); // Now restore saved marks, if any. @@ -4161,6 +4035,7 @@ markOop m = _preserved_marks_of_objs->at(i); obj->set_mark(m); } + // Delete the preserved marks growable arrays (allocated on the C heap). delete _objs_with_preserved_marks; delete _preserved_marks_of_objs; diff -r 5fd354a959c5 -r 023652e49ac0 src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp Thu Jan 05 21:21:55 2012 -0800 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp Fri Dec 23 11:14:18 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -1313,6 +1313,10 @@ // It resets all the region claim values to the default. void reset_heap_region_claim_values(); + // Resets the claim values of regions in the current + // collection set to the default. + void reset_cset_heap_region_claim_values(); + #ifdef ASSERT bool check_heap_region_claim_values(jint claim_value); diff -r 5fd354a959c5 -r 023652e49ac0 src/share/vm/gc_implementation/g1/g1EvacFailure.hpp --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/src/share/vm/gc_implementation/g1/g1EvacFailure.hpp Fri Dec 23 11:14:18 2011 -0800 @@ -0,0 +1,228 @@ +/* + * Copyright (c) 2012, Oracle and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA + * or visit www.oracle.com if you need additional information or have any + * questions. + * + */ + +#ifndef SHARE_VM_GC_IMPLEMENTATION_G1_G1EVACFAILURE_HPP +#define SHARE_VM_GC_IMPLEMENTATION_G1_G1EVACFAILURE_HPP + +#include "gc_implementation/g1/concurrentMark.inline.hpp" +#include "gc_implementation/g1/dirtyCardQueue.hpp" +#include "gc_implementation/g1/g1CollectedHeap.inline.hpp" +#include "gc_implementation/g1/g1_globals.hpp" +#include "gc_implementation/g1/g1OopClosures.inline.hpp" +#include "gc_implementation/g1/heapRegion.hpp" +#include "gc_implementation/g1/heapRegionRemSet.hpp" +#include "utilities/workgroup.hpp" + +// Closures and tasks associated with any self-forwarding pointers +// installed as a result of an evacuation failure. + +class UpdateRSetDeferred : public OopsInHeapRegionClosure { +private: + G1CollectedHeap* _g1; + DirtyCardQueue *_dcq; + CardTableModRefBS* _ct_bs; + +public: + UpdateRSetDeferred(G1CollectedHeap* g1, DirtyCardQueue* dcq) : + _g1(g1), _ct_bs((CardTableModRefBS*)_g1->barrier_set()), _dcq(dcq) {} + + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop( oop* p) { do_oop_work(p); } + template void do_oop_work(T* p) { + assert(_from->is_in_reserved(p), "paranoia"); + if (!_from->is_in_reserved(oopDesc::load_decode_heap_oop(p)) && + !_from->is_survivor()) { + size_t card_index = _ct_bs->index_for(p); + if (_ct_bs->mark_card_deferred(card_index)) { + _dcq->enqueue((jbyte*)_ct_bs->byte_for_index(card_index)); + } + } + } +}; + +class RemoveSelfForwardPtrObjClosure: public ObjectClosure { +private: + G1CollectedHeap* _g1; + ConcurrentMark* _cm; + HeapRegion* _hr; + size_t _prev_marked_bytes; + size_t _next_marked_bytes; + OopsInHeapRegionClosure *_update_rset_cl; +public: + RemoveSelfForwardPtrObjClosure(G1CollectedHeap* g1, ConcurrentMark* cm, + HeapRegion* hr, + OopsInHeapRegionClosure* update_rset_cl) : + _g1(g1), _cm(cm), _hr(hr), + _update_rset_cl(update_rset_cl), + _prev_marked_bytes(0), _next_marked_bytes(0) {} + + size_t prev_marked_bytes() { return _prev_marked_bytes; } + size_t next_marked_bytes() { return _next_marked_bytes; } + + // + // The original idea here was to coalesce evacuated and dead objects. + // However that caused complications with the block offset table (BOT). + // In particular if there were two TLABs, one of them partially refined. + // |----- TLAB_1--------|----TLAB_2-~~~(partially refined part)~~~| + // The BOT entries of the unrefined part of TLAB_2 point to the start + // of TLAB_2. If the last object of the TLAB_1 and the first object + // of TLAB_2 are coalesced, then the cards of the unrefined part + // would point into middle of the filler object. + // The current approach is to not coalesce and leave the BOT contents intact. + // + // + // We now reset the BOT when we start the object iteration over the + // region and refine its entries for every object we come across. So + // the above comment is not really relevant and we should be able + // to coalesce dead objects if we want to. + void do_object(oop obj) { + HeapWord* obj_addr = (HeapWord*) obj; + assert(_hr->is_in(obj_addr), "sanity"); + size_t obj_size = obj->size(); + + _hr->update_bot_for_object(obj_addr, obj_size); + + if (obj->is_forwarded() && obj->forwardee() == obj) { + // The object failed to move. + assert(!_g1->is_obj_dead(obj), "We should not be preserving dead objs."); + _cm->markPrev(obj); + assert(_cm->isPrevMarked(obj), "Should be marked!"); + _prev_marked_bytes += (obj_size * HeapWordSize); + if (_g1->mark_in_progress() && !_g1->is_obj_ill(obj)) { + _cm->markAndGrayObjectIfNecessary(obj); + } + obj->set_mark(markOopDesc::prototype()); + + // While we were processing RSet buffers during the collection, + // we actually didn't scan any cards on the collection set, + // since we didn't want to update remembered sets with entries + // that point into the collection set, given that live objects + // from the collection set are about to move and such entries + // will be stale very soon. + // This change also dealt with a reliability issue which + // involved scanning a card in the collection set and coming + // across an array that was being chunked and looking malformed. + // The problem is that, if evacuation fails, we might have + // remembered set entries missing given that we skipped cards on + // the collection set. So, we'll recreate such entries now. + + obj->oop_iterate(_update_rset_cl); + assert(_cm->isPrevMarked(obj), "Should be marked!"); + } else { + // The object has been either evacuated or is dead. Fill it with a + // dummy object. + MemRegion mr((HeapWord*)obj, obj_size); + CollectedHeap::fill_with_object(mr); + _cm->clearRangeBothMaps(mr); + } + } +}; + +class RemoveSelfForwardPtrHRClosure: public HeapRegionClosure { + G1CollectedHeap* _g1h; + ConcurrentMark* _cm; + OopsInHeapRegionClosure *_update_rset_cl; + +public: + RemoveSelfForwardPtrHRClosure(G1CollectedHeap* g1h, + OopsInHeapRegionClosure* update_rset_cl) : + _g1h(g1h), _update_rset_cl(update_rset_cl), + _cm(_g1h->concurrent_mark()) { } + + bool doHeapRegion(HeapRegion *hr) { + assert(!hr->isHumongous(), "sanity"); + assert(hr->in_collection_set(), "bad CS"); + + if (hr->claimHeapRegion(HeapRegion::ParEvacFailureClaimValue)) { + if (hr->evacuation_failed()) { + RemoveSelfForwardPtrObjClosure rspc(_g1h, _cm, hr, _update_rset_cl); + + // In the common case (i.e. when there is no evacuation + // failure) we make sure that the following is done when + // the region is freed so that it is "ready-to-go" when it's + // re-allocated. However, when evacuation failure happens, a + // region will remain in the heap and might ultimately be added + // to a CSet in the future. So we have to be careful here and + // make sure the region's RSet is ready for parallel iteration + // whenever this might be required in the future. + hr->rem_set()->reset_for_par_iteration(); + hr->reset_bot(); + _update_rset_cl->set_region(hr); + hr->object_iterate(&rspc); + + // A number of manipulations to make the TAMS for this region + // be the current top, and the marked bytes be the ones observed + // in the iteration. + if (_cm->at_least_one_mark_complete()) { + // The comments below are the postconditions achieved by the + // calls. Note especially the last such condition, which says that + // the count of marked bytes has been properly restored. + hr->note_start_of_marking(false); + // _next_top_at_mark_start == top, _next_marked_bytes == 0 + hr->add_to_marked_bytes(rspc.prev_marked_bytes()); + // _next_marked_bytes == prev_marked_bytes. + hr->note_end_of_marking(); + // _prev_top_at_mark_start == top(), + // _prev_marked_bytes == prev_marked_bytes + } + // If there is no mark in progress, we modified the _next variables + // above needlessly, but harmlessly. + if (_g1h->mark_in_progress()) { + hr->note_start_of_marking(false); + // _next_top_at_mark_start == top, _next_marked_bytes == 0 + // _next_marked_bytes == next_marked_bytes. + } + } + } + return false; + } +}; + +class G1ParRemoveSelfForwardPtrsTask: public AbstractGangTask { +protected: + G1CollectedHeap* _g1h; + +public: + G1ParRemoveSelfForwardPtrsTask(G1CollectedHeap* g1h) : + AbstractGangTask("G1 Remove Self-forwarding Pointers"), + _g1h(g1h) { } + + void work(uint worker_id) { + UpdateRSetImmediate immediate_update(_g1h->g1_rem_set()); + DirtyCardQueue dcq(&_g1h->dirty_card_queue_set()); + UpdateRSetDeferred deferred_update(_g1h, &dcq); + + OopsInHeapRegionClosure *update_rset_cl = &deferred_update; + if (!G1DeferredRSUpdate) { + update_rset_cl = &immediate_update; + } + + RemoveSelfForwardPtrHRClosure rsfp_cl(_g1h, update_rset_cl); + + HeapRegion* hr = _g1h->start_cset_region_for_worker(worker_id); + _g1h->collection_set_iterate_from(hr, &rsfp_cl); + } +}; + +#endif // SHARE_VM_GC_IMPLEMENTATION_G1_G1EVACFAILURE_HPP diff -r 5fd354a959c5 -r 023652e49ac0 src/share/vm/gc_implementation/g1/heapRegion.hpp --- a/src/share/vm/gc_implementation/g1/heapRegion.hpp Thu Jan 05 21:21:55 2012 -0800 +++ b/src/share/vm/gc_implementation/g1/heapRegion.hpp Fri Dec 23 11:14:18 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -373,7 +373,8 @@ ScrubRemSetClaimValue = 3, ParVerifyClaimValue = 4, RebuildRSClaimValue = 5, - CompleteMarkCSetClaimValue = 6 + CompleteMarkCSetClaimValue = 6, + ParEvacFailureClaimValue = 7 }; inline HeapWord* par_allocate_no_bot_updates(size_t word_size) {