graal-jvmci-8: src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp comparison

comparison src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp @ 3979:4dfb2df418f2

6484982: G1: process references during evacuation pauses Summary: G1 now uses two reference processors - one is used by concurrent marking and the other is used by STW GCs (both full and incremental evacuation pauses). In an evacuation pause, the reference processor is embedded into the closures used to scan objects. Doing so causes causes reference objects to be 'discovered' by the reference processor. At the end of the evacuation pause, these discovered reference objects are processed - preserving (and copying) referent objects (and their reachable graphs) as appropriate. Reviewed-by: ysr, jwilhelm, brutisso, stefank, tonyp

author	johnc
date	Thu, 22 Sep 2011 10:57:37 -0700
parents	663cb89032b1
children	8229bd737950

comparison

equal deleted inserted replaced

-:f0ecbe78fc7b
+:4dfb2df418f2
 #include "gc_implementation/g1/vm_operations_g1.hpp"
 #include "gc_implementation/shared/isGCActiveMark.hpp"
 #include "memory/gcLocker.inline.hpp"
 #include "memory/genOopClosures.inline.hpp"
 #include "memory/generationSpec.hpp"
+#include "memory/referenceProcessor.hpp"
 #include "oops/oop.inline.hpp"
 #include "oops/oop.pcgc.inline.hpp"
 #include "runtime/aprofiler.hpp"
 #include "runtime/vmThread.hpp"
 }
 pre_full_gc_dump();
 COMPILER2_PRESENT(DerivedPointerTable::clear());
-// We want to discover references, but not process them yet.
+// Disable discovery and empty the discovered lists
-// This mode is disabled in
+// for the CM ref processor.
-// instanceRefKlass::process_discovered_references if the
+ref_processor_cm()->disable_discovery();
-// generation does some collection work, or
+ref_processor_cm()->abandon_partial_discovery();
-// instanceRefKlass::enqueue_discovered_references if the
+ref_processor_cm()->verify_no_references_recorded();
-// generation returns without doing any work.
-ref_processor()->disable_discovery();
-ref_processor()->abandon_partial_discovery();
-ref_processor()->verify_no_references_recorded();
 // Abandon current iterations of concurrent marking and concurrent
 // refinement, if any are in progress.
 concurrent_mark()->abort();
 g1_policy()->stop_incremental_cset_building();
 empty_young_list();
 g1_policy()->set_full_young_gcs(true);
-// See the comment in G1CollectedHeap::ref_processing_init() about
+// See the comments in g1CollectedHeap.hpp and
+// G1CollectedHeap::ref_processing_init() about
 // how reference processing currently works in G1.
-// Temporarily make reference _discovery_ single threaded (non-MT).
+// Temporarily make discovery by the STW ref processor single threaded (non-MT).
-ReferenceProcessorMTDiscoveryMutator rp_disc_ser(ref_processor(), false);
+ReferenceProcessorMTDiscoveryMutator stw_rp_disc_ser(ref_processor_stw(), false);
-// Temporarily make refs discovery atomic
+// Temporarily clear the STW ref processor's _is_alive_non_header field.
-ReferenceProcessorAtomicMutator rp_disc_atomic(ref_processor(), true);
+ReferenceProcessorIsAliveMutator stw_rp_is_alive_null(ref_processor_stw(), NULL);
-// Temporarily clear _is_alive_non_header
+ref_processor_stw()->enable_discovery(true /*verify_disabled*/, true /*verify_no_refs*/);
-ReferenceProcessorIsAliveMutator rp_is_alive_null(ref_processor(), NULL);
+ref_processor_stw()->setup_policy(do_clear_all_soft_refs);
-ref_processor()->enable_discovery();
-ref_processor()->setup_policy(do_clear_all_soft_refs);
 // Do collection work
 {
 HandleMark hm;  // Discard invalid handles created during gc
-G1MarkSweep::invoke_at_safepoint(ref_processor(), do_clear_all_soft_refs);
+G1MarkSweep::invoke_at_safepoint(ref_processor_stw(), do_clear_all_soft_refs);
 }
 assert(free_regions() == 0, "we should not have added any free regions");
 rebuild_region_lists();
 _summary_bytes_used = recalculate_used();
-ref_processor()->enqueue_discovered_references();
+// Enqueue any discovered reference objects that have
+// not been removed from the discovered lists.
+ref_processor_stw()->enqueue_discovered_references();
 COMPILER2_PRESENT(DerivedPointerTable::update_pointers());
 MemoryService::track_memory_usage();
 Universe::verify(/* allow dirty */ false,
 /* silent      */ false,
 /* option      */ VerifyOption_G1UsePrevMarking);
 }
-NOT_PRODUCT(ref_processor()->verify_no_references_recorded());
+assert(!ref_processor_stw()->discovery_enabled(), "Postcondition");
+ref_processor_stw()->verify_no_references_recorded();
+// Note: since we've just done a full GC, concurrent
+// marking is no longer active. Therefore we need not
+// re-enable reference discovery for the CM ref processor.
+// That will be done at the start of the next marking cycle.
+assert(!ref_processor_cm()->discovery_enabled(), "Postcondition");
+ref_processor_cm()->verify_no_references_recorded();
 reset_gc_time_stamp();
 // Since everything potentially moved, we will clear all remembered
 // sets, and clear all cards.  Later we will rebuild remebered
 // sets. We will also reset the GC time stamps of the regions.
 G1CollectedHeap::G1CollectedHeap(G1CollectorPolicy* policy_) :
 SharedHeap(policy_),
 _g1_policy(policy_),
 _dirty_card_queue_set(false),
 _into_cset_dirty_card_queue_set(false),
-_is_alive_closure(this),
+_is_alive_closure_cm(this),
-_ref_processor(NULL),
+_is_alive_closure_stw(this),
+_ref_processor_cm(NULL),
+_ref_processor_stw(NULL),
 _process_strong_tasks(new SubTasksDone(G1H_PS_NumElements)),
 _bot_shared(NULL),
 _objs_with_preserved_marks(NULL), _preserved_marks_of_objs(NULL),
 _evac_failure_scan_stack(NULL) ,
 _mark_in_progress(false),
 }
 void G1CollectedHeap::ref_processing_init() {
 // Reference processing in G1 currently works as follows:
 //
-// * There is only one reference processor instance that
+// * There are two reference processor instances. One is
-//   'spans' the entire heap. It is created by the code
+//   used to record and process discovered references
-//   below.
+//   during concurrent marking; the other is used to
-// * Reference discovery is not enabled during an incremental
+//   record and process references during STW pauses
-//   pause (see 6484982).
+//   (both full and incremental).
-// * Discoverered refs are not enqueued nor are they processed
+// * Both ref processors need to 'span' the entire heap as
-//   during an incremental pause (see 6484982).
+//   the regions in the collection set may be dotted around.
-// * Reference discovery is enabled at initial marking.
+//
-// * Reference discovery is disabled and the discovered
+// * For the concurrent marking ref processor:
-//   references processed etc during remarking.
+//   * Reference discovery is enabled at initial marking.
-// * Reference discovery is MT (see below).
+//   * Reference discovery is disabled and the discovered
-// * Reference discovery requires a barrier (see below).
+//     references processed etc during remarking.
-// * Reference processing is currently not MT (see 6608385).
+//   * Reference discovery is MT (see below).
-// * A full GC enables (non-MT) reference discovery and
+//   * Reference discovery requires a barrier (see below).
-//   processes any discovered references.
+//   * Reference processing may or may not be MT
+//     (depending on the value of ParallelRefProcEnabled
+//     and ParallelGCThreads).
+//   * A full GC disables reference discovery by the CM
+//     ref processor and abandons any entries on it's
+//     discovered lists.
+//
+// * For the STW processor:
+//   * Non MT discovery is enabled at the start of a full GC.
+//   * Processing and enqueueing during a full GC is non-MT.
+//   * During a full GC, references are processed after marking.
+//
+//   * Discovery (may or may not be MT) is enabled at the start
+//     of an incremental evacuation pause.
+//   * References are processed near the end of a STW evacuation pause.
+//   * For both types of GC:
+//     * Discovery is atomic - i.e. not concurrent.
+//     * Reference discovery will not need a barrier.
 SharedHeap::ref_processing_init();
 MemRegion mr = reserved_region();
-_ref_processor =
+// Concurrent Mark ref processor
+_ref_processor_cm =
 new ReferenceProcessor(mr,    // span
-ParallelRefProcEnabled && (ParallelGCThreads > 1),    // mt processing
+ParallelRefProcEnabled && (ParallelGCThreads > 1),
-(int) ParallelGCThreads,   // degree of mt processing
+// mt processing
-ParallelGCThreads > 1 || ConcGCThreads > 1,  // mt discovery
+(int) ParallelGCThreads,
-(int) MAX2(ParallelGCThreads, ConcGCThreads), // degree of mt discovery
+// degree of mt processing
-false,                     // Reference discovery is not atomic
+(ParallelGCThreads > 1) || (ConcGCThreads > 1),
-&_is_alive_closure,        // is alive closure for efficiency
+// mt discovery
-true);                     // Setting next fields of discovered
+(int) MAX2(ParallelGCThreads, ConcGCThreads),
-// lists requires a barrier.
+// degree of mt discovery
+false,
+// Reference discovery is not atomic
+&_is_alive_closure_cm,
+// is alive closure
+// (for efficiency/performance)
+true);
+// Setting next fields of discovered
+// lists requires a barrier.
+// STW ref processor
+_ref_processor_stw =
+new ReferenceProcessor(mr,    // span
+ParallelRefProcEnabled && (ParallelGCThreads > 1),
+// mt processing
+MAX2((int)ParallelGCThreads, 1),
+// degree of mt processing
+(ParallelGCThreads > 1),
+// mt discovery
+MAX2((int)ParallelGCThreads, 1),
+// degree of mt discovery
+true,
+// Reference discovery is atomic
+&_is_alive_closure_stw,
+// is alive closure
+// (for efficiency/performance)
+false);
+// Setting next fields of discovered
+// lists requires a barrier.
 }
 size_t G1CollectedHeap::capacity() const {
 return _g1_committed.byte_size();
 }
 // I'm ignoring the "fill_newgen()" call if "alloc_event_enabled"
 // is set.
 COMPILER2_PRESENT(assert(DerivedPointerTable::is_empty(),
 "derived pointer present"));
 // always_do_update_barrier = true;
+// We have just completed a GC. Update the soft reference
+// policy with the new heap occupancy
+Universe::update_heap_info_at_gc();
 }
 HeapWord* G1CollectedHeap::do_collection_pause(size_t word_size,
 unsigned int gc_count_before,
 bool* succeeded) {
 }
 COMPILER2_PRESENT(DerivedPointerTable::clear());
-// Please see comment in G1CollectedHeap::ref_processing_init()
+// Please see comment in g1CollectedHeap.hpp and
-// to see how reference processing currently works in G1.
+// G1CollectedHeap::ref_processing_init() to see how
-//
+// reference processing currently works in G1.
-// We want to turn off ref discovery, if necessary, and turn it back on
-// on again later if we do. XXX Dubious: why is discovery disabled?
+// Enable discovery in the STW reference processor
-bool was_enabled = ref_processor()->discovery_enabled();
+ref_processor_stw()->enable_discovery(true /*verify_disabled*/,
-if (was_enabled) ref_processor()->disable_discovery();
+true /*verify_no_refs*/);
-// Forget the current alloc region (we might even choose it to be part
+{
-// of the collection set!).
+// We want to temporarily turn off discovery by the
-release_mutator_alloc_region();
+// CM ref processor, if necessary, and turn it back on
+// on again later if we do. Using a scoped
-// We should call this after we retire the mutator alloc
+// NoRefDiscovery object will do this.
-// region(s) so that all the ALLOC / RETIRE events are generated
+NoRefDiscovery no_cm_discovery(ref_processor_cm());
-// before the start GC event.
-_hr_printer.start_gc(false /* full */, (size_t) total_collections());
+// Forget the current alloc region (we might even choose it to be part
+// of the collection set!).
-// The elapsed time induced by the start time below deliberately elides
+release_mutator_alloc_region();
-// the possible verification above.
-double start_time_sec = os::elapsedTime();
+// We should call this after we retire the mutator alloc
-size_t start_used_bytes = used();
+// region(s) so that all the ALLOC / RETIRE events are generated
+// before the start GC event.
+_hr_printer.start_gc(false /* full */, (size_t) total_collections());
+// The elapsed time induced by the start time below deliberately elides
+// the possible verification above.
+double start_time_sec = os::elapsedTime();
+size_t start_used_bytes = used();
 #if YOUNG_LIST_VERBOSE
 gclog_or_tty->print_cr("\nBefore recording pause start.\nYoung_list:");
 _young_list->print();
 g1_policy()->print_collection_set(g1_policy()->inc_cset_head(), gclog_or_tty);
 #endif // YOUNG_LIST_VERBOSE
 g1_policy()->record_collection_pause_start(start_time_sec,
 start_used_bytes);
 #if YOUNG_LIST_VERBOSE
 gclog_or_tty->print_cr("\nAfter recording pause start.\nYoung_list:");
 _young_list->print();
 #endif // YOUNG_LIST_VERBOSE
 if (g1_policy()->during_initial_mark_pause()) {
 concurrent_mark()->checkpointRootsInitialPre();
 }
 perm_gen()->save_marks();
 // We must do this before any possible evacuation that should propagate
 // marks.
 if (mark_in_progress()) {
 double start_time_sec = os::elapsedTime();
 _cm->drainAllSATBBuffers();
 double finish_mark_ms = (os::elapsedTime() - start_time_sec) * 1000.0;
 g1_policy()->record_satb_drain_time(finish_mark_ms);
 }
 // Record the number of elements currently on the mark stack, so we
 // only iterate over these.  (Since evacuation may add to the mark
 // stack, doing more exposes race conditions.)  If no mark is in
 // progress, this will be zero.
 _cm->set_oops_do_bound();
 if (mark_in_progress()) {
 concurrent_mark()->newCSet();
 }
 #if YOUNG_LIST_VERBOSE
 gclog_or_tty->print_cr("\nBefore choosing collection set.\nYoung_list:");
 _young_list->print();
 g1_policy()->print_collection_set(g1_policy()->inc_cset_head(), gclog_or_tty);
 #endif // YOUNG_LIST_VERBOSE
 g1_policy()->choose_collection_set(target_pause_time_ms);
 if (_hr_printer.is_active()) {
 HeapRegion* hr = g1_policy()->collection_set();
 while (hr != NULL) {
 G1HRPrinter::RegionType type;
 if (!hr->is_young()) {
 type = G1HRPrinter::Old;
 } else if (hr->is_survivor()) {
 type = G1HRPrinter::Survivor;
 } else {
 type = G1HRPrinter::Eden;
 }
 _hr_printer.cset(hr);
 hr = hr->next_in_collection_set();
-}
-}
-// We have chosen the complete collection set. If marking is
-// active then, we clear the region fields of any of the
-// concurrent marking tasks whose region fields point into
-// the collection set as these values will become stale. This
-// will cause the owning marking threads to claim a new region
-// when marking restarts.
-if (mark_in_progress()) {
-concurrent_mark()->reset_active_task_region_fields_in_cset();
-}
-#ifdef ASSERT
-VerifyCSetClosure cl;
-collection_set_iterate(&cl);
-#endif // ASSERT
-setup_surviving_young_words();
-// Initialize the GC alloc regions.
-init_gc_alloc_regions();
-// Actually do the work...
-evacuate_collection_set();
-free_collection_set(g1_policy()->collection_set());
-g1_policy()->clear_collection_set();
-cleanup_surviving_young_words();
-// Start a new incremental collection set for the next pause.
-g1_policy()->start_incremental_cset_building();
-// Clear the _cset_fast_test bitmap in anticipation of adding
-// regions to the incremental collection set for the next
-// evacuation pause.
-clear_cset_fast_test();
-_young_list->reset_sampled_info();
-// Don't check the whole heap at this point as the
-// GC alloc regions from this pause have been tagged
-// as survivors and moved on to the survivor list.
-// Survivor regions will fail the !is_young() check.
-assert(check_young_list_empty(false /* check_heap */),
-"young list should be empty");
-#if YOUNG_LIST_VERBOSE
-gclog_or_tty->print_cr("Before recording survivors.\nYoung List:");
-_young_list->print();
-#endif // YOUNG_LIST_VERBOSE
-g1_policy()->record_survivor_regions(_young_list->survivor_length(),
-_young_list->first_survivor_region(),
-_young_list->last_survivor_region());
-_young_list->reset_auxilary_lists();
-if (evacuation_failed()) {
-_summary_bytes_used = recalculate_used();
-} else {
-// The "used" of the the collection set have already been subtracted
-// when they were freed.  Add in the bytes evacuated.
-_summary_bytes_used += g1_policy()->bytes_copied_during_gc();
-}
-if (g1_policy()->during_initial_mark_pause()) {
-concurrent_mark()->checkpointRootsInitialPost();
-set_marking_started();
-// CAUTION: after the doConcurrentMark() call below,
-// the concurrent marking thread(s) could be running
-// concurrently with us. Make sure that anything after
-// this point does not assume that we are the only GC thread
-// running. Note: of course, the actual marking work will
-// not start until the safepoint itself is released in
-// ConcurrentGCThread::safepoint_desynchronize().
-doConcurrentMark();
-}
-allocate_dummy_regions();
-#if YOUNG_LIST_VERBOSE
-gclog_or_tty->print_cr("\nEnd of the pause.\nYoung_list:");
-_young_list->print();
-g1_policy()->print_collection_set(g1_policy()->inc_cset_head(), gclog_or_tty);
-#endif // YOUNG_LIST_VERBOSE
-init_mutator_alloc_region();
-{
-size_t expand_bytes = g1_policy()->expansion_amount();
-if (expand_bytes > 0) {
-size_t bytes_before = capacity();
-if (!expand(expand_bytes)) {
-// We failed to expand the heap so let's verify that
-// committed/uncommitted amount match the backing store
-assert(capacity() == _g1_storage.committed_size(), "committed size mismatch");
-assert(max_capacity() == _g1_storage.reserved_size(), "reserved size mismatch");
 }
 }
+// We have chosen the complete collection set. If marking is
+// active then, we clear the region fields of any of the
+// concurrent marking tasks whose region fields point into
+// the collection set as these values will become stale. This
+// will cause the owning marking threads to claim a new region
+// when marking restarts.
+if (mark_in_progress()) {
+concurrent_mark()->reset_active_task_region_fields_in_cset();
+}
+#ifdef ASSERT
+VerifyCSetClosure cl;
+collection_set_iterate(&cl);
+#endif // ASSERT
+setup_surviving_young_words();
+// Initialize the GC alloc regions.
+init_gc_alloc_regions();
+// Actually do the work...
+evacuate_collection_set();
+free_collection_set(g1_policy()->collection_set());
+g1_policy()->clear_collection_set();
+cleanup_surviving_young_words();
+// Start a new incremental collection set for the next pause.
+g1_policy()->start_incremental_cset_building();
+// Clear the _cset_fast_test bitmap in anticipation of adding
+// regions to the incremental collection set for the next
+// evacuation pause.
+clear_cset_fast_test();
+_young_list->reset_sampled_info();
+// Don't check the whole heap at this point as the
+// GC alloc regions from this pause have been tagged
+// as survivors and moved on to the survivor list.
+// Survivor regions will fail the !is_young() check.
+assert(check_young_list_empty(false /* check_heap */),
+"young list should be empty");
+#if YOUNG_LIST_VERBOSE
+gclog_or_tty->print_cr("Before recording survivors.\nYoung List:");
+_young_list->print();
+#endif // YOUNG_LIST_VERBOSE
+g1_policy()->record_survivor_regions(_young_list->survivor_length(),
+_young_list->first_survivor_region(),
+_young_list->last_survivor_region());
+_young_list->reset_auxilary_lists();
+if (evacuation_failed()) {
+_summary_bytes_used = recalculate_used();
+} else {
+// The "used" of the the collection set have already been subtracted
+// when they were freed.  Add in the bytes evacuated.
+_summary_bytes_used += g1_policy()->bytes_copied_during_gc();
+}
+if (g1_policy()->during_initial_mark_pause()) {
+concurrent_mark()->checkpointRootsInitialPost();
+set_marking_started();
+// CAUTION: after the doConcurrentMark() call below,
+// the concurrent marking thread(s) could be running
+// concurrently with us. Make sure that anything after
+// this point does not assume that we are the only GC thread
+// running. Note: of course, the actual marking work will
+// not start until the safepoint itself is released in
+// ConcurrentGCThread::safepoint_desynchronize().
+doConcurrentMark();
+}
+allocate_dummy_regions();
+#if YOUNG_LIST_VERBOSE
+gclog_or_tty->print_cr("\nEnd of the pause.\nYoung_list:");
+_young_list->print();
+g1_policy()->print_collection_set(g1_policy()->inc_cset_head(), gclog_or_tty);
+#endif // YOUNG_LIST_VERBOSE
+init_mutator_alloc_region();
+{
+size_t expand_bytes = g1_policy()->expansion_amount();
+if (expand_bytes > 0) {
+size_t bytes_before = capacity();
+if (!expand(expand_bytes)) {
+// We failed to expand the heap so let's verify that
+// committed/uncommitted amount match the backing store
+assert(capacity() == _g1_storage.committed_size(), "committed size mismatch");
+assert(max_capacity() == _g1_storage.reserved_size(), "reserved size mismatch");
+}
+}
+}
+double end_time_sec = os::elapsedTime();
+double pause_time_ms = (end_time_sec - start_time_sec) * MILLIUNITS;
+g1_policy()->record_pause_time_ms(pause_time_ms);
+g1_policy()->record_collection_pause_end();
+MemoryService::track_memory_usage();
+// In prepare_for_verify() below we'll need to scan the deferred
+// update buffers to bring the RSets up-to-date if
+// G1HRRSFlushLogBuffersOnVerify has been set. While scanning
+// the update buffers we'll probably need to scan cards on the
+// regions we just allocated to (i.e., the GC alloc
+// regions). However, during the last GC we called
+// set_saved_mark() on all the GC alloc regions, so card
+// scanning might skip the [saved_mark_word()...top()] area of
+// those regions (i.e., the area we allocated objects into
+// during the last GC). But it shouldn't. Given that
+// saved_mark_word() is conditional on whether the GC time stamp
+// on the region is current or not, by incrementing the GC time
+// stamp here we invalidate all the GC time stamps on all the
+// regions and saved_mark_word() will simply return top() for
+// all the regions. This is a nicer way of ensuring this rather
+// than iterating over the regions and fixing them. In fact, the
+// GC time stamp increment here also ensures that
+// saved_mark_word() will return top() between pauses, i.e.,
+// during concurrent refinement. So we don't need the
+// is_gc_active() check to decided which top to use when
+// scanning cards (see CR 7039627).
+increment_gc_time_stamp();
+if (VerifyAfterGC && total_collections() >= VerifyGCStartAt) {
+HandleMark hm;  // Discard invalid handles created during verification
+gclog_or_tty->print(" VerifyAfterGC:");
+prepare_for_verify();
+Universe::verify(/* allow dirty */ true,
+/* silent      */ false,
+/* option      */ VerifyOption_G1UsePrevMarking);
+}
+assert(!ref_processor_stw()->discovery_enabled(), "Postcondition");
+ref_processor_stw()->verify_no_references_recorded();
+// CM reference discovery will be re-enabled if necessary.
 }
-double end_time_sec = os::elapsedTime();
-double pause_time_ms = (end_time_sec - start_time_sec) * MILLIUNITS;
-g1_policy()->record_pause_time_ms(pause_time_ms);
-g1_policy()->record_collection_pause_end();
-MemoryService::track_memory_usage();
-// In prepare_for_verify() below we'll need to scan the deferred
-// update buffers to bring the RSets up-to-date if
-// G1HRRSFlushLogBuffersOnVerify has been set. While scanning
-// the update buffers we'll probably need to scan cards on the
-// regions we just allocated to (i.e., the GC alloc
-// regions). However, during the last GC we called
-// set_saved_mark() on all the GC alloc regions, so card
-// scanning might skip the [saved_mark_word()...top()] area of
-// those regions (i.e., the area we allocated objects into
-// during the last GC). But it shouldn't. Given that
-// saved_mark_word() is conditional on whether the GC time stamp
-// on the region is current or not, by incrementing the GC time
-// stamp here we invalidate all the GC time stamps on all the
-// regions and saved_mark_word() will simply return top() for
-// all the regions. This is a nicer way of ensuring this rather
-// than iterating over the regions and fixing them. In fact, the
-// GC time stamp increment here also ensures that
-// saved_mark_word() will return top() between pauses, i.e.,
-// during concurrent refinement. So we don't need the
-// is_gc_active() check to decided which top to use when
-// scanning cards (see CR 7039627).
-increment_gc_time_stamp();
-if (VerifyAfterGC && total_collections() >= VerifyGCStartAt) {
-HandleMark hm;  // Discard invalid handles created during verification
-gclog_or_tty->print(" VerifyAfterGC:");
-prepare_for_verify();
-Universe::verify(/* allow dirty */ true,
-/* silent      */ false,
-/* option      */ VerifyOption_G1UsePrevMarking);
-}
-if (was_enabled) ref_processor()->enable_discovery();
 {
 size_t expand_bytes = g1_policy()->expansion_amount();
 if (expand_bytes > 0) {
 size_t bytes_before = capacity();
 "Postcondition");
 assert(!_drain_in_progress, "Postcondition");
 delete _evac_failure_scan_stack;
 _evac_failure_scan_stack = NULL;
 }
-// *** Sequential G1 Evacuation
-class G1IsAliveClosure: public BoolObjectClosure {
-G1CollectedHeap* _g1;
-public:
-G1IsAliveClosure(G1CollectedHeap* g1) : _g1(g1) {}
-void do_object(oop p) { assert(false, "Do not call."); }
-bool do_object_b(oop p) {
-// It is reachable if it is outside the collection set, or is inside
-// and forwarded.
-return !_g1->obj_in_cs(p) || p->is_forwarded();
-}
-};
-class G1KeepAliveClosure: public OopClosure {
-G1CollectedHeap* _g1;
-public:
-G1KeepAliveClosure(G1CollectedHeap* g1) : _g1(g1) {}
-void do_oop(narrowOop* p) { guarantee(false, "Not needed"); }
-void do_oop(      oop* p) {
-oop obj = *p;
-if (_g1->obj_in_cs(obj)) {
-assert( obj->is_forwarded(), "invariant" );
-*p = obj->forwardee();
-}
-}
-};
 class UpdateRSetDeferred : public OopsInHeapRegionClosure {
 private:
 G1CollectedHeap* _g1;
 DirtyCardQueue *_dcq;
 }
 }
 #endif // ASSERT
 void G1ParScanThreadState::trim_queue() {
+assert(_evac_cl != NULL, "not set");
+assert(_evac_failure_cl != NULL, "not set");
+assert(_partial_scan_cl != NULL, "not set");
 StarTask ref;
 do {
 // Drain the overflow stack first, so other threads can steal.
 while (refs()->pop_overflow(ref)) {
 deal_with_reference(ref);
 }
 while (refs()->pop_local(ref)) {
 deal_with_reference(ref);
 }
 } while (!refs()->is_empty());
 }
 _g1h->g1_policy()->record_gc_worker_start_time(i, start_time_ms);
 ResourceMark rm;
 HandleMark   hm;
+ReferenceProcessor*             rp = _g1h->ref_processor_stw();
 G1ParScanThreadState            pss(_g1h, i);
-G1ParScanHeapEvacClosure        scan_evac_cl(_g1h, &pss);
+G1ParScanHeapEvacClosure        scan_evac_cl(_g1h, &pss, rp);
-G1ParScanHeapEvacFailureClosure evac_failure_cl(_g1h, &pss);
+G1ParScanHeapEvacFailureClosure evac_failure_cl(_g1h, &pss, rp);
-G1ParScanPartialArrayClosure    partial_scan_cl(_g1h, &pss);
+G1ParScanPartialArrayClosure    partial_scan_cl(_g1h, &pss, rp);
 pss.set_evac_closure(&scan_evac_cl);
 pss.set_evac_failure_closure(&evac_failure_cl);
 pss.set_partial_scan_closure(&partial_scan_cl);
-G1ParScanExtRootClosure         only_scan_root_cl(_g1h, &pss);
+G1ParScanExtRootClosure        only_scan_root_cl(_g1h, &pss, rp);
-G1ParScanPermClosure            only_scan_perm_cl(_g1h, &pss);
+G1ParScanPermClosure           only_scan_perm_cl(_g1h, &pss, rp);
-G1ParScanHeapRSClosure          only_scan_heap_rs_cl(_g1h, &pss);
-G1ParPushHeapRSClosure          push_heap_rs_cl(_g1h, &pss);
+G1ParScanAndMarkExtRootClosure scan_mark_root_cl(_g1h, &pss, rp);
+G1ParScanAndMarkPermClosure    scan_mark_perm_cl(_g1h, &pss, rp);
-G1ParScanAndMarkExtRootClosure  scan_mark_root_cl(_g1h, &pss);
-G1ParScanAndMarkPermClosure     scan_mark_perm_cl(_g1h, &pss);
+OopClosure*                    scan_root_cl = &only_scan_root_cl;
-G1ParScanAndMarkHeapRSClosure   scan_mark_heap_rs_cl(_g1h, &pss);
+OopsInHeapRegionClosure*       scan_perm_cl = &only_scan_perm_cl;
-OopsInHeapRegionClosure        *scan_root_cl;
-OopsInHeapRegionClosure        *scan_perm_cl;
 if (_g1h->g1_policy()->during_initial_mark_pause()) {
+// We also need to mark copied objects.
 scan_root_cl = &scan_mark_root_cl;
 scan_perm_cl = &scan_mark_perm_cl;
-} else {
+}
-scan_root_cl = &only_scan_root_cl;
-scan_perm_cl = &only_scan_perm_cl;
+// The following closure is used to scan RSets looking for reference
-}
+// fields that point into the collection set. The actual field iteration
+// is performed by a FilterIntoCSClosure, whose do_oop method calls the
+// do_oop method of the following closure.
+// Therefore we want to record the reference processor in the
+// FilterIntoCSClosure. To do so we record the STW reference
+// processor into the following closure and pass it to the
+// FilterIntoCSClosure in HeapRegionDCTOC::walk_mem_region_with_cl.
+G1ParPushHeapRSClosure          push_heap_rs_cl(_g1h, &pss, rp);
 pss.start_strong_roots();
 _g1h->g1_process_strong_roots(/* not collecting perm */ false,
 SharedHeap::SO_AllClasses,
 scan_root_cl,
 SharedHeap::ScanningOption so,
 OopClosure* scan_non_heap_roots,
 OopsInHeapRegionClosure* scan_rs,
 OopsInGenClosure* scan_perm,
 int worker_i) {
 // First scan the strong roots, including the perm gen.
 double ext_roots_start = os::elapsedTime();
 double closure_app_time_sec = 0.0;
 BufferingOopClosure buf_scan_non_heap_roots(scan_non_heap_roots);
 collecting_perm_gen, so,
 &buf_scan_non_heap_roots,
 &eager_scan_code_roots,
 &buf_scan_perm);
-// Now the ref_processor roots.
+// Now the CM ref_processor roots.
 if (!_process_strong_tasks->is_task_claimed(G1H_PS_refProcessor_oops_do)) {
-// We need to treat the discovered reference lists as roots and
+// We need to treat the discovered reference lists of the
-// keep entries (which are added by the marking threads) on them
+// concurrent mark ref processor as roots and keep entries
-// live until they can be processed at the end of marking.
+// (which are added by the marking threads) on them live
-ref_processor()->weak_oops_do(&buf_scan_non_heap_roots);
+// until they can be processed at the end of marking.
+ref_processor_cm()->weak_oops_do(&buf_scan_non_heap_roots);
 }
 // Finish up any enqueued closure apps (attributed as object copy time).
 buf_scan_non_heap_roots.done();
 buf_scan_perm.done();
 OopClosure* non_root_closure) {
 CodeBlobToOopClosure roots_in_blobs(root_closure, /*do_marking=*/ false);
 SharedHeap::process_weak_roots(root_closure, &roots_in_blobs, non_root_closure);
 }
+// Weak Reference Processing support
+// An always "is_alive" closure that is used to preserve referents.
+// If the object is non-null then it's alive.  Used in the preservation
+// of referent objects that are pointed to by reference objects
+// discovered by the CM ref processor.
+class G1AlwaysAliveClosure: public BoolObjectClosure {
+G1CollectedHeap* _g1;
+public:
+G1AlwaysAliveClosure(G1CollectedHeap* g1) : _g1(g1) {}
+void do_object(oop p) { assert(false, "Do not call."); }
+bool do_object_b(oop p) {
+if (p != NULL) {
+return true;
+}
+return false;
+}
+};
+bool G1STWIsAliveClosure::do_object_b(oop p) {
+// An object is reachable if it is outside the collection set,
+// or is inside and copied.
+return !_g1->obj_in_cs(p) || p->is_forwarded();
+}
+// Non Copying Keep Alive closure
+class G1KeepAliveClosure: public OopClosure {
+G1CollectedHeap* _g1;
+public:
+G1KeepAliveClosure(G1CollectedHeap* g1) : _g1(g1) {}
+void do_oop(narrowOop* p) { guarantee(false, "Not needed"); }
+void do_oop(      oop* p) {
+oop obj = *p;
+if (_g1->obj_in_cs(obj)) {
+assert( obj->is_forwarded(), "invariant" );
+*p = obj->forwardee();
+}
+}
+};
+// Copying Keep Alive closure - can be called from both
+// serial and parallel code as long as different worker
+// threads utilize different G1ParScanThreadState instances
+// and different queues.
+class G1CopyingKeepAliveClosure: public OopClosure {
+G1CollectedHeap*         _g1h;
+OopClosure*              _copy_non_heap_obj_cl;
+OopsInHeapRegionClosure* _copy_perm_obj_cl;
+G1ParScanThreadState*    _par_scan_state;
+public:
+G1CopyingKeepAliveClosure(G1CollectedHeap* g1h,
+OopClosure* non_heap_obj_cl,
+OopsInHeapRegionClosure* perm_obj_cl,
+G1ParScanThreadState* pss):
+_g1h(g1h),
+_copy_non_heap_obj_cl(non_heap_obj_cl),
+_copy_perm_obj_cl(perm_obj_cl),
+_par_scan_state(pss)
+{}
+virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+virtual void do_oop(      oop* p) { do_oop_work(p); }
+template <class T> void do_oop_work(T* p) {
+oop obj = oopDesc::load_decode_heap_oop(p);
+if (_g1h->obj_in_cs(obj)) {
+// If the referent object has been forwarded (either copied
+// to a new location or to itself in the event of an
+// evacuation failure) then we need to update the reference
+// field and, if both reference and referent are in the G1
+// heap, update the RSet for the referent.
+//
+// If the referent has not been forwarded then we have to keep
+// it alive by policy. Therefore we have copy the referent.
+//
+// If the reference field is in the G1 heap then we can push
+// on the PSS queue. When the queue is drained (after each
+// phase of reference processing) the object and it's followers
+// will be copied, the reference field set to point to the
+// new location, and the RSet updated. Otherwise we need to
+// use the the non-heap or perm closures directly to copy
+// the refernt object and update the pointer, while avoiding
+// updating the RSet.
+if (_g1h->is_in_g1_reserved(p)) {
+_par_scan_state->push_on_queue(p);
+} else {
+// The reference field is not in the G1 heap.
+if (_g1h->perm_gen()->is_in(p)) {
+_copy_perm_obj_cl->do_oop(p);
+} else {
+_copy_non_heap_obj_cl->do_oop(p);
+}
+}
+}
+}
+};
+// Serial drain queue closure. Called as the 'complete_gc'
+// closure for each discovered list in some of the
+// reference processing phases.
+class G1STWDrainQueueClosure: public VoidClosure {
+protected:
+G1CollectedHeap* _g1h;
+G1ParScanThreadState* _par_scan_state;
+G1ParScanThreadState*   par_scan_state() { return _par_scan_state; }
+public:
+G1STWDrainQueueClosure(G1CollectedHeap* g1h, G1ParScanThreadState* pss) :
+_g1h(g1h),
+_par_scan_state(pss)
+{ }
+void do_void() {
+G1ParScanThreadState* const pss = par_scan_state();
+pss->trim_queue();
+}
+};
+// Parallel Reference Processing closures
+// Implementation of AbstractRefProcTaskExecutor for parallel reference
+// processing during G1 evacuation pauses.
+class G1STWRefProcTaskExecutor: public AbstractRefProcTaskExecutor {
+private:
+G1CollectedHeap*   _g1h;
+RefToScanQueueSet* _queues;
+WorkGang*          _workers;
+int                _active_workers;
+public:
+G1STWRefProcTaskExecutor(G1CollectedHeap* g1h,
+WorkGang* workers,
+RefToScanQueueSet *task_queues,
+int n_workers) :
+_g1h(g1h),
+_queues(task_queues),
+_workers(workers),
+_active_workers(n_workers)
+{
+assert(n_workers > 0, "shouldn't call this otherwise");
+}
+// Executes the given task using concurrent marking worker threads.
+virtual void execute(ProcessTask& task);
+virtual void execute(EnqueueTask& task);
+};
+// Gang task for possibly parallel reference processing
+class G1STWRefProcTaskProxy: public AbstractGangTask {
+typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask;
+ProcessTask&     _proc_task;
+G1CollectedHeap* _g1h;
+RefToScanQueueSet *_task_queues;
+ParallelTaskTerminator* _terminator;
+public:
+G1STWRefProcTaskProxy(ProcessTask& proc_task,
+G1CollectedHeap* g1h,
+RefToScanQueueSet *task_queues,
+ParallelTaskTerminator* terminator) :
+AbstractGangTask("Process reference objects in parallel"),
+_proc_task(proc_task),
+_g1h(g1h),
+_task_queues(task_queues),
+_terminator(terminator)
+{}
+virtual void work(int i) {
+// The reference processing task executed by a single worker.
+ResourceMark rm;
+HandleMark   hm;
+G1STWIsAliveClosure is_alive(_g1h);
+G1ParScanThreadState pss(_g1h, i);
+G1ParScanHeapEvacClosure        scan_evac_cl(_g1h, &pss, NULL);
+G1ParScanHeapEvacFailureClosure evac_failure_cl(_g1h, &pss, NULL);
+G1ParScanPartialArrayClosure    partial_scan_cl(_g1h, &pss, NULL);
+pss.set_evac_closure(&scan_evac_cl);
+pss.set_evac_failure_closure(&evac_failure_cl);
+pss.set_partial_scan_closure(&partial_scan_cl);
+G1ParScanExtRootClosure        only_copy_non_heap_cl(_g1h, &pss, NULL);
+G1ParScanPermClosure           only_copy_perm_cl(_g1h, &pss, NULL);
+G1ParScanAndMarkExtRootClosure copy_mark_non_heap_cl(_g1h, &pss, NULL);
+G1ParScanAndMarkPermClosure    copy_mark_perm_cl(_g1h, &pss, NULL);
+OopClosure*                    copy_non_heap_cl = &only_copy_non_heap_cl;
+OopsInHeapRegionClosure*       copy_perm_cl = &only_copy_perm_cl;
+if (_g1h->g1_policy()->during_initial_mark_pause()) {
+// We also need to mark copied objects.
+copy_non_heap_cl = &copy_mark_non_heap_cl;
+copy_perm_cl = &copy_mark_perm_cl;
+}
+// Keep alive closure.
+G1CopyingKeepAliveClosure keep_alive(_g1h, copy_non_heap_cl, copy_perm_cl, &pss);
+// Complete GC closure
+G1ParEvacuateFollowersClosure drain_queue(_g1h, &pss, _task_queues, _terminator);
+// Call the reference processing task's work routine.
+_proc_task.work(i, is_alive, keep_alive, drain_queue);
+// Note we cannot assert that the refs array is empty here as not all
+// of the processing tasks (specifically phase2 - pp2_work) execute
+// the complete_gc closure (which ordinarily would drain the queue) so
+// the queue may not be empty.
+}
+};
+// Driver routine for parallel reference processing.
+// Creates an instance of the ref processing gang
+// task and has the worker threads execute it.
+void G1STWRefProcTaskExecutor::execute(ProcessTask& proc_task) {
+assert(_workers != NULL, "Need parallel worker threads.");
+ParallelTaskTerminator terminator(_active_workers, _queues);
+G1STWRefProcTaskProxy proc_task_proxy(proc_task, _g1h, _queues, &terminator);
+_g1h->set_par_threads(_active_workers);
+_workers->run_task(&proc_task_proxy);
+_g1h->set_par_threads(0);
+}
+// Gang task for parallel reference enqueueing.
+class G1STWRefEnqueueTaskProxy: public AbstractGangTask {
+typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask;
+EnqueueTask& _enq_task;
+public:
+G1STWRefEnqueueTaskProxy(EnqueueTask& enq_task) :
+AbstractGangTask("Enqueue reference objects in parallel"),
+_enq_task(enq_task)
+{ }
+virtual void work(int i) {
+_enq_task.work(i);
+}
+};
+// Driver routine for parallel reference enqueing.
+// Creates an instance of the ref enqueueing gang
+// task and has the worker threads execute it.
+void G1STWRefProcTaskExecutor::execute(EnqueueTask& enq_task) {
+assert(_workers != NULL, "Need parallel worker threads.");
+G1STWRefEnqueueTaskProxy enq_task_proxy(enq_task);
+_g1h->set_par_threads(_active_workers);
+_workers->run_task(&enq_task_proxy);
+_g1h->set_par_threads(0);
+}
+// End of weak reference support closures
+// Abstract task used to preserve (i.e. copy) any referent objects
+// that are in the collection set and are pointed to by reference
+// objects discovered by the CM ref processor.
+class G1ParPreserveCMReferentsTask: public AbstractGangTask {
+protected:
+G1CollectedHeap* _g1h;
+RefToScanQueueSet      *_queues;
+ParallelTaskTerminator _terminator;
+int _n_workers;
+public:
+G1ParPreserveCMReferentsTask(G1CollectedHeap* g1h,int workers, RefToScanQueueSet *task_queues) :
+AbstractGangTask("ParPreserveCMReferents"),
+_g1h(g1h),
+_queues(task_queues),
+_terminator(workers, _queues),
+_n_workers(workers)
+{ }
+void work(int i) {
+ResourceMark rm;
+HandleMark   hm;
+G1ParScanThreadState            pss(_g1h, i);
+G1ParScanHeapEvacClosure        scan_evac_cl(_g1h, &pss, NULL);
+G1ParScanHeapEvacFailureClosure evac_failure_cl(_g1h, &pss, NULL);
+G1ParScanPartialArrayClosure    partial_scan_cl(_g1h, &pss, NULL);
+pss.set_evac_closure(&scan_evac_cl);
+pss.set_evac_failure_closure(&evac_failure_cl);
+pss.set_partial_scan_closure(&partial_scan_cl);
+assert(pss.refs()->is_empty(), "both queue and overflow should be empty");
+G1ParScanExtRootClosure        only_copy_non_heap_cl(_g1h, &pss, NULL);
+G1ParScanPermClosure           only_copy_perm_cl(_g1h, &pss, NULL);
+G1ParScanAndMarkExtRootClosure copy_mark_non_heap_cl(_g1h, &pss, NULL);
+G1ParScanAndMarkPermClosure    copy_mark_perm_cl(_g1h, &pss, NULL);
+OopClosure*                    copy_non_heap_cl = &only_copy_non_heap_cl;
+OopsInHeapRegionClosure*       copy_perm_cl = &only_copy_perm_cl;
+if (_g1h->g1_policy()->during_initial_mark_pause()) {
+// We also need to mark copied objects.
+copy_non_heap_cl = &copy_mark_non_heap_cl;
+copy_perm_cl = &copy_mark_perm_cl;
+}
+// Is alive closure
+G1AlwaysAliveClosure always_alive(_g1h);
+// Copying keep alive closure. Applied to referent objects that need
+// to be copied.
+G1CopyingKeepAliveClosure keep_alive(_g1h, copy_non_heap_cl, copy_perm_cl, &pss);
+ReferenceProcessor* rp = _g1h->ref_processor_cm();
+int limit = ReferenceProcessor::number_of_subclasses_of_ref() * rp->max_num_q();
+int stride = MIN2(MAX2(_n_workers, 1), limit);
+// limit is set using max_num_q() - which was set using ParallelGCThreads.
+// So this must be true - but assert just in case someone decides to
+// change the worker ids.
+assert(0 <= i && i < limit, "sanity");
+assert(!rp->discovery_is_atomic(), "check this code");
+// Select discovered lists [i, i+stride, i+2*stride,...,limit)
+for (int idx = i; idx < limit; idx += stride) {
+DiscoveredList& ref_list = rp->discovered_soft_refs()[idx];
+DiscoveredListIterator iter(ref_list, &keep_alive, &always_alive);
+while (iter.has_next()) {
+// Since discovery is not atomic for the CM ref processor, we
+// can see some null referent objects.
+iter.load_ptrs(DEBUG_ONLY(true));
+oop ref = iter.obj();
+// This will filter nulls.
+if (iter.is_referent_alive()) {
+iter.make_referent_alive();
+}
+iter.move_to_next();
+}
+}
+// Drain the queue - which may cause stealing
+G1ParEvacuateFollowersClosure drain_queue(_g1h, &pss, _queues, &_terminator);
+drain_queue.do_void();
+// Allocation buffers were retired at the end of G1ParEvacuateFollowersClosure
+assert(pss.refs()->is_empty(), "should be");
+}
+};
+// Weak Reference processing during an evacuation pause (part 1).
+void G1CollectedHeap::process_discovered_references() {
+double ref_proc_start = os::elapsedTime();
+ReferenceProcessor* rp = _ref_processor_stw;
+assert(rp->discovery_enabled(), "should have been enabled");
+// Any reference objects, in the collection set, that were 'discovered'
+// by the CM ref processor should have already been copied (either by
+// applying the external root copy closure to the discovered lists, or
+// by following an RSet entry).
+//
+// But some of the referents, that are in the collection set, that these
+// reference objects point to may not have been copied: the STW ref
+// processor would have seen that the reference object had already
+// been 'discovered' and would have skipped discovering the reference,
+// but would not have treated the reference object as a regular oop.
+// As a reult the copy closure would not have been applied to the
+// referent object.
+//
+// We need to explicitly copy these referent objects - the references
+// will be processed at the end of remarking.
+//
+// We also need to do this copying before we process the reference
+// objects discovered by the STW ref processor in case one of these
+// referents points to another object which is also referenced by an
+// object discovered by the STW ref processor.
+int n_workers = (G1CollectedHeap::use_parallel_gc_threads() ?
+workers()->total_workers() : 1);
+set_par_threads(n_workers);
+G1ParPreserveCMReferentsTask keep_cm_referents(this, n_workers, _task_queues);
+if (G1CollectedHeap::use_parallel_gc_threads()) {
+workers()->run_task(&keep_cm_referents);
+} else {
+keep_cm_referents.work(0);
+}
+set_par_threads(0);
+// Closure to test whether a referent is alive.
+G1STWIsAliveClosure is_alive(this);
+// Even when parallel reference processing is enabled, the processing
+// of JNI refs is serial and performed serially by the current thread
+// rather than by a worker. The following PSS will be used for processing
+// JNI refs.
+// Use only a single queue for this PSS.
+G1ParScanThreadState pss(this, 0);
+// We do not embed a reference processor in the copying/scanning
+// closures while we're actually processing the discovered
+// reference objects.
+G1ParScanHeapEvacClosure        scan_evac_cl(this, &pss, NULL);
+G1ParScanHeapEvacFailureClosure evac_failure_cl(this, &pss, NULL);
+G1ParScanPartialArrayClosure    partial_scan_cl(this, &pss, NULL);
+pss.set_evac_closure(&scan_evac_cl);
+pss.set_evac_failure_closure(&evac_failure_cl);
+pss.set_partial_scan_closure(&partial_scan_cl);
+assert(pss.refs()->is_empty(), "pre-condition");
+G1ParScanExtRootClosure        only_copy_non_heap_cl(this, &pss, NULL);
+G1ParScanPermClosure           only_copy_perm_cl(this, &pss, NULL);
+G1ParScanAndMarkExtRootClosure copy_mark_non_heap_cl(this, &pss, NULL);
+G1ParScanAndMarkPermClosure    copy_mark_perm_cl(this, &pss, NULL);
+OopClosure*                    copy_non_heap_cl = &only_copy_non_heap_cl;
+OopsInHeapRegionClosure*       copy_perm_cl = &only_copy_perm_cl;
+if (_g1h->g1_policy()->during_initial_mark_pause()) {
+// We also need to mark copied objects.
+copy_non_heap_cl = &copy_mark_non_heap_cl;
+copy_perm_cl = &copy_mark_perm_cl;
+}
+// Keep alive closure.
+G1CopyingKeepAliveClosure keep_alive(this, copy_non_heap_cl, copy_perm_cl, &pss);
+// Serial Complete GC closure
+G1STWDrainQueueClosure drain_queue(this, &pss);
+// Setup the soft refs policy...
+rp->setup_policy(false);
+if (!rp->processing_is_mt()) {
+// Serial reference processing...
+rp->process_discovered_references(&is_alive,
+&keep_alive,
+&drain_queue,
+NULL);
+} else {
+// Parallel reference processing
+int active_workers = (ParallelGCThreads > 0 ? workers()->total_workers() : 1);
+assert(rp->num_q() == active_workers, "sanity");
+assert(active_workers <= rp->max_num_q(), "sanity");
+G1STWRefProcTaskExecutor par_task_executor(this, workers(), _task_queues, active_workers);
+rp->process_discovered_references(&is_alive, &keep_alive, &drain_queue, &par_task_executor);
+}
+// We have completed copying any necessary live referent objects
+// (that were not copied during the actual pause) so we can
+// retire any active alloc buffers
+pss.retire_alloc_buffers();
+assert(pss.refs()->is_empty(), "both queue and overflow should be empty");
+double ref_proc_time = os::elapsedTime() - ref_proc_start;
+g1_policy()->record_ref_proc_time(ref_proc_time * 1000.0);
+}
+// Weak Reference processing during an evacuation pause (part 2).
+void G1CollectedHeap::enqueue_discovered_references() {
+double ref_enq_start = os::elapsedTime();
+ReferenceProcessor* rp = _ref_processor_stw;
+assert(!rp->discovery_enabled(), "should have been disabled as part of processing");
+// Now enqueue any remaining on the discovered lists on to
+// the pending list.
+if (!rp->processing_is_mt()) {
+// Serial reference processing...
+rp->enqueue_discovered_references();
+} else {
+// Parallel reference enqueuing
+int active_workers = (ParallelGCThreads > 0 ? workers()->total_workers() : 1);
+assert(rp->num_q() == active_workers, "sanity");
+assert(active_workers <= rp->max_num_q(), "sanity");
+G1STWRefProcTaskExecutor par_task_executor(this, workers(), _task_queues, active_workers);
+rp->enqueue_discovered_references(&par_task_executor);
+}
+rp->verify_no_references_recorded();
+assert(!rp->discovery_enabled(), "should have been disabled");
+// FIXME
+// CM's reference processing also cleans up the string and symbol tables.
+// Should we do that here also? We could, but it is a serial operation
+// and could signicantly increase the pause time.
+double ref_enq_time = os::elapsedTime() - ref_enq_start;
+g1_policy()->record_ref_enq_time(ref_enq_time * 1000.0);
+}
 void G1CollectedHeap::evacuate_collection_set() {
 set_evacuation_failed(false);
 g1_rem_set()->prepare_for_oops_into_collection_set_do();
 concurrent_g1_refine()->set_use_cache(false);
 rem_set()->prepare_for_younger_refs_iterate(true);
 assert(dirty_card_queue_set().completed_buffers_num() == 0, "Should be empty");
 double start_par = os::elapsedTime();
 if (G1CollectedHeap::use_parallel_gc_threads()) {
 // The individual threads will set their evac-failure closures.
 StrongRootsScope srs(this);
 if (ParallelGCVerbose) G1ParScanThreadState::print_termination_stats_hdr();
 workers()->run_task(&g1_par_task);
 double par_time = (os::elapsedTime() - start_par) * 1000.0;
 g1_policy()->record_par_time(par_time);
 set_par_threads(0);
+// Process any discovered reference objects - we have
+// to do this _before_ we retire the GC alloc regions
+// as we may have to copy some 'reachable' referent
+// objects (and their reachable sub-graphs) that were
+// not copied during the pause.
+process_discovered_references();
 // Weak root processing.
 // Note: when JSR 292 is enabled and code blobs can contain
 // non-perm oops then we will need to process the code blobs
 // here too.
 {
-G1IsAliveClosure is_alive(this);
+G1STWIsAliveClosure is_alive(this);
 G1KeepAliveClosure keep_alive(this);
 JNIHandles::weak_oops_do(&is_alive, &keep_alive);
 }
 release_gc_alloc_regions();
 g1_rem_set()->cleanup_after_oops_into_collection_set_do();
 concurrent_g1_refine()->clear_hot_cache();
 concurrent_g1_refine()->set_use_cache(true);
 gclog_or_tty->print(" (to-space overflow)");
 } else if (PrintGC) {
 gclog_or_tty->print("--");
 }
 }
+// Enqueue any remaining references remaining on the STW
+// reference processor's discovered lists. We need to do
+// this after the card table is cleaned (and verified) as
+// the act of enqueuing entries on to the pending list
+// will log these updates (and dirty their associated
+// cards). We need these updates logged to update any
+// RSets.
+enqueue_discovered_references();
 if (G1DeferredRSUpdate) {
 RedirtyLoggedCardTableEntryFastClosure redirty;
 dirty_card_queue_set().set_closure(&redirty);
 dirty_card_queue_set().apply_closure_to_all_completed_buffers();
 r->set_next_dirty_cards_region(NULL);
 }
 }
 double elapsed = os::elapsedTime() - start;
-g1_policy()->record_clear_ct_time( elapsed * 1000.0);
+g1_policy()->record_clear_ct_time(elapsed * 1000.0);
 #ifndef PRODUCT
 if (G1VerifyCTCleanup || VerifyAfterGC) {
 G1VerifyCardTableCleanup cleanup_verifier(this, ct_bs);
 heap_region_iterate(&cleanup_verifier);
 }

Mercurial > hg > graal-jvmci-8

comparison src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp @ 3979:4dfb2df418f2