truffle: src/share/vm/gc_implementation/g1/concurrentMark.cpp comparison

comparison src/share/vm/gc_implementation/g1/concurrentMark.cpp @ 2181:d25d4ca69222

Merge.

author	Thomas Wuerthinger <wuerthinger@ssw.jku.at>
date	Wed, 16 Feb 2011 13:47:20 +0100
parents	3582bf76420e
children	1216415d8e35

comparison

equal deleted inserted replaced

-:50b45e2d9725
+:d25d4ca69222
 /*
-* Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved.
+* Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved.
 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
 *
 * This code is free software; you can redistribute it and/or modify it
 * under the terms of the GNU General Public License version 2 only, as
 * published by the Free Software Foundation.
 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp"
 #include "gc_implementation/g1/g1CollectorPolicy.hpp"
 #include "gc_implementation/g1/g1RemSet.hpp"
 #include "gc_implementation/g1/heapRegionRemSet.hpp"
 #include "gc_implementation/g1/heapRegionSeq.inline.hpp"
+#include "gc_implementation/shared/vmGCOperations.hpp"
 #include "memory/genOopClosures.inline.hpp"
 #include "memory/referencePolicy.hpp"
 #include "memory/resourceArea.hpp"
 #include "oops/oop.inline.hpp"
 #include "runtime/handles.inline.hpp"
 _parallel_marking_threads(0),
 _sleep_factor(0.0),
 _marking_task_overhead(1.0),
 _cleanup_sleep_factor(0.0),
 _cleanup_task_overhead(1.0),
+_cleanup_list("Cleanup List"),
 _region_bm(max_regions, false /* in_resource_area*/),
 _card_bm((rs.size() + CardTableModRefBS::card_size - 1) >>
 CardTableModRefBS::card_shift,
 false /* in_resource_area*/),
 _prevMarkBitMap(&_markBitMap1),
 assert(_markBitMap1.covers(rs), "_markBitMap1 inconsistency");
 assert(_markBitMap2.covers(rs), "_markBitMap2 inconsistency");
 SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set();
 satb_qs.set_buffer_size(G1SATBBufferSize);
-int size = (int) MAX2(ParallelGCThreads, (size_t)1);
-_par_cleanup_thread_state = NEW_C_HEAP_ARRAY(ParCleanupThreadState*, size);
-for (int i = 0 ; i < size; i++) {
-_par_cleanup_thread_state[i] = new ParCleanupThreadState;
-}
 _tasks = NEW_C_HEAP_ARRAY(CMTask*, _max_task_num);
 _accum_task_vtime = NEW_C_HEAP_ARRAY(double, _max_task_num);
 // so that the assertion in MarkingTaskQueue::task_queue doesn't fail
 _active_tasks = 0;
 clear_concurrent_marking_in_progress();
 }
 ConcurrentMark::~ConcurrentMark() {
-int size = (int) MAX2(ParallelGCThreads, (size_t)1);
-for (int i = 0; i < size; i++) delete _par_cleanup_thread_state[i];
-FREE_C_HEAP_ARRAY(ParCleanupThreadState*,
-_par_cleanup_thread_state);
 for (int i = 0; i < (int) _max_task_num; ++i) {
 delete _task_queues->queue(i);
 delete _tasks[i];
 }
 delete _task_queues;
 the_task->record_start_time();
 if (!_cm->has_aborted()) {
 do {
 double start_vtime_sec = os::elapsedVTime();
 double start_time_sec = os::elapsedTime();
-the_task->do_marking_step(10.0);
+double mark_step_duration_ms = G1ConcMarkStepDurationMillis;
+the_task->do_marking_step(mark_step_duration_ms,
+true /* do_stealing    */,
+true /* do_termination */);
 double end_time_sec = os::elapsedTime();
 double end_vtime_sec = os::elapsedVTime();
 double elapsed_vtime_sec = end_vtime_sec - start_vtime_sec;
 double elapsed_time_sec = end_time_sec - start_time_sec;
 _cm->clear_has_overflown();
 // a safepoint is indeed in progress as a younger generation
 // stop-the-world GC happens even as we mark in this generation.
 _restart_for_overflow = false;
-set_phase(MAX2((size_t) 1, parallel_marking_threads()), true);
+size_t active_workers = MAX2((size_t) 1, parallel_marking_threads());
+set_phase(active_workers, true /* concurrent */);
 CMConcurrentMarkingTask markingTask(this, cmThread());
 if (parallel_marking_threads() > 0)
 _parallel_workers->run_task(&markingTask);
 else
 // If a full collection has happened, we shouldn't do this.
 if (has_aborted()) {
 g1h->set_marking_complete(); // So bitmap clearing isn't confused
 return;
 }
+SvcGCMarker sgcm(SvcGCMarker::OTHER);
 if (VerifyDuringGC) {
 HandleMark hm;  // handle scope
 gclog_or_tty->print(" VerifyDuringGC:(before)");
 Universe::heap()->prepare_for_verify();
 // Clear the flag. We do not need it any more.
 clear_has_overflown();
 if (G1TraceMarkStackOverflow)
 gclog_or_tty->print_cr("\nRemark led to restart for overflow.");
 } else {
+SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
 // We're done with marking.
 // This is the end of  the marking cycle, we're expected all
 // threads to have SATB queues with active set to true.
-JavaThread::satb_mark_queue_set().set_active_all_threads(
+satb_mq_set.set_active_all_threads(false, /* new active value */
-false, /* new active value */
+true /* expected_active */);
-true /* expected_active */);
 if (VerifyDuringGC) {
 HandleMark hm;  // handle scope
 gclog_or_tty->print(" VerifyDuringGC:(after)");
 Universe::heap()->prepare_for_verify();
 Universe::heap()->verify(/* allow_dirty */      true,
 /* silent */           false,
 /* use_prev_marking */ false);
 }
+assert(!restart_for_overflow(), "sanity");
+}
+// Reset the marking state if marking completed
+if (!restart_for_overflow()) {
+set_non_marking_state();
 }
 #if VERIFY_OBJS_PROCESSED
 _scan_obj_cl.objs_processed = 0;
 ThreadLocalObjQueue::objs_enqueued = 0;
 G1CollectedHeap* _g1;
 int _worker_num;
 size_t _max_live_bytes;
 size_t _regions_claimed;
 size_t _freed_bytes;
-size_t _cleared_h_regions;
+FreeRegionList* _local_cleanup_list;
-size_t _freed_regions;
+HumongousRegionSet* _humongous_proxy_set;
-UncleanRegionList* _unclean_region_list;
+HRRSCleanupTask* _hrrs_cleanup_task;
 double _claimed_region_time;
 double _max_region_time;
 public:
 G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1,
-UncleanRegionList* list,
+int worker_num,
-int worker_num);
+FreeRegionList* local_cleanup_list,
+HumongousRegionSet* humongous_proxy_set,
+HRRSCleanupTask* hrrs_cleanup_task);
 size_t freed_bytes() { return _freed_bytes; }
-size_t cleared_h_regions() { return _cleared_h_regions; }
-size_t freed_regions() { return  _freed_regions; }
-UncleanRegionList* unclean_region_list() {
-return _unclean_region_list;
-}
 bool doHeapRegion(HeapRegion *r);
 size_t max_live_bytes() { return _max_live_bytes; }
 size_t regions_claimed() { return _regions_claimed; }
 double max_region_time_sec() { return _max_region_time; }
 };
 class G1ParNoteEndTask: public AbstractGangTask {
 friend class G1NoteEndOfConcMarkClosure;
 protected:
 G1CollectedHeap* _g1h;
 size_t _max_live_bytes;
 size_t _freed_bytes;
-ConcurrentMark::ParCleanupThreadState** _par_cleanup_thread_state;
+FreeRegionList* _cleanup_list;
 public:
 G1ParNoteEndTask(G1CollectedHeap* g1h,
-ConcurrentMark::ParCleanupThreadState**
+FreeRegionList* cleanup_list) :
-par_cleanup_thread_state) :
 AbstractGangTask("G1 note end"), _g1h(g1h),
-_max_live_bytes(0), _freed_bytes(0),
+_max_live_bytes(0), _freed_bytes(0), _cleanup_list(cleanup_list) { }
-_par_cleanup_thread_state(par_cleanup_thread_state)
-{}
 void work(int i) {
 double start = os::elapsedTime();
-G1NoteEndOfConcMarkClosure g1_note_end(_g1h,
+FreeRegionList local_cleanup_list("Local Cleanup List");
-&_par_cleanup_thread_state[i]->list,
+HumongousRegionSet humongous_proxy_set("Local Cleanup Humongous Proxy Set");
-i);
+HRRSCleanupTask hrrs_cleanup_task;
+G1NoteEndOfConcMarkClosure g1_note_end(_g1h, i, &local_cleanup_list,
+&humongous_proxy_set,
+&hrrs_cleanup_task);
 if (G1CollectedHeap::use_parallel_gc_threads()) {
 _g1h->heap_region_par_iterate_chunked(&g1_note_end, i,
 HeapRegion::NoteEndClaimValue);
 } else {
 _g1h->heap_region_iterate(&g1_note_end);
 }
 assert(g1_note_end.complete(), "Shouldn't have yielded!");
-// Now finish up freeing the current thread's regions.
+// Now update the lists
-_g1h->finish_free_region_work(g1_note_end.freed_bytes(),
+_g1h->update_sets_after_freeing_regions(g1_note_end.freed_bytes(),
-g1_note_end.cleared_h_regions(),
+NULL /* free_list */,
-0, NULL);
+&humongous_proxy_set,
+true /* par */);
 {
 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag);
 _max_live_bytes += g1_note_end.max_live_bytes();
 _freed_bytes += g1_note_end.freed_bytes();
+_cleanup_list->add_as_tail(&local_cleanup_list);
+assert(local_cleanup_list.is_empty(), "post-condition");
+HeapRegionRemSet::finish_cleanup_task(&hrrs_cleanup_task);
 }
 double end = os::elapsedTime();
 if (G1PrintParCleanupStats) {
 gclog_or_tty->print("     Worker thread %d [%8.3f..%8.3f = %8.3f ms] "
 "claimed %d regions (tot = %8.3f ms, max = %8.3f ms).\n",
 };
 G1NoteEndOfConcMarkClosure::
 G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1,
-UncleanRegionList* list,
+int worker_num,
-int worker_num)
+FreeRegionList* local_cleanup_list,
+HumongousRegionSet* humongous_proxy_set,
+HRRSCleanupTask* hrrs_cleanup_task)
 : _g1(g1), _worker_num(worker_num),
 _max_live_bytes(0), _regions_claimed(0),
-_freed_bytes(0), _cleared_h_regions(0), _freed_regions(0),
+_freed_bytes(0),
 _claimed_region_time(0.0), _max_region_time(0.0),
-_unclean_region_list(list)
+_local_cleanup_list(local_cleanup_list),
-{}
+_humongous_proxy_set(humongous_proxy_set),
+_hrrs_cleanup_task(hrrs_cleanup_task) { }
-bool G1NoteEndOfConcMarkClosure::doHeapRegion(HeapRegion *r) {
+bool G1NoteEndOfConcMarkClosure::doHeapRegion(HeapRegion *hr) {
 // We use a claim value of zero here because all regions
 // were claimed with value 1 in the FinalCount task.
-r->reset_gc_time_stamp();
+hr->reset_gc_time_stamp();
-if (!r->continuesHumongous()) {
+if (!hr->continuesHumongous()) {
 double start = os::elapsedTime();
 _regions_claimed++;
-r->note_end_of_marking();
+hr->note_end_of_marking();
-_max_live_bytes += r->max_live_bytes();
+_max_live_bytes += hr->max_live_bytes();
-_g1->free_region_if_totally_empty_work(r,
+_g1->free_region_if_empty(hr,
-_freed_bytes,
+&_freed_bytes,
-_cleared_h_regions,
+_local_cleanup_list,
-_freed_regions,
+_humongous_proxy_set,
-_unclean_region_list,
+_hrrs_cleanup_task,
-true /*par*/);
+true /* par */);
 double region_time = (os::elapsedTime() - start);
 _claimed_region_time += region_time;
 if (region_time > _max_region_time) _max_region_time = region_time;
 }
 return false;
 // If a full collection has happened, we shouldn't do this.
 if (has_aborted()) {
 g1h->set_marking_complete(); // So bitmap clearing isn't confused
 return;
 }
+g1h->verify_region_sets_optional();
 if (VerifyDuringGC) {
 HandleMark hm;  // handle scope
 gclog_or_tty->print(" VerifyDuringGC:(before)");
 Universe::heap()->prepare_for_verify();
 G1CollectorPolicy* g1p = G1CollectedHeap::heap()->g1_policy();
 g1p->record_concurrent_mark_cleanup_start();
 double start = os::elapsedTime();
+HeapRegionRemSet::reset_for_cleanup_tasks();
 // Do counting once more with the world stopped for good measure.
 G1ParFinalCountTask g1_par_count_task(g1h, nextMarkBitMap(),
 &_region_bm, &_card_bm);
 if (G1CollectedHeap::use_parallel_gc_threads()) {
 g1h->reset_gc_time_stamp();
 // Note end of marking in all heap regions.
 double note_end_start = os::elapsedTime();
-G1ParNoteEndTask g1_par_note_end_task(g1h, _par_cleanup_thread_state);
+G1ParNoteEndTask g1_par_note_end_task(g1h, &_cleanup_list);
 if (G1CollectedHeap::use_parallel_gc_threads()) {
 int n_workers = g1h->workers()->total_workers();
 g1h->set_par_threads(n_workers);
 g1h->workers()->run_task(&g1_par_note_end_task);
 g1h->set_par_threads(0);
 assert(g1h->check_heap_region_claim_values(HeapRegion::NoteEndClaimValue),
 "sanity check");
 } else {
 g1_par_note_end_task.work(0);
 }
-g1h->set_unclean_regions_coming(true);
+if (!cleanup_list_is_empty()) {
+// The cleanup list is not empty, so we'll have to process it
+// concurrently. Notify anyone else that might be wanting free
+// regions that there will be more free regions coming soon.
+g1h->set_free_regions_coming();
+}
 double note_end_end = os::elapsedTime();
-// Tell the mutators that there might be unclean regions coming...
 if (G1PrintParCleanupStats) {
 gclog_or_tty->print_cr("  note end of marking: %8.3f ms.",
 (note_end_end - note_end_start)*1000.0);
 }
 Universe::heap()->prepare_for_verify();
 Universe::verify(/* allow dirty  */ true,
 /* silent       */ false,
 /* prev marking */ true);
 }
+g1h->verify_region_sets_optional();
 }
 void ConcurrentMark::completeCleanup() {
-// A full collection intervened.
 if (has_aborted()) return;
-int first = 0;
+G1CollectedHeap* g1h = G1CollectedHeap::heap();
-int last = (int)MAX2(ParallelGCThreads, (size_t)1);
-for (int t = 0; t < last; t++) {
+_cleanup_list.verify_optional();
-UncleanRegionList* list = &_par_cleanup_thread_state[t]->list;
+FreeRegionList local_free_list("Local Cleanup List");
-assert(list->well_formed(), "Inv");
-HeapRegion* hd = list->hd();
+if (G1ConcRegionFreeingVerbose) {
-while (hd != NULL) {
+gclog_or_tty->print_cr("G1ConcRegionFreeing [complete cleanup] : "
-// Now finish up the other stuff.
+"cleanup list has "SIZE_FORMAT" entries",
-hd->rem_set()->clear();
+_cleanup_list.length());
-HeapRegion* next_hd = hd->next_from_unclean_list();
+}
-(void)list->pop();
-assert(list->hd() == next_hd, "how not?");
+// Noone else should be accessing the _cleanup_list at this point,
-_g1h->put_region_on_unclean_list(hd);
+// so it's not necessary to take any locks
-if (!hd->isHumongous()) {
+while (!_cleanup_list.is_empty()) {
-// Add this to the _free_regions count by 1.
+HeapRegion* hr = _cleanup_list.remove_head();
-_g1h->finish_free_region_work(0, 0, 1, NULL);
+assert(hr != NULL, "the list was not empty");
+hr->rem_set()->clear();
+local_free_list.add_as_tail(hr);
+// Instead of adding one region at a time to the secondary_free_list,
+// we accumulate them in the local list and move them a few at a
+// time. This also cuts down on the number of notify_all() calls
+// we do during this process. We'll also append the local list when
+// _cleanup_list is empty (which means we just removed the last
+// region from the _cleanup_list).
+if ((local_free_list.length() % G1SecondaryFreeListAppendLength == 0) ||
+_cleanup_list.is_empty()) {
+if (G1ConcRegionFreeingVerbose) {
+gclog_or_tty->print_cr("G1ConcRegionFreeing [complete cleanup] : "
+"appending "SIZE_FORMAT" entries to the "
+"secondary_free_list, clean list still has "
+SIZE_FORMAT" entries",
+local_free_list.length(),
+_cleanup_list.length());
 }
-hd = list->hd();
-assert(hd == next_hd, "how not?");
+{
-}
+MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag);
-}
+g1h->secondary_free_list_add_as_tail(&local_free_list);
-}
+SecondaryFreeList_lock->notify_all();
+}
+if (G1StressConcRegionFreeing) {
+for (uintx i = 0; i < G1StressConcRegionFreeingDelayMillis; ++i) {
+os::sleep(Thread::current(), (jlong) 1, false);
+}
+}
+}
+}
+assert(local_free_list.is_empty(), "post-condition");
+}
+// Support closures for reference procssing in G1
 bool G1CMIsAliveClosure::do_object_b(oop obj) {
 HeapWord* addr = (HeapWord*)obj;
 return addr != NULL &&
 (!_g1->is_in_g1_reserved(addr) || !_g1->is_obj_ill(obj));
 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
 virtual void do_oop(      oop* p) { do_oop_work(p); }
 template <class T> void do_oop_work(T* p) {
-oop thisOop = oopDesc::load_decode_heap_oop(p);
+oop obj = oopDesc::load_decode_heap_oop(p);
-HeapWord* addr = (HeapWord*)thisOop;
+HeapWord* addr = (HeapWord*)obj;
-if (_g1->is_in_g1_reserved(addr) && _g1->is_obj_ill(thisOop)) {
+if (_cm->verbose_high())
+gclog_or_tty->print_cr("\t[0] we're looking at location "
+"*"PTR_FORMAT" = "PTR_FORMAT,
+p, (void*) obj);
+if (_g1->is_in_g1_reserved(addr) && _g1->is_obj_ill(obj)) {
 _bitMap->mark(addr);
-_cm->mark_stack_push(thisOop);
+_cm->mark_stack_push(obj);
 }
 }
 };
 class G1CMDrainMarkingStackClosure: public VoidClosure {
 void do_void() {
 _markStack->drain((OopClosure*)_oopClosure, _bitMap, false);
 }
 };
+// 'Keep Alive' closure used by parallel reference processing.
+// An instance of this closure is used in the parallel reference processing
+// code rather than an instance of G1CMKeepAliveClosure. We could have used
+// the G1CMKeepAliveClosure as it is MT-safe. Also reference objects are
+// placed on to discovered ref lists once so we can mark and push with no
+// need to check whether the object has already been marked. Using the
+// G1CMKeepAliveClosure would mean, however, having all the worker threads
+// operating on the global mark stack. This means that an individual
+// worker would be doing lock-free pushes while it processes its own
+// discovered ref list followed by drain call. If the discovered ref lists
+// are unbalanced then this could cause interference with the other
+// workers. Using a CMTask (and its embedded local data structures)
+// avoids that potential interference.
+class G1CMParKeepAliveAndDrainClosure: public OopClosure {
+ConcurrentMark*  _cm;
+CMTask*          _task;
+CMBitMap*        _bitMap;
+int              _ref_counter_limit;
+int              _ref_counter;
+public:
+G1CMParKeepAliveAndDrainClosure(ConcurrentMark* cm,
+CMTask* task,
+CMBitMap* bitMap) :
+_cm(cm), _task(task), _bitMap(bitMap),
+_ref_counter_limit(G1RefProcDrainInterval)
+{
+assert(_ref_counter_limit > 0, "sanity");
+_ref_counter = _ref_counter_limit;
+}
+virtual void do_oop(narrowOop* p) { do_oop_work(p); }
+virtual void do_oop(      oop* p) { do_oop_work(p); }
+template <class T> void do_oop_work(T* p) {
+if (!_cm->has_overflown()) {
+oop obj = oopDesc::load_decode_heap_oop(p);
+if (_cm->verbose_high())
+gclog_or_tty->print_cr("\t[%d] we're looking at location "
+"*"PTR_FORMAT" = "PTR_FORMAT,
+_task->task_id(), p, (void*) obj);
+_task->deal_with_reference(obj);
+_ref_counter--;
+if (_ref_counter == 0) {
+// We have dealt with _ref_counter_limit references, pushing them and objects
+// reachable from them on to the local stack (and possibly the global stack).
+// Call do_marking_step() to process these entries. We call the routine in a
+// loop, which we'll exit if there's nothing more to do (i.e. we're done
+// with the entries that we've pushed as a result of the deal_with_reference
+// calls above) or we overflow.
+// Note: CMTask::do_marking_step() can set the CMTask::has_aborted() flag
+// while there may still be some work to do. (See the comment at the
+// beginning of CMTask::do_marking_step() for those conditions - one of which
+// is reaching the specified time target.) It is only when
+// CMTask::do_marking_step() returns without setting the has_aborted() flag
+// that the marking has completed.
+do {
+double mark_step_duration_ms = G1ConcMarkStepDurationMillis;
+_task->do_marking_step(mark_step_duration_ms,
+false /* do_stealing    */,
+false /* do_termination */);
+} while (_task->has_aborted() && !_cm->has_overflown());
+_ref_counter = _ref_counter_limit;
+}
+} else {
+if (_cm->verbose_high())
+gclog_or_tty->print_cr("\t[%d] CM Overflow", _task->task_id());
+}
+}
+};
+class G1CMParDrainMarkingStackClosure: public VoidClosure {
+ConcurrentMark* _cm;
+CMTask* _task;
+public:
+G1CMParDrainMarkingStackClosure(ConcurrentMark* cm, CMTask* task) :
+_cm(cm), _task(task)
+{}
+void do_void() {
+do {
+if (_cm->verbose_high())
+gclog_or_tty->print_cr("\t[%d] Drain: Calling do marking_step", _task->task_id());
+// We call CMTask::do_marking_step() to completely drain the local and
+// global marking stacks. The routine is called in a loop, which we'll
+// exit if there's nothing more to do (i.e. we'completely drained the
+// entries that were pushed as a result of applying the
+// G1CMParKeepAliveAndDrainClosure to the entries on the discovered ref
+// lists above) or we overflow the global marking stack.
+// Note: CMTask::do_marking_step() can set the CMTask::has_aborted() flag
+// while there may still be some work to do. (See the comment at the
+// beginning of CMTask::do_marking_step() for those conditions - one of which
+// is reaching the specified time target.) It is only when
+// CMTask::do_marking_step() returns without setting the has_aborted() flag
+// that the marking has completed.
+_task->do_marking_step(1000000000.0 /* something very large */,
+true /* do_stealing    */,
+true /* do_termination */);
+} while (_task->has_aborted() && !_cm->has_overflown());
+}
+};
+// Implementation of AbstractRefProcTaskExecutor for G1
+class G1RefProcTaskExecutor: public AbstractRefProcTaskExecutor {
+private:
+G1CollectedHeap* _g1h;
+ConcurrentMark*  _cm;
+CMBitMap*        _bitmap;
+WorkGang*        _workers;
+int              _active_workers;
+public:
+G1RefProcTaskExecutor(G1CollectedHeap* g1h,
+ConcurrentMark* cm,
+CMBitMap* bitmap,
+WorkGang* workers,
+int n_workers) :
+_g1h(g1h), _cm(cm), _bitmap(bitmap),
+_workers(workers), _active_workers(n_workers)
+{ }
+// Executes the given task using concurrent marking worker threads.
+virtual void execute(ProcessTask& task);
+virtual void execute(EnqueueTask& task);
+};
+class G1RefProcTaskProxy: public AbstractGangTask {
+typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask;
+ProcessTask&     _proc_task;
+G1CollectedHeap* _g1h;
+ConcurrentMark*  _cm;
+CMBitMap*        _bitmap;
+public:
+G1RefProcTaskProxy(ProcessTask& proc_task,
+G1CollectedHeap* g1h,
+ConcurrentMark* cm,
+CMBitMap* bitmap) :
+AbstractGangTask("Process reference objects in parallel"),
+_proc_task(proc_task), _g1h(g1h), _cm(cm), _bitmap(bitmap)
+{}
+virtual void work(int i) {
+CMTask* marking_task = _cm->task(i);
+G1CMIsAliveClosure g1_is_alive(_g1h);
+G1CMParKeepAliveAndDrainClosure g1_par_keep_alive(_cm, marking_task, _bitmap);
+G1CMParDrainMarkingStackClosure g1_par_drain(_cm, marking_task);
+_proc_task.work(i, g1_is_alive, g1_par_keep_alive, g1_par_drain);
+}
+};
+void G1RefProcTaskExecutor::execute(ProcessTask& proc_task) {
+assert(_workers != NULL, "Need parallel worker threads.");
+G1RefProcTaskProxy proc_task_proxy(proc_task, _g1h, _cm, _bitmap);
+// We need to reset the phase for each task execution so that
+// the termination protocol of CMTask::do_marking_step works.
+_cm->set_phase(_active_workers, false /* concurrent */);
+_g1h->set_par_threads(_active_workers);
+_workers->run_task(&proc_task_proxy);
+_g1h->set_par_threads(0);
+}
+class G1RefEnqueueTaskProxy: public AbstractGangTask {
+typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask;
+EnqueueTask& _enq_task;
+public:
+G1RefEnqueueTaskProxy(EnqueueTask& enq_task) :
+AbstractGangTask("Enqueue reference objects in parallel"),
+_enq_task(enq_task)
+{ }
+virtual void work(int i) {
+_enq_task.work(i);
+}
+};
+void G1RefProcTaskExecutor::execute(EnqueueTask& enq_task) {
+assert(_workers != NULL, "Need parallel worker threads.");
+G1RefEnqueueTaskProxy enq_task_proxy(enq_task);
+_g1h->set_par_threads(_active_workers);
+_workers->run_task(&enq_task_proxy);
+_g1h->set_par_threads(0);
+}
 void ConcurrentMark::weakRefsWork(bool clear_all_soft_refs) {
 ResourceMark rm;
 HandleMark   hm;
 G1CollectedHeap* g1h   = G1CollectedHeap::heap();
 ReferenceProcessor* rp = g1h->ref_processor();
 G1CMIsAliveClosure   g1_is_alive(g1h);
 G1CMKeepAliveClosure g1_keep_alive(g1h, this, nextMarkBitMap());
 G1CMDrainMarkingStackClosure
 g1_drain_mark_stack(nextMarkBitMap(), &_markStack, &g1_keep_alive);
-// XXXYYY  Also: copy the parallel ref processing code from CMS.
+// We use the work gang from the G1CollectedHeap and we utilize all
-rp->process_discovered_references(&g1_is_alive,
+// the worker threads.
-&g1_keep_alive,
+int active_workers = MAX2(MIN2(g1h->workers()->total_workers(), (int)_max_task_num), 1);
-&g1_drain_mark_stack,
-NULL);
+G1RefProcTaskExecutor par_task_executor(g1h, this, nextMarkBitMap(),
+g1h->workers(), active_workers);
+if (rp->processing_is_mt()) {
+// Set the degree of MT here.  If the discovery is done MT, there
+// may have been a different number of threads doing the discovery
+// and a different number of discovered lists may have Ref objects.
+// That is OK as long as the Reference lists are balanced (see
+// balance_all_queues() and balance_queues()).
+rp->set_mt_degree(active_workers);
+rp->process_discovered_references(&g1_is_alive,
+&g1_keep_alive,
+&g1_drain_mark_stack,
+&par_task_executor);
+// The work routines of the parallel keep_alive and drain_marking_stack
+// will set the has_overflown flag if we overflow the global marking
+// stack.
+} else {
+rp->process_discovered_references(&g1_is_alive,
+&g1_keep_alive,
+&g1_drain_mark_stack,
+NULL);
+}
 assert(_markStack.overflow() || _markStack.isEmpty(),
 "mark stack should be empty (unless it overflowed)");
 if (_markStack.overflow()) {
+// Should have been done already when we tried to push an
+// entry on to the global mark stack. But let's do it again.
 set_has_overflown();
 }
-rp->enqueue_discovered_references();
+if (rp->processing_is_mt()) {
+assert(rp->num_q() == active_workers, "why not");
+rp->enqueue_discovered_references(&par_task_executor);
+} else {
+rp->enqueue_discovered_references();
+}
 rp->verify_no_references_recorded();
 assert(!rp->discovery_enabled(), "should have been disabled");
-// Now clean up stale oops in SymbolTable and StringTable
+// Now clean up stale oops in StringTable
-SymbolTable::unlink(&g1_is_alive);
 StringTable::unlink(&g1_is_alive);
+// Clean up unreferenced symbols in symbol table.
+SymbolTable::unlink();
 }
 void ConcurrentMark::swapMarkBitMaps() {
 CMBitMapRO* temp = _prevMarkBitMap;
 _prevMarkBitMap  = (CMBitMapRO*)_nextMarkBitMap;
 // only proceed if we're supposed to be actived.
 if ((size_t)worker_i < _cm->active_tasks()) {
 CMTask* task = _cm->task(worker_i);
 task->record_start_time();
 do {
-task->do_marking_step(1000000000.0 /* something very large */);
+task->do_marking_step(1000000000.0 /* something very large */,
+true /* do_stealing    */,
+true /* do_termination */);
 } while (task->has_aborted() && !_cm->has_overflown());
 // If we overflow, then we do not want to restart. We instead
 // want to abort remark and do concurrent marking again.
 task->record_end_time();
 }
 if (G1CollectedHeap::use_parallel_gc_threads()) {
 G1CollectedHeap::StrongRootsScope srs(g1h);
 // this is remark, so we'll use up all available threads
 int active_workers = ParallelGCThreads;
-set_phase(active_workers, false);
+set_phase(active_workers, false /* concurrent */);
 CMRemarkTask remarkTask(this);
 // We will start all available threads, even if we decide that the
 // active_workers will be fewer. The extra ones will just bail out
 // immediately.
 g1h->set_par_threads(0);
 } else {
 G1CollectedHeap::StrongRootsScope srs(g1h);
 // this is remark, so we'll use up all available threads
 int active_workers = 1;
-set_phase(active_workers, false);
+set_phase(active_workers, false /* concurrent */);
 CMRemarkTask remarkTask(this);
 // We will start all available threads, even if we decide that the
 // active_workers will be fewer. The extra ones will just bail out
 // immediately.
 }
 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set();
 guarantee(satb_mq_set.completed_buffers_num() == 0, "invariant");
 print_stats();
-if (!restart_for_overflow())
-set_non_marking_state();
 #if VERIFY_OBJS_PROCESSED
 if (_scan_obj_cl.objs_processed != ThreadLocalObjQueue::objs_enqueued) {
 gclog_or_tty->print_cr("Processed = %d, enqueued = %d.",
 _scan_obj_cl.objs_processed,
 public:
 virtual void do_oop(narrowOop* p) { do_oop_work(p); }
 virtual void do_oop(      oop* p) { do_oop_work(p); }
 template <class T> void do_oop_work(T* p) {
-assert(_g1h->is_in_g1_reserved((HeapWord*) p), "invariant");
+assert( _g1h->is_in_g1_reserved((HeapWord*) p), "invariant");
-assert(!_g1h->heap_region_containing((HeapWord*) p)->is_on_free_list(),
+assert(!_g1h->is_on_free_list(
-"invariant");
+_g1h->heap_region_containing((HeapWord*) p)), "invariant");
 oop obj = oopDesc::load_decode_heap_oop(p);
 if (_cm->verbose_high())
 gclog_or_tty->print_cr("[%d] we're looking at location "
 "*"PTR_FORMAT" = "PTR_FORMAT,
 push(obj);
 } else {
 // do nothing
 }
 #else // _CHECK_BOTH_FINGERS_
 // we will only check the global finger
 if (objAddr < global_finger) {
 // see long comment above
 if (_cm->verbose_high())
 }
 void CMTask::push(oop obj) {
 HeapWord* objAddr = (HeapWord*) obj;
 assert(_g1h->is_in_g1_reserved(objAddr), "invariant");
-assert(!_g1h->heap_region_containing(objAddr)->is_on_free_list(),
+assert(!_g1h->is_on_free_list(
-"invariant");
+_g1h->heap_region_containing((HeapWord*) objAddr)), "invariant");
 assert(!_g1h->is_obj_ill(obj), "invariant");
 assert(_nextMarkBitMap->isMarked(objAddr), "invariant");
 if (_cm->verbose_high())
 gclog_or_tty->print_cr("[%d] pushing "PTR_FORMAT, _task_id, (void*) obj);
 // (5) We check whether we've reached our time quota. If we have,
 // then we abort.
 double elapsed_time_ms = curr_time_ms - _start_time_ms;
 if (elapsed_time_ms > _time_target_ms) {
 set_has_aborted();
-_has_aborted_timed_out = true;
+_has_timed_out = true;
 statsOnly( ++_aborted_timed_out );
 return;
 }
 // (6) Finally, we check whether there are enough completed STAB
 if (_cm->verbose_high())
 gclog_or_tty->print_cr("[%d] popped "PTR_FORMAT, _task_id,
 (void*) obj);
 assert(_g1h->is_in_g1_reserved((HeapWord*) obj), "invariant" );
-assert(!_g1h->heap_region_containing(obj)->is_on_free_list(),
+assert(!_g1h->is_on_free_list(
-"invariant");
+_g1h->heap_region_containing((HeapWord*) obj)), "invariant");
 scan_object(obj);
 if (_task_queue->size() <= target_size || has_aborted())
 ret = false;
 place, it was natural to piggy-back all the other conditions on it
 too and not constantly check them throughout the code.
 *****************************************************************************/
-void CMTask::do_marking_step(double time_target_ms) {
+void CMTask::do_marking_step(double time_target_ms,
+bool do_stealing,
+bool do_termination) {
 assert(time_target_ms >= 1.0, "minimum granularity is 1ms");
 assert(concurrent() == _cm->concurrent(), "they should be the same");
 assert(concurrent() || _cm->region_stack_empty(),
 "the region stack should have been cleared before remark");
 _refs_reached  = 0;
 recalculate_limits();
 // clear all flags
 clear_has_aborted();
-_has_aborted_timed_out = false;
+_has_timed_out = false;
 _draining_satb_buffers = false;
 ++_calls;
 if (_cm->verbose_low())
 // local queue and global stack.
 drain_local_queue(false);
 drain_global_stack(false);
 // Attempt at work stealing from other task's queues.
-if (!has_aborted()) {
+if (do_stealing && !has_aborted()) {
 // We have not aborted. This means that we have finished all that
 // we could. Let's try to do some stealing...
 // We cannot check whether the global stack is empty, since other
 // tasks might be pushing objects to it concurrently. We also cannot
 }
 }
 // We still haven't aborted. Now, let's try to get into the
 // termination protocol.
-if (!has_aborted()) {
+if (do_termination && !has_aborted()) {
 // We cannot check whether the global stack is empty, since other
 // tasks might be concurrently pushing objects on it. We also cannot
 // check if the region stack is empty because if a thread is aborting
 // it can push a partially done region back.
 // Separated the asserts so that we know which one fires.
 if (has_aborted()) {
 // The task was aborted for some reason.
 statsOnly( ++_aborted );
-if (_has_aborted_timed_out) {
+if (_has_timed_out) {
 double diff_ms = elapsed_time_ms - _time_target_ms;
 // Keep statistics of how well we did with respect to hitting
 // our target only if we actually timed out (if we aborted for
 // other reasons, then the results might get skewed).
 _marking_step_diffs_ms.add(diff_ms);

Mercurial > hg > truffle

comparison src/share/vm/gc_implementation/g1/concurrentMark.cpp @ 2181:d25d4ca69222