# HG changeset patch # User johnc # Date 1296164548 28800 # Node ID 27e4ea99855d05a6af1a6f7abcaf618cdb1af33d # Parent 9846d99e16d3e44b22607c2ad19ac96acaba36c7# Parent 81668b1f48775545c3a181d186479d770666334f Merge diff -r 9846d99e16d3 -r 27e4ea99855d src/share/vm/gc_implementation/g1/concurrentMark.cpp --- a/src/share/vm/gc_implementation/g1/concurrentMark.cpp Thu Jan 27 14:05:59 2011 -0500 +++ b/src/share/vm/gc_implementation/g1/concurrentMark.cpp Thu Jan 27 13:42:28 2011 -0800 @@ -1055,7 +1055,12 @@ do { double start_vtime_sec = os::elapsedVTime(); double start_time_sec = os::elapsedTime(); - the_task->do_marking_step(10.0); + double mark_step_duration_ms = G1ConcMarkStepDurationMillis; + + the_task->do_marking_step(mark_step_duration_ms, + true /* do_stealing */, + true /* do_termination */); + double end_time_sec = os::elapsedTime(); double end_vtime_sec = os::elapsedVTime(); double elapsed_vtime_sec = end_vtime_sec - start_vtime_sec; @@ -1111,7 +1116,8 @@ _restart_for_overflow = false; - set_phase(MAX2((size_t) 1, parallel_marking_threads()), true); + size_t active_workers = MAX2((size_t) 1, parallel_marking_threads()); + set_phase(active_workers, true /* concurrent */); CMConcurrentMarkingTask markingTask(this, cmThread()); if (parallel_marking_threads() > 0) @@ -1176,6 +1182,12 @@ /* silent */ false, /* use_prev_marking */ false); } + assert(!restart_for_overflow(), "sanity"); + } + + // Reset the marking state if marking completed + if (!restart_for_overflow()) { + set_non_marking_state(); } #if VERIFY_OBJS_PROCESSED @@ -1500,21 +1512,19 @@ size_t _max_live_bytes; size_t _regions_claimed; size_t _freed_bytes; - FreeRegionList _local_cleanup_list; - HumongousRegionSet _humongous_proxy_set; + FreeRegionList* _local_cleanup_list; + HumongousRegionSet* _humongous_proxy_set; + HRRSCleanupTask* _hrrs_cleanup_task; double _claimed_region_time; double _max_region_time; public: G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1, - int worker_num); + int worker_num, + FreeRegionList* local_cleanup_list, + HumongousRegionSet* humongous_proxy_set, + HRRSCleanupTask* hrrs_cleanup_task); size_t freed_bytes() { return _freed_bytes; } - FreeRegionList* local_cleanup_list() { - return &_local_cleanup_list; - } - HumongousRegionSet* humongous_proxy_set() { - return &_humongous_proxy_set; - } bool doHeapRegion(HeapRegion *r); @@ -1541,7 +1551,12 @@ void work(int i) { double start = os::elapsedTime(); - G1NoteEndOfConcMarkClosure g1_note_end(_g1h, i); + FreeRegionList local_cleanup_list("Local Cleanup List"); + HumongousRegionSet humongous_proxy_set("Local Cleanup Humongous Proxy Set"); + HRRSCleanupTask hrrs_cleanup_task; + G1NoteEndOfConcMarkClosure g1_note_end(_g1h, i, &local_cleanup_list, + &humongous_proxy_set, + &hrrs_cleanup_task); if (G1CollectedHeap::use_parallel_gc_threads()) { _g1h->heap_region_par_iterate_chunked(&g1_note_end, i, HeapRegion::NoteEndClaimValue); @@ -1553,15 +1568,17 @@ // Now update the lists _g1h->update_sets_after_freeing_regions(g1_note_end.freed_bytes(), NULL /* free_list */, - g1_note_end.humongous_proxy_set(), + &humongous_proxy_set, true /* par */); { MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); _max_live_bytes += g1_note_end.max_live_bytes(); _freed_bytes += g1_note_end.freed_bytes(); - _cleanup_list->add_as_tail(g1_note_end.local_cleanup_list()); - assert(g1_note_end.local_cleanup_list()->is_empty(), "post-condition"); + _cleanup_list->add_as_tail(&local_cleanup_list); + assert(local_cleanup_list.is_empty(), "post-condition"); + + HeapRegionRemSet::finish_cleanup_task(&hrrs_cleanup_task); } double end = os::elapsedTime(); if (G1PrintParCleanupStats) { @@ -1602,13 +1619,17 @@ G1NoteEndOfConcMarkClosure:: G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1, - int worker_num) + int worker_num, + FreeRegionList* local_cleanup_list, + HumongousRegionSet* humongous_proxy_set, + HRRSCleanupTask* hrrs_cleanup_task) : _g1(g1), _worker_num(worker_num), _max_live_bytes(0), _regions_claimed(0), _freed_bytes(0), _claimed_region_time(0.0), _max_region_time(0.0), - _local_cleanup_list("Local Cleanup List"), - _humongous_proxy_set("Local Cleanup Humongous Proxy Set") { } + _local_cleanup_list(local_cleanup_list), + _humongous_proxy_set(humongous_proxy_set), + _hrrs_cleanup_task(hrrs_cleanup_task) { } bool G1NoteEndOfConcMarkClosure::doHeapRegion(HeapRegion *hr) { // We use a claim value of zero here because all regions @@ -1619,11 +1640,12 @@ _regions_claimed++; hr->note_end_of_marking(); _max_live_bytes += hr->max_live_bytes(); - _g1->free_region_if_totally_empty(hr, - &_freed_bytes, - &_local_cleanup_list, - &_humongous_proxy_set, - true /* par */); + _g1->free_region_if_empty(hr, + &_freed_bytes, + _local_cleanup_list, + _humongous_proxy_set, + _hrrs_cleanup_task, + true /* par */); double region_time = (os::elapsedTime() - start); _claimed_region_time += region_time; if (region_time > _max_region_time) _max_region_time = region_time; @@ -1659,6 +1681,8 @@ double start = os::elapsedTime(); + HeapRegionRemSet::reset_for_cleanup_tasks(); + // Do counting once more with the world stopped for good measure. G1ParFinalCountTask g1_par_count_task(g1h, nextMarkBitMap(), &_region_bm, &_card_bm); @@ -1853,6 +1877,8 @@ assert(local_free_list.is_empty(), "post-condition"); } +// Support closures for reference procssing in G1 + bool G1CMIsAliveClosure::do_object_b(oop obj) { HeapWord* addr = (HeapWord*)obj; return addr != NULL && @@ -1873,11 +1899,17 @@ virtual void do_oop( oop* p) { do_oop_work(p); } template void do_oop_work(T* p) { - oop thisOop = oopDesc::load_decode_heap_oop(p); - HeapWord* addr = (HeapWord*)thisOop; - if (_g1->is_in_g1_reserved(addr) && _g1->is_obj_ill(thisOop)) { + oop obj = oopDesc::load_decode_heap_oop(p); + HeapWord* addr = (HeapWord*)obj; + + if (_cm->verbose_high()) + gclog_or_tty->print_cr("\t[0] we're looking at location " + "*"PTR_FORMAT" = "PTR_FORMAT, + p, (void*) obj); + + if (_g1->is_in_g1_reserved(addr) && _g1->is_obj_ill(obj)) { _bitMap->mark(addr); - _cm->mark_stack_push(thisOop); + _cm->mark_stack_push(obj); } } }; @@ -1899,6 +1931,199 @@ } }; +// 'Keep Alive' closure used by parallel reference processing. +// An instance of this closure is used in the parallel reference processing +// code rather than an instance of G1CMKeepAliveClosure. We could have used +// the G1CMKeepAliveClosure as it is MT-safe. Also reference objects are +// placed on to discovered ref lists once so we can mark and push with no +// need to check whether the object has already been marked. Using the +// G1CMKeepAliveClosure would mean, however, having all the worker threads +// operating on the global mark stack. This means that an individual +// worker would be doing lock-free pushes while it processes its own +// discovered ref list followed by drain call. If the discovered ref lists +// are unbalanced then this could cause interference with the other +// workers. Using a CMTask (and its embedded local data structures) +// avoids that potential interference. +class G1CMParKeepAliveAndDrainClosure: public OopClosure { + ConcurrentMark* _cm; + CMTask* _task; + CMBitMap* _bitMap; + int _ref_counter_limit; + int _ref_counter; + public: + G1CMParKeepAliveAndDrainClosure(ConcurrentMark* cm, + CMTask* task, + CMBitMap* bitMap) : + _cm(cm), _task(task), _bitMap(bitMap), + _ref_counter_limit(G1RefProcDrainInterval) + { + assert(_ref_counter_limit > 0, "sanity"); + _ref_counter = _ref_counter_limit; + } + + virtual void do_oop(narrowOop* p) { do_oop_work(p); } + virtual void do_oop( oop* p) { do_oop_work(p); } + + template void do_oop_work(T* p) { + if (!_cm->has_overflown()) { + oop obj = oopDesc::load_decode_heap_oop(p); + if (_cm->verbose_high()) + gclog_or_tty->print_cr("\t[%d] we're looking at location " + "*"PTR_FORMAT" = "PTR_FORMAT, + _task->task_id(), p, (void*) obj); + + _task->deal_with_reference(obj); + _ref_counter--; + + if (_ref_counter == 0) { + // We have dealt with _ref_counter_limit references, pushing them and objects + // reachable from them on to the local stack (and possibly the global stack). + // Call do_marking_step() to process these entries. We call the routine in a + // loop, which we'll exit if there's nothing more to do (i.e. we're done + // with the entries that we've pushed as a result of the deal_with_reference + // calls above) or we overflow. + // Note: CMTask::do_marking_step() can set the CMTask::has_aborted() flag + // while there may still be some work to do. (See the comment at the + // beginning of CMTask::do_marking_step() for those conditions - one of which + // is reaching the specified time target.) It is only when + // CMTask::do_marking_step() returns without setting the has_aborted() flag + // that the marking has completed. + do { + double mark_step_duration_ms = G1ConcMarkStepDurationMillis; + _task->do_marking_step(mark_step_duration_ms, + false /* do_stealing */, + false /* do_termination */); + } while (_task->has_aborted() && !_cm->has_overflown()); + _ref_counter = _ref_counter_limit; + } + } else { + if (_cm->verbose_high()) + gclog_or_tty->print_cr("\t[%d] CM Overflow", _task->task_id()); + } + } +}; + +class G1CMParDrainMarkingStackClosure: public VoidClosure { + ConcurrentMark* _cm; + CMTask* _task; + public: + G1CMParDrainMarkingStackClosure(ConcurrentMark* cm, CMTask* task) : + _cm(cm), _task(task) + {} + + void do_void() { + do { + if (_cm->verbose_high()) + gclog_or_tty->print_cr("\t[%d] Drain: Calling do marking_step", _task->task_id()); + + // We call CMTask::do_marking_step() to completely drain the local and + // global marking stacks. The routine is called in a loop, which we'll + // exit if there's nothing more to do (i.e. we'completely drained the + // entries that were pushed as a result of applying the + // G1CMParKeepAliveAndDrainClosure to the entries on the discovered ref + // lists above) or we overflow the global marking stack. + // Note: CMTask::do_marking_step() can set the CMTask::has_aborted() flag + // while there may still be some work to do. (See the comment at the + // beginning of CMTask::do_marking_step() for those conditions - one of which + // is reaching the specified time target.) It is only when + // CMTask::do_marking_step() returns without setting the has_aborted() flag + // that the marking has completed. + + _task->do_marking_step(1000000000.0 /* something very large */, + true /* do_stealing */, + true /* do_termination */); + } while (_task->has_aborted() && !_cm->has_overflown()); + } +}; + +// Implementation of AbstractRefProcTaskExecutor for G1 +class G1RefProcTaskExecutor: public AbstractRefProcTaskExecutor { +private: + G1CollectedHeap* _g1h; + ConcurrentMark* _cm; + CMBitMap* _bitmap; + WorkGang* _workers; + int _active_workers; + +public: + G1RefProcTaskExecutor(G1CollectedHeap* g1h, + ConcurrentMark* cm, + CMBitMap* bitmap, + WorkGang* workers, + int n_workers) : + _g1h(g1h), _cm(cm), _bitmap(bitmap), + _workers(workers), _active_workers(n_workers) + { } + + // Executes the given task using concurrent marking worker threads. + virtual void execute(ProcessTask& task); + virtual void execute(EnqueueTask& task); +}; + +class G1RefProcTaskProxy: public AbstractGangTask { + typedef AbstractRefProcTaskExecutor::ProcessTask ProcessTask; + ProcessTask& _proc_task; + G1CollectedHeap* _g1h; + ConcurrentMark* _cm; + CMBitMap* _bitmap; + +public: + G1RefProcTaskProxy(ProcessTask& proc_task, + G1CollectedHeap* g1h, + ConcurrentMark* cm, + CMBitMap* bitmap) : + AbstractGangTask("Process reference objects in parallel"), + _proc_task(proc_task), _g1h(g1h), _cm(cm), _bitmap(bitmap) + {} + + virtual void work(int i) { + CMTask* marking_task = _cm->task(i); + G1CMIsAliveClosure g1_is_alive(_g1h); + G1CMParKeepAliveAndDrainClosure g1_par_keep_alive(_cm, marking_task, _bitmap); + G1CMParDrainMarkingStackClosure g1_par_drain(_cm, marking_task); + + _proc_task.work(i, g1_is_alive, g1_par_keep_alive, g1_par_drain); + } +}; + +void G1RefProcTaskExecutor::execute(ProcessTask& proc_task) { + assert(_workers != NULL, "Need parallel worker threads."); + + G1RefProcTaskProxy proc_task_proxy(proc_task, _g1h, _cm, _bitmap); + + // We need to reset the phase for each task execution so that + // the termination protocol of CMTask::do_marking_step works. + _cm->set_phase(_active_workers, false /* concurrent */); + _g1h->set_par_threads(_active_workers); + _workers->run_task(&proc_task_proxy); + _g1h->set_par_threads(0); +} + +class G1RefEnqueueTaskProxy: public AbstractGangTask { + typedef AbstractRefProcTaskExecutor::EnqueueTask EnqueueTask; + EnqueueTask& _enq_task; + +public: + G1RefEnqueueTaskProxy(EnqueueTask& enq_task) : + AbstractGangTask("Enqueue reference objects in parallel"), + _enq_task(enq_task) + { } + + virtual void work(int i) { + _enq_task.work(i); + } +}; + +void G1RefProcTaskExecutor::execute(EnqueueTask& enq_task) { + assert(_workers != NULL, "Need parallel worker threads."); + + G1RefEnqueueTaskProxy enq_task_proxy(enq_task); + + _g1h->set_par_threads(_active_workers); + _workers->run_task(&enq_task_proxy); + _g1h->set_par_threads(0); +} + void ConcurrentMark::weakRefsWork(bool clear_all_soft_refs) { ResourceMark rm; HandleMark hm; @@ -1917,18 +2142,52 @@ G1CMDrainMarkingStackClosure g1_drain_mark_stack(nextMarkBitMap(), &_markStack, &g1_keep_alive); - // XXXYYY Also: copy the parallel ref processing code from CMS. - rp->process_discovered_references(&g1_is_alive, - &g1_keep_alive, - &g1_drain_mark_stack, - NULL); + // We use the work gang from the G1CollectedHeap and we utilize all + // the worker threads. + int active_workers = MAX2(MIN2(g1h->workers()->total_workers(), (int)_max_task_num), 1); + + G1RefProcTaskExecutor par_task_executor(g1h, this, nextMarkBitMap(), + g1h->workers(), active_workers); + + if (rp->processing_is_mt()) { + // Set the degree of MT here. If the discovery is done MT, there + // may have been a different number of threads doing the discovery + // and a different number of discovered lists may have Ref objects. + // That is OK as long as the Reference lists are balanced (see + // balance_all_queues() and balance_queues()). + rp->set_mt_degree(active_workers); + + rp->process_discovered_references(&g1_is_alive, + &g1_keep_alive, + &g1_drain_mark_stack, + &par_task_executor); + + // The work routines of the parallel keep_alive and drain_marking_stack + // will set the has_overflown flag if we overflow the global marking + // stack. + } else { + rp->process_discovered_references(&g1_is_alive, + &g1_keep_alive, + &g1_drain_mark_stack, + NULL); + + } + assert(_markStack.overflow() || _markStack.isEmpty(), - "mark stack should be empty (unless it overflowed)"); + "mark stack should be empty (unless it overflowed)"); if (_markStack.overflow()) { + // Should have been done already when we tried to push an + // entry on to the global mark stack. But let's do it again. set_has_overflown(); } - rp->enqueue_discovered_references(); + if (rp->processing_is_mt()) { + assert(rp->num_q() == active_workers, "why not"); + rp->enqueue_discovered_references(&par_task_executor); + } else { + rp->enqueue_discovered_references(); + } + rp->verify_no_references_recorded(); assert(!rp->discovery_enabled(), "should have been disabled"); @@ -1955,7 +2214,9 @@ CMTask* task = _cm->task(worker_i); task->record_start_time(); do { - task->do_marking_step(1000000000.0 /* something very large */); + task->do_marking_step(1000000000.0 /* something very large */, + true /* do_stealing */, + true /* do_termination */); } while (task->has_aborted() && !_cm->has_overflown()); // If we overflow, then we do not want to restart. We instead // want to abort remark and do concurrent marking again. @@ -1978,7 +2239,7 @@ G1CollectedHeap::StrongRootsScope srs(g1h); // this is remark, so we'll use up all available threads int active_workers = ParallelGCThreads; - set_phase(active_workers, false); + set_phase(active_workers, false /* concurrent */); CMRemarkTask remarkTask(this); // We will start all available threads, even if we decide that the @@ -1992,7 +2253,7 @@ G1CollectedHeap::StrongRootsScope srs(g1h); // this is remark, so we'll use up all available threads int active_workers = 1; - set_phase(active_workers, false); + set_phase(active_workers, false /* concurrent */); CMRemarkTask remarkTask(this); // We will start all available threads, even if we decide that the @@ -2005,9 +2266,6 @@ print_stats(); - if (!restart_for_overflow()) - set_non_marking_state(); - #if VERIFY_OBJS_PROCESSED if (_scan_obj_cl.objs_processed != ThreadLocalObjQueue::objs_enqueued) { gclog_or_tty->print_cr("Processed = %d, enqueued = %d.", @@ -3124,7 +3382,7 @@ // do nothing } #else // _CHECK_BOTH_FINGERS_ - // we will only check the global finger + // we will only check the global finger if (objAddr < global_finger) { // see long comment above @@ -3249,7 +3507,7 @@ double elapsed_time_ms = curr_time_ms - _start_time_ms; if (elapsed_time_ms > _time_target_ms) { set_has_aborted(); - _has_aborted_timed_out = true; + _has_timed_out = true; statsOnly( ++_aborted_timed_out ); return; } @@ -3754,7 +4012,9 @@ *****************************************************************************/ -void CMTask::do_marking_step(double time_target_ms) { +void CMTask::do_marking_step(double time_target_ms, + bool do_stealing, + bool do_termination) { assert(time_target_ms >= 1.0, "minimum granularity is 1ms"); assert(concurrent() == _cm->concurrent(), "they should be the same"); @@ -3794,7 +4054,7 @@ // clear all flags clear_has_aborted(); - _has_aborted_timed_out = false; + _has_timed_out = false; _draining_satb_buffers = false; ++_calls; @@ -3970,7 +4230,7 @@ drain_global_stack(false); // Attempt at work stealing from other task's queues. - if (!has_aborted()) { + if (do_stealing && !has_aborted()) { // We have not aborted. This means that we have finished all that // we could. Let's try to do some stealing... @@ -4011,7 +4271,7 @@ // We still haven't aborted. Now, let's try to get into the // termination protocol. - if (!has_aborted()) { + if (do_termination && !has_aborted()) { // We cannot check whether the global stack is empty, since other // tasks might be concurrently pushing objects on it. We also cannot // check if the region stack is empty because if a thread is aborting @@ -4087,7 +4347,7 @@ statsOnly( ++_aborted ); - if (_has_aborted_timed_out) { + if (_has_timed_out) { double diff_ms = elapsed_time_ms - _time_target_ms; // Keep statistics of how well we did with respect to hitting // our target only if we actually timed out (if we aborted for diff -r 9846d99e16d3 -r 27e4ea99855d src/share/vm/gc_implementation/g1/concurrentMark.hpp --- a/src/share/vm/gc_implementation/g1/concurrentMark.hpp Thu Jan 27 14:05:59 2011 -0500 +++ b/src/share/vm/gc_implementation/g1/concurrentMark.hpp Thu Jan 27 13:42:28 2011 -0800 @@ -353,6 +353,10 @@ friend class CMConcurrentMarkingTask; friend class G1ParNoteEndTask; friend class CalcLiveObjectsClosure; + friend class G1RefProcTaskProxy; + friend class G1RefProcTaskExecutor; + friend class G1CMParKeepAliveAndDrainClosure; + friend class G1CMParDrainMarkingStackClosure; protected: ConcurrentMarkThread* _cmThread; // the thread doing the work @@ -936,7 +940,7 @@ // if this is true, then the task has aborted for some reason bool _has_aborted; // set when the task aborts because it has met its time quota - bool _has_aborted_timed_out; + bool _has_timed_out; // true when we're draining SATB buffers; this avoids the task // aborting due to SATB buffers being available (as we're already // dealing with them) @@ -1041,7 +1045,7 @@ // trying not to exceed the given duration. However, it might exit // prematurely, according to some conditions (i.e. SATB buffers are // available for processing). - void do_marking_step(double target_ms); + void do_marking_step(double target_ms, bool do_stealing, bool do_termination); // These two calls start and stop the timer void record_start_time() { @@ -1063,7 +1067,8 @@ bool has_aborted() { return _has_aborted; } void set_has_aborted() { _has_aborted = true; } void clear_has_aborted() { _has_aborted = false; } - bool claimed() { return _claimed; } + bool has_timed_out() { return _has_timed_out; } + bool claimed() { return _claimed; } // Support routines for the partially scanned region that may be // recorded as a result of aborting while draining the CMRegionStack diff -r 9846d99e16d3 -r 27e4ea99855d src/share/vm/gc_implementation/g1/concurrentMarkThread.cpp --- a/src/share/vm/gc_implementation/g1/concurrentMarkThread.cpp Thu Jan 27 14:05:59 2011 -0500 +++ b/src/share/vm/gc_implementation/g1/concurrentMarkThread.cpp Thu Jan 27 13:42:28 2011 -0800 @@ -251,7 +251,9 @@ // Now do the remainder of the cleanup operation. _cm->completeCleanup(); + _sts.join(); g1_policy->record_concurrent_mark_cleanup_completed(); + _sts.leave(); double cleanup_end_sec = os::elapsedTime(); if (PrintGC) { diff -r 9846d99e16d3 -r 27e4ea99855d src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Thu Jan 27 14:05:59 2011 -0500 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Thu Jan 27 13:42:28 2011 -0800 @@ -4925,10 +4925,11 @@ COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); } -void G1CollectedHeap::free_region_if_totally_empty(HeapRegion* hr, +void G1CollectedHeap::free_region_if_empty(HeapRegion* hr, size_t* pre_used, FreeRegionList* free_list, HumongousRegionSet* humongous_proxy_set, + HRRSCleanupTask* hrrs_cleanup_task, bool par) { if (hr->used() > 0 && hr->max_live_bytes() == 0 && !hr->is_young()) { if (hr->isHumongous()) { @@ -4937,6 +4938,8 @@ } else { free_region(hr, pre_used, free_list, par); } + } else { + hr->rem_set()->do_cleanup_work(hrrs_cleanup_task); } } diff -r 9846d99e16d3 -r 27e4ea99855d src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp --- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp Thu Jan 27 14:05:59 2011 -0500 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.hpp Thu Jan 27 13:42:28 2011 -0800 @@ -40,6 +40,7 @@ class HeapRegion; class HeapRegionSeq; +class HRRSCleanupTask; class PermanentGenerationSpec; class GenerationSpec; class OopsInHeapRegionClosure; @@ -1099,11 +1100,12 @@ // all dead. It calls either free_region() or // free_humongous_region() depending on the type of the region that // is passed to it. - void free_region_if_totally_empty(HeapRegion* hr, - size_t* pre_used, - FreeRegionList* free_list, - HumongousRegionSet* humongous_proxy_set, - bool par); + void free_region_if_empty(HeapRegion* hr, + size_t* pre_used, + FreeRegionList* free_list, + HumongousRegionSet* humongous_proxy_set, + HRRSCleanupTask* hrrs_cleanup_task, + bool par); // It appends the free list to the master free list and updates the // master humongous list according to the contents of the proxy diff -r 9846d99e16d3 -r 27e4ea99855d src/share/vm/gc_implementation/g1/g1_globals.hpp --- a/src/share/vm/gc_implementation/g1/g1_globals.hpp Thu Jan 27 14:05:59 2011 -0500 +++ b/src/share/vm/gc_implementation/g1/g1_globals.hpp Thu Jan 27 13:42:28 2011 -0800 @@ -81,6 +81,14 @@ product(intx, G1MarkRegionStackSize, 1024 * 1024, \ "Size of the region stack for concurrent marking.") \ \ + product(double, G1ConcMarkStepDurationMillis, 10.0, \ + "Target duration of individual concurrent marking steps " \ + "in milliseconds.") \ + \ + product(intx, G1RefProcDrainInterval, 10, \ + "The number of discovered reference objects to process before " \ + "draining concurrent marking work queues.") \ + \ develop(bool, G1SATBBarrierPrintNullPreVals, false, \ "If true, count frac of ptr writes with null pre-vals.") \ \ diff -r 9846d99e16d3 -r 27e4ea99855d src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp --- a/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp Thu Jan 27 14:05:59 2011 -0500 +++ b/src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp Thu Jan 27 13:42:28 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -463,7 +463,6 @@ } static void par_contract_all(); - }; void PosParPRT::par_contract_all() { @@ -1070,6 +1069,11 @@ } +void +OtherRegionsTable::do_cleanup_work(HRRSCleanupTask* hrrs_cleanup_task) { + _sparse_table.do_cleanup_work(hrrs_cleanup_task); +} + // Determines how many threads can add records to an rset in parallel. // This can be done by either mutator threads together with the // concurrent refinement threads or GC threads. @@ -1384,6 +1388,19 @@ } } +void HeapRegionRemSet::reset_for_cleanup_tasks() { + SparsePRT::reset_for_cleanup_tasks(); +} + +void HeapRegionRemSet::do_cleanup_work(HRRSCleanupTask* hrrs_cleanup_task) { + _other_regions.do_cleanup_work(hrrs_cleanup_task); +} + +void +HeapRegionRemSet::finish_cleanup_task(HRRSCleanupTask* hrrs_cleanup_task) { + SparsePRT::finish_cleanup_task(hrrs_cleanup_task); +} + #ifndef PRODUCT void HeapRegionRemSet::test() { os::sleep(Thread::current(), (jlong)5000, false); diff -r 9846d99e16d3 -r 27e4ea99855d src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp --- a/src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp Thu Jan 27 14:05:59 2011 -0500 +++ b/src/share/vm/gc_implementation/g1/heapRegionRemSet.hpp Thu Jan 27 13:42:28 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -38,6 +38,10 @@ class PosParPRT; class SparsePRT; +// Essentially a wrapper around SparsePRTCleanupTask. See +// sparsePRT.hpp for more details. +class HRRSCleanupTask : public SparsePRTCleanupTask { +}; // The "_coarse_map" is a bitmap with one bit for each region, where set // bits indicate that the corresponding region may contain some pointer @@ -156,6 +160,8 @@ // "from_hr" is being cleared; remove any entries from it. void clear_incoming_entry(HeapRegion* from_hr); + void do_cleanup_work(HRRSCleanupTask* hrrs_cleanup_task); + // Declare the heap size (in # of regions) to the OtherRegionsTable. // (Uses it to initialize from_card_cache). static void init_from_card_cache(size_t max_regions); @@ -165,10 +171,8 @@ static void shrink_from_card_cache(size_t new_n_regs); static void print_from_card_cache(); - }; - class HeapRegionRemSet : public CHeapObj { friend class VMStructs; friend class HeapRegionRemSetIterator; @@ -342,11 +346,16 @@ static void print_recorded(); static void record_event(Event evnt); + // These are wrappers for the similarly-named methods on + // SparsePRT. Look at sparsePRT.hpp for more details. + static void reset_for_cleanup_tasks(); + void do_cleanup_work(HRRSCleanupTask* hrrs_cleanup_task); + static void finish_cleanup_task(HRRSCleanupTask* hrrs_cleanup_task); + // Run unit tests. #ifndef PRODUCT static void test(); #endif - }; class HeapRegionRemSetIterator : public CHeapObj { diff -r 9846d99e16d3 -r 27e4ea99855d src/share/vm/gc_implementation/g1/heapRegionSeq.cpp --- a/src/share/vm/gc_implementation/g1/heapRegionSeq.cpp Thu Jan 27 14:05:59 2011 -0500 +++ b/src/share/vm/gc_implementation/g1/heapRegionSeq.cpp Thu Jan 27 13:42:28 2011 -0800 @@ -195,10 +195,10 @@ assert(0 <= res && res < _regions.length(), err_msg("res: %d should be valid", res)); _alloc_search_start = res + (int) num; + assert(0 < _alloc_search_start && _alloc_search_start <= _regions.length(), + err_msg("_alloc_search_start: %d should be valid", + _alloc_search_start)); } - assert(0 < _alloc_search_start && _alloc_search_start <= _regions.length(), - err_msg("_alloc_search_start: %d should be valid", - _alloc_search_start)); return res; } diff -r 9846d99e16d3 -r 27e4ea99855d src/share/vm/gc_implementation/g1/sparsePRT.cpp --- a/src/share/vm/gc_implementation/g1/sparsePRT.cpp Thu Jan 27 14:05:59 2011 -0500 +++ b/src/share/vm/gc_implementation/g1/sparsePRT.cpp Thu Jan 27 13:42:28 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -415,6 +415,38 @@ return NULL; } +void SparsePRT::reset_for_cleanup_tasks() { + _head_expanded_list = NULL; +} + +void SparsePRT::do_cleanup_work(SparsePRTCleanupTask* sprt_cleanup_task) { + if (should_be_on_expanded_list()) { + sprt_cleanup_task->add(this); + } +} + +void SparsePRT::finish_cleanup_task(SparsePRTCleanupTask* sprt_cleanup_task) { + assert(ParGCRareEvent_lock->owned_by_self(), "pre-condition"); + SparsePRT* head = sprt_cleanup_task->head(); + SparsePRT* tail = sprt_cleanup_task->tail(); + if (head != NULL) { + assert(tail != NULL, "if head is not NULL, so should tail"); + + tail->set_next_expanded(_head_expanded_list); + _head_expanded_list = head; + } else { + assert(tail == NULL, "if head is NULL, so should tail"); + } +} + +bool SparsePRT::should_be_on_expanded_list() { + if (_expanded) { + assert(_cur != _next, "if _expanded is true, cur should be != _next"); + } else { + assert(_cur == _next, "if _expanded is false, cur should be == _next"); + } + return expanded(); +} void SparsePRT::cleanup_all() { // First clean up all expanded tables so they agree on next and cur. @@ -484,6 +516,7 @@ _cur->clear(); } _next = _cur; + _expanded = false; } void SparsePRT::cleanup() { @@ -518,3 +551,15 @@ } add_to_expanded_list(this); } + +void SparsePRTCleanupTask::add(SparsePRT* sprt) { + assert(sprt->should_be_on_expanded_list(), "pre-condition"); + + sprt->set_next_expanded(NULL); + if (_tail != NULL) { + _tail->set_next_expanded(sprt); + } else { + _head = sprt; + } + _tail = sprt; +} diff -r 9846d99e16d3 -r 27e4ea99855d src/share/vm/gc_implementation/g1/sparsePRT.hpp --- a/src/share/vm/gc_implementation/g1/sparsePRT.hpp Thu Jan 27 14:05:59 2011 -0500 +++ b/src/share/vm/gc_implementation/g1/sparsePRT.hpp Thu Jan 27 13:42:28 2011 -0800 @@ -1,5 +1,5 @@ /* - * Copyright (c) 2001, 2010, Oracle and/or its affiliates. All rights reserved. + * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. * * This code is free software; you can redistribute it and/or modify it @@ -212,8 +212,11 @@ // mutex. class SparsePRTIter; +class SparsePRTCleanupTask; class SparsePRT VALUE_OBJ_CLASS_SPEC { + friend class SparsePRTCleanupTask; + // Iterations are done on the _cur hash table, since they only need to // see entries visible at the start of a collection pause. // All other operations are done using the _next hash table. @@ -238,6 +241,8 @@ SparsePRT* next_expanded() { return _next_expanded; } void set_next_expanded(SparsePRT* nxt) { _next_expanded = nxt; } + bool should_be_on_expanded_list(); + static SparsePRT* _head_expanded_list; public: @@ -284,12 +289,36 @@ static void add_to_expanded_list(SparsePRT* sprt); static SparsePRT* get_from_expanded_list(); + // The purpose of these three methods is to help the GC workers + // during the cleanup pause to recreate the expanded list, purging + // any tables from it that belong to regions that are freed during + // cleanup (if we don't purge those tables, there is a race that + // causes various crashes; see CR 7014261). + // + // We chose to recreate the expanded list, instead of purging + // entries from it by iterating over it, to avoid this serial phase + // at the end of the cleanup pause. + // + // The three methods below work as follows: + // * reset_for_cleanup_tasks() : Nulls the expanded list head at the + // start of the cleanup pause. + // * do_cleanup_work() : Called by the cleanup workers for every + // region that is not free / is being freed by the cleanup + // pause. It creates a list of expanded tables whose head / tail + // are on the thread-local SparsePRTCleanupTask object. + // * finish_cleanup_task() : Called by the cleanup workers after + // they complete their cleanup task. It adds the local list into + // the global expanded list. It assumes that the + // ParGCRareEvent_lock is being held to ensure MT-safety. + static void reset_for_cleanup_tasks(); + void do_cleanup_work(SparsePRTCleanupTask* sprt_cleanup_task); + static void finish_cleanup_task(SparsePRTCleanupTask* sprt_cleanup_task); + bool contains_card(RegionIdx_t region_id, CardIdx_t card_index) const { return _next->contains_card(region_id, card_index); } }; - class SparsePRTIter: public RSHashTableIter { public: void init(const SparsePRT* sprt) { @@ -300,4 +329,22 @@ } }; +// This allows each worker during a cleanup pause to create a +// thread-local list of sparse tables that have been expanded and need +// to be processed at the beginning of the next GC pause. This lists +// are concatenated into the single expanded list at the end of the +// cleanup pause. +class SparsePRTCleanupTask VALUE_OBJ_CLASS_SPEC { +private: + SparsePRT* _head; + SparsePRT* _tail; + +public: + SparsePRTCleanupTask() : _head(NULL), _tail(NULL) { } + + void add(SparsePRT* sprt); + SparsePRT* head() { return _head; } + SparsePRT* tail() { return _tail; } +}; + #endif // SHARE_VM_GC_IMPLEMENTATION_G1_SPARSEPRT_HPP diff -r 9846d99e16d3 -r 27e4ea99855d src/share/vm/runtime/arguments.cpp --- a/src/share/vm/runtime/arguments.cpp Thu Jan 27 14:05:59 2011 -0500 +++ b/src/share/vm/runtime/arguments.cpp Thu Jan 27 13:42:28 2011 -0800 @@ -1941,10 +1941,16 @@ status = false; } +#ifndef SERIALGC if (UseG1GC) { status = status && verify_percentage(InitiatingHeapOccupancyPercent, "InitiatingHeapOccupancyPercent"); + status = status && verify_min_value(G1RefProcDrainInterval, 1, + "G1RefProcDrainInterval"); + status = status && verify_min_value((intx)G1ConcMarkStepDurationMillis, 1, + "G1ConcMarkStepDurationMillis"); } +#endif status = status && verify_interval(RefDiscoveryPolicy, ReferenceProcessor::DiscoveryPolicyMin,