Mercurial > hg > truffle
changeset 1111:44f61c24ddab
6862387: tune concurrent refinement further
Summary: Reworked the concurrent refinement: threads activation, feedback-based threshold adjustment, other miscellaneous fixes.
Reviewed-by: apetrusenko, tonyp
line wrap: on
line diff
--- a/src/share/vm/gc_implementation/g1/concurrentG1Refine.cpp Fri Dec 11 09:30:48 2009 -0800 +++ b/src/share/vm/gc_implementation/g1/concurrentG1Refine.cpp Wed Dec 16 15:12:51 2009 -0800 @@ -42,28 +42,49 @@ _n_periods(0), _threads(NULL), _n_threads(0) { - if (G1ConcRefine) { - _n_threads = (int)thread_num(); - if (_n_threads > 0) { - _threads = NEW_C_HEAP_ARRAY(ConcurrentG1RefineThread*, _n_threads); - int worker_id_offset = (int)DirtyCardQueueSet::num_par_ids(); - ConcurrentG1RefineThread *next = NULL; - for (int i = _n_threads - 1; i >= 0; i--) { - ConcurrentG1RefineThread* t = new ConcurrentG1RefineThread(this, next, worker_id_offset, i); - assert(t != NULL, "Conc refine should have been created"); - assert(t->cg1r() == this, "Conc refine thread should refer to this"); - _threads[i] = t; - next = t; - } - } + + // Ergomonically select initial concurrent refinement parameters + if (FLAG_IS_DEFAULT(G1ConcRefineGreenZone)) { + FLAG_SET_DEFAULT(G1ConcRefineGreenZone, MAX2<int>(ParallelGCThreads, 1)); + } + set_green_zone(G1ConcRefineGreenZone); + + if (FLAG_IS_DEFAULT(G1ConcRefineYellowZone)) { + FLAG_SET_DEFAULT(G1ConcRefineYellowZone, green_zone() * 3); + } + set_yellow_zone(MAX2<int>(G1ConcRefineYellowZone, green_zone())); + + if (FLAG_IS_DEFAULT(G1ConcRefineRedZone)) { + FLAG_SET_DEFAULT(G1ConcRefineRedZone, yellow_zone() * 2); + } + set_red_zone(MAX2<int>(G1ConcRefineRedZone, yellow_zone())); + _n_worker_threads = thread_num(); + // We need one extra thread to do the young gen rset size sampling. + _n_threads = _n_worker_threads + 1; + reset_threshold_step(); + + _threads = NEW_C_HEAP_ARRAY(ConcurrentG1RefineThread*, _n_threads); + int worker_id_offset = (int)DirtyCardQueueSet::num_par_ids(); + ConcurrentG1RefineThread *next = NULL; + for (int i = _n_threads - 1; i >= 0; i--) { + ConcurrentG1RefineThread* t = new ConcurrentG1RefineThread(this, next, worker_id_offset, i); + assert(t != NULL, "Conc refine should have been created"); + assert(t->cg1r() == this, "Conc refine thread should refer to this"); + _threads[i] = t; + next = t; } } -size_t ConcurrentG1Refine::thread_num() { - if (G1ConcRefine) { - return (G1ParallelRSetThreads > 0) ? G1ParallelRSetThreads : ParallelGCThreads; +void ConcurrentG1Refine::reset_threshold_step() { + if (FLAG_IS_DEFAULT(G1ConcRefineThresholdStep)) { + _thread_threshold_step = (yellow_zone() - green_zone()) / (worker_thread_num() + 1); + } else { + _thread_threshold_step = G1ConcRefineThresholdStep; } - return 0; +} + +int ConcurrentG1Refine::thread_num() { + return MAX2<int>((G1ParallelRSetThreads > 0) ? G1ParallelRSetThreads : ParallelGCThreads, 1); } void ConcurrentG1Refine::init() { @@ -123,6 +144,15 @@ } } +void ConcurrentG1Refine::reinitialize_threads() { + reset_threshold_step(); + if (_threads != NULL) { + for (int i = 0; i < _n_threads; i++) { + _threads[i]->initialize(); + } + } +} + ConcurrentG1Refine::~ConcurrentG1Refine() { if (G1ConcRSLogCacheSize > 0) { assert(_card_counts != NULL, "Logic"); @@ -384,4 +414,3 @@ st->cr(); } } -
--- a/src/share/vm/gc_implementation/g1/concurrentG1Refine.hpp Fri Dec 11 09:30:48 2009 -0800 +++ b/src/share/vm/gc_implementation/g1/concurrentG1Refine.hpp Wed Dec 16 15:12:51 2009 -0800 @@ -29,6 +29,31 @@ class ConcurrentG1Refine: public CHeapObj { ConcurrentG1RefineThread** _threads; int _n_threads; + int _n_worker_threads; + /* + * The value of the update buffer queue length falls into one of 3 zones: + * green, yellow, red. If the value is in [0, green) nothing is + * done, the buffers are left unprocessed to enable the caching effect of the + * dirtied cards. In the yellow zone [green, yellow) the concurrent refinement + * threads are gradually activated. In [yellow, red) all threads are + * running. If the length becomes red (max queue length) the mutators start + * processing the buffers. + * + * There are some interesting cases (with G1AdaptiveConcRefine turned off): + * 1) green = yellow = red = 0. In this case the mutator will process all + * buffers. Except for those that are created by the deferred updates + * machinery during a collection. + * 2) green = 0. Means no caching. Can be a good way to minimize the + * amount of time spent updating rsets during a collection. + */ + int _green_zone; + int _yellow_zone; + int _red_zone; + + int _thread_threshold_step; + + // Reset the threshold step value based of the current zone boundaries. + void reset_threshold_step(); // The cache for card refinement. bool _use_cache; @@ -147,6 +172,8 @@ void init(); // Accomplish some initialization that has to wait. void stop(); + void reinitialize_threads(); + // Iterate over the conc refine threads void threads_do(ThreadClosure *tc); @@ -178,7 +205,20 @@ void clear_and_record_card_counts(); - static size_t thread_num(); + static int thread_num(); void print_worker_threads_on(outputStream* st) const; + + void set_green_zone(int x) { _green_zone = x; } + void set_yellow_zone(int x) { _yellow_zone = x; } + void set_red_zone(int x) { _red_zone = x; } + + int green_zone() const { return _green_zone; } + int yellow_zone() const { return _yellow_zone; } + int red_zone() const { return _red_zone; } + + int total_thread_num() const { return _n_threads; } + int worker_thread_num() const { return _n_worker_threads; } + + int thread_threshold_step() const { return _thread_threshold_step; } };
--- a/src/share/vm/gc_implementation/g1/concurrentG1RefineThread.cpp Fri Dec 11 09:30:48 2009 -0800 +++ b/src/share/vm/gc_implementation/g1/concurrentG1RefineThread.cpp Wed Dec 16 15:12:51 2009 -0800 @@ -25,10 +25,6 @@ #include "incls/_precompiled.incl" #include "incls/_concurrentG1RefineThread.cpp.incl" -// ======= Concurrent Mark Thread ======== - -// The CM thread is created when the G1 garbage collector is used - ConcurrentG1RefineThread:: ConcurrentG1RefineThread(ConcurrentG1Refine* cg1r, ConcurrentG1RefineThread *next, int worker_id_offset, int worker_id) : @@ -37,19 +33,42 @@ _worker_id(worker_id), _active(false), _next(next), + _monitor(NULL), _cg1r(cg1r), - _vtime_accum(0.0), - _interval_ms(5.0) + _vtime_accum(0.0) { + + // Each thread has its own monitor. The i-th thread is responsible for signalling + // to thread i+1 if the number of buffers in the queue exceeds a threashold for this + // thread. Monitors are also used to wake up the threads during termination. + // The 0th worker in notified by mutator threads and has a special monitor. + // The last worker is used for young gen rset size sampling. + if (worker_id > 0) { + _monitor = new Monitor(Mutex::nonleaf, "Refinement monitor", true); + } else { + _monitor = DirtyCardQ_CBL_mon; + } + initialize(); create_and_start(); } +void ConcurrentG1RefineThread::initialize() { + if (_worker_id < cg1r()->worker_thread_num()) { + // Current thread activation threshold + _threshold = MIN2<int>(cg1r()->thread_threshold_step() * (_worker_id + 1) + cg1r()->green_zone(), + cg1r()->yellow_zone()); + // A thread deactivates once the number of buffer reached a deactivation threshold + _deactivation_threshold = MAX2<int>(_threshold - cg1r()->thread_threshold_step(), cg1r()->green_zone()); + } else { + set_active(true); + } +} + void ConcurrentG1RefineThread::sample_young_list_rs_lengths() { G1CollectedHeap* g1h = G1CollectedHeap::heap(); G1CollectorPolicy* g1p = g1h->g1_policy(); if (g1p->adaptive_young_list_length()) { int regions_visited = 0; - g1h->young_list_rs_length_sampling_init(); while (g1h->young_list_rs_length_sampling_more()) { g1h->young_list_rs_length_sampling_next(); @@ -70,99 +89,121 @@ } } -void ConcurrentG1RefineThread::run() { - initialize_in_thread(); +void ConcurrentG1RefineThread::run_young_rs_sampling() { + DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); _vtime_start = os::elapsedVTime(); - wait_for_universe_init(); + while(!_should_terminate) { + _sts.join(); + sample_young_list_rs_lengths(); + _sts.leave(); - while (!_should_terminate) { - DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); - // Wait for completed log buffers to exist. - { - MutexLockerEx x(DirtyCardQ_CBL_mon, Mutex::_no_safepoint_check_flag); - while (((_worker_id == 0 && !dcqs.process_completed_buffers()) || - (_worker_id > 0 && !is_active())) && - !_should_terminate) { - DirtyCardQ_CBL_mon->wait(Mutex::_no_safepoint_check_flag); - } - } - - if (_should_terminate) { - return; + if (os::supports_vtime()) { + _vtime_accum = (os::elapsedVTime() - _vtime_start); + } else { + _vtime_accum = 0.0; } - // Now we take them off (this doesn't hold locks while it applies - // closures.) (If we did a full collection, then we'll do a full - // traversal. - _sts.join(); - int n_logs = 0; - int lower_limit = 0; - double start_vtime_sec; // only used when G1SmoothConcRefine is on - int prev_buffer_num; // only used when G1SmoothConcRefine is on - // This thread activation threshold - int threshold = G1UpdateBufferQueueProcessingThreshold * _worker_id; - // Next thread activation threshold - int next_threshold = threshold + G1UpdateBufferQueueProcessingThreshold; - int deactivation_threshold = MAX2<int>(threshold - G1UpdateBufferQueueProcessingThreshold / 2, 0); + MutexLockerEx x(_monitor, Mutex::_no_safepoint_check_flag); + if (_should_terminate) { + break; + } + _monitor->wait(Mutex::_no_safepoint_check_flag, G1ConcRefineServiceInterval); + } +} - if (G1SmoothConcRefine) { - lower_limit = 0; - start_vtime_sec = os::elapsedVTime(); - prev_buffer_num = (int) dcqs.completed_buffers_num(); - } else { - lower_limit = G1UpdateBufferQueueProcessingThreshold / 4; // For now. +void ConcurrentG1RefineThread::wait_for_completed_buffers() { + DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); + MutexLockerEx x(_monitor, Mutex::_no_safepoint_check_flag); + while (!_should_terminate && !is_active()) { + _monitor->wait(Mutex::_no_safepoint_check_flag); + } +} + +bool ConcurrentG1RefineThread::is_active() { + DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); + return _worker_id > 0 ? _active : dcqs.process_completed_buffers(); +} + +void ConcurrentG1RefineThread::activate() { + MutexLockerEx x(_monitor, Mutex::_no_safepoint_check_flag); + if (_worker_id > 0) { + if (G1TraceConcurrentRefinement) { + DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); + gclog_or_tty->print_cr("G1-Refine-activated worker %d, on threshold %d, current %d", + _worker_id, _threshold, (int)dcqs.completed_buffers_num()); } - while (dcqs.apply_closure_to_completed_buffer(_worker_id + _worker_id_offset, lower_limit)) { - double end_vtime_sec; - double elapsed_vtime_sec; - int elapsed_vtime_ms; - int curr_buffer_num = (int) dcqs.completed_buffers_num(); + set_active(true); + } else { + DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); + dcqs.set_process_completed(true); + } + _monitor->notify(); +} - if (G1SmoothConcRefine) { - end_vtime_sec = os::elapsedVTime(); - elapsed_vtime_sec = end_vtime_sec - start_vtime_sec; - elapsed_vtime_ms = (int) (elapsed_vtime_sec * 1000.0); +void ConcurrentG1RefineThread::deactivate() { + MutexLockerEx x(_monitor, Mutex::_no_safepoint_check_flag); + if (_worker_id > 0) { + if (G1TraceConcurrentRefinement) { + DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); + gclog_or_tty->print_cr("G1-Refine-deactivated worker %d, off threshold %d, current %d", + _worker_id, _deactivation_threshold, (int)dcqs.completed_buffers_num()); + } + set_active(false); + } else { + DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); + dcqs.set_process_completed(false); + } +} + +void ConcurrentG1RefineThread::run() { + initialize_in_thread(); + wait_for_universe_init(); - if (curr_buffer_num > prev_buffer_num || - curr_buffer_num > next_threshold) { - decreaseInterval(elapsed_vtime_ms); - } else if (curr_buffer_num < prev_buffer_num) { - increaseInterval(elapsed_vtime_ms); - } + if (_worker_id >= cg1r()->worker_thread_num()) { + run_young_rs_sampling(); + terminate(); + } + + _vtime_start = os::elapsedVTime(); + while (!_should_terminate) { + DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); + + // Wait for work + wait_for_completed_buffers(); + + if (_should_terminate) { + break; + } + + _sts.join(); + + do { + int curr_buffer_num = (int)dcqs.completed_buffers_num(); + // If the number of the buffers falls down into the yellow zone, + // that means that the transition period after the evacuation pause has ended. + if (dcqs.completed_queue_padding() > 0 && curr_buffer_num <= cg1r()->yellow_zone()) { + dcqs.set_completed_queue_padding(0); } - if (_worker_id == 0) { - sample_young_list_rs_lengths(); - } else if (curr_buffer_num < deactivation_threshold) { + + if (_worker_id > 0 && curr_buffer_num <= _deactivation_threshold) { // If the number of the buffer has fallen below our threshold // we should deactivate. The predecessor will reactivate this // thread should the number of the buffers cross the threshold again. - MutexLockerEx x(DirtyCardQ_CBL_mon, Mutex::_no_safepoint_check_flag); deactivate(); - if (G1TraceConcurrentRefinement) { - gclog_or_tty->print_cr("G1-Refine-deactivated worker %d", _worker_id); - } break; } // Check if we need to activate the next thread. - if (curr_buffer_num > next_threshold && _next != NULL && !_next->is_active()) { - MutexLockerEx x(DirtyCardQ_CBL_mon, Mutex::_no_safepoint_check_flag); + if (_next != NULL && !_next->is_active() && curr_buffer_num > _next->_threshold) { _next->activate(); - DirtyCardQ_CBL_mon->notify_all(); - if (G1TraceConcurrentRefinement) { - gclog_or_tty->print_cr("G1-Refine-activated worker %d", _next->_worker_id); - } } + } while (dcqs.apply_closure_to_completed_buffer(_worker_id + _worker_id_offset, cg1r()->green_zone())); - if (G1SmoothConcRefine) { - prev_buffer_num = curr_buffer_num; - _sts.leave(); - os::sleep(Thread::current(), (jlong) _interval_ms, false); - _sts.join(); - start_vtime_sec = os::elapsedVTime(); - } - n_logs++; + // We can exit the loop above while being active if there was a yield request. + if (is_active()) { + deactivate(); } + _sts.leave(); if (os::supports_vtime()) { @@ -172,7 +213,6 @@ } } assert(_should_terminate, "just checking"); - terminate(); } @@ -191,8 +231,8 @@ } { - MutexLockerEx x(DirtyCardQ_CBL_mon, Mutex::_no_safepoint_check_flag); - DirtyCardQ_CBL_mon->notify_all(); + MutexLockerEx x(_monitor, Mutex::_no_safepoint_check_flag); + _monitor->notify(); } {
--- a/src/share/vm/gc_implementation/g1/concurrentG1RefineThread.hpp Fri Dec 11 09:30:48 2009 -0800 +++ b/src/share/vm/gc_implementation/g1/concurrentG1RefineThread.hpp Wed Dec 16 15:12:51 2009 -0800 @@ -40,42 +40,36 @@ // when the number of the rset update buffer crosses a certain threshold. A successor // would self-deactivate when the number of the buffers falls below the threshold. bool _active; - ConcurrentG1RefineThread * _next; - public: - virtual void run(); - - bool is_active() { return _active; } - void activate() { _active = true; } - void deactivate() { _active = false; } - - private: - ConcurrentG1Refine* _cg1r; - - double _interval_ms; + ConcurrentG1RefineThread* _next; + Monitor* _monitor; + ConcurrentG1Refine* _cg1r; - void decreaseInterval(int processing_time_ms) { - double min_interval_ms = (double) processing_time_ms; - _interval_ms = 0.8 * _interval_ms; - if (_interval_ms < min_interval_ms) - _interval_ms = min_interval_ms; - } - void increaseInterval(int processing_time_ms) { - double max_interval_ms = 9.0 * (double) processing_time_ms; - _interval_ms = 1.1 * _interval_ms; - if (max_interval_ms > 0 && _interval_ms > max_interval_ms) - _interval_ms = max_interval_ms; - } + int _thread_threshold_step; + // This thread activation threshold + int _threshold; + // This thread deactivation threshold + int _deactivation_threshold; - void sleepBeforeNextCycle(); + void sample_young_list_rs_lengths(); + void run_young_rs_sampling(); + void wait_for_completed_buffers(); + + void set_active(bool x) { _active = x; } + bool is_active(); + void activate(); + void deactivate(); // For use by G1CollectedHeap, which is a friend. static SuspendibleThreadSet* sts() { return &_sts; } - public: +public: + virtual void run(); // Constructor ConcurrentG1RefineThread(ConcurrentG1Refine* cg1r, ConcurrentG1RefineThread* next, int worker_id_offset, int worker_id); + void initialize(); + // Printing void print() const; void print_on(outputStream* st) const; @@ -83,13 +77,10 @@ // Total virtual time so far. double vtime_accum() { return _vtime_accum; } - ConcurrentG1Refine* cg1r() { return _cg1r; } - - void sample_young_list_rs_lengths(); + ConcurrentG1Refine* cg1r() { return _cg1r; } // Yield for GC - void yield(); - + void yield(); // shutdown void stop(); };
--- a/src/share/vm/gc_implementation/g1/concurrentMark.cpp Fri Dec 11 09:30:48 2009 -0800 +++ b/src/share/vm/gc_implementation/g1/concurrentMark.cpp Wed Dec 16 15:12:51 2009 -0800 @@ -760,7 +760,6 @@ rp->setup_policy(false); // snapshot the soft ref policy to be used in this cycle SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); - satb_mq_set.set_process_completed_threshold(G1SATBProcessCompletedThreshold); satb_mq_set.set_active_all_threads(true); // update_g1_committed() will be called at the end of an evac pause
--- a/src/share/vm/gc_implementation/g1/dirtyCardQueue.cpp Fri Dec 11 09:30:48 2009 -0800 +++ b/src/share/vm/gc_implementation/g1/dirtyCardQueue.cpp Wed Dec 16 15:12:51 2009 -0800 @@ -61,8 +61,8 @@ #pragma warning( disable:4355 ) // 'this' : used in base member initializer list #endif // _MSC_VER -DirtyCardQueueSet::DirtyCardQueueSet() : - PtrQueueSet(true /*notify_when_complete*/), +DirtyCardQueueSet::DirtyCardQueueSet(bool notify_when_complete) : + PtrQueueSet(notify_when_complete), _closure(NULL), _shared_dirty_card_queue(this, true /*perm*/), _free_ids(NULL), @@ -77,12 +77,12 @@ } void DirtyCardQueueSet::initialize(Monitor* cbl_mon, Mutex* fl_lock, + int process_completed_threshold, int max_completed_queue, Mutex* lock, PtrQueueSet* fl_owner) { - PtrQueueSet::initialize(cbl_mon, fl_lock, max_completed_queue, fl_owner); + PtrQueueSet::initialize(cbl_mon, fl_lock, process_completed_threshold, + max_completed_queue, fl_owner); set_buffer_size(G1UpdateBufferSize); - set_process_completed_threshold(G1UpdateBufferQueueProcessingThreshold); - _shared_dirty_card_queue.set_lock(lock); _free_ids = new FreeIdSet((int) num_par_ids(), _cbl_mon); } @@ -154,9 +154,10 @@ return b; } -DirtyCardQueueSet::CompletedBufferNode* + +BufferNode* DirtyCardQueueSet::get_completed_buffer(int stop_at) { - CompletedBufferNode* nd = NULL; + BufferNode* nd = NULL; MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); if ((int)_n_completed_buffers <= stop_at) { @@ -166,10 +167,11 @@ if (_completed_buffers_head != NULL) { nd = _completed_buffers_head; - _completed_buffers_head = nd->next; + _completed_buffers_head = nd->next(); if (_completed_buffers_head == NULL) _completed_buffers_tail = NULL; _n_completed_buffers--; + assert(_n_completed_buffers >= 0, "Invariant"); } debug_only(assert_completed_buffer_list_len_correct_locked()); return nd; @@ -177,20 +179,19 @@ bool DirtyCardQueueSet:: apply_closure_to_completed_buffer_helper(int worker_i, - CompletedBufferNode* nd) { + BufferNode* nd) { if (nd != NULL) { + void **buf = BufferNode::make_buffer_from_node(nd); + size_t index = nd->index(); bool b = - DirtyCardQueue::apply_closure_to_buffer(_closure, nd->buf, - nd->index, _sz, + DirtyCardQueue::apply_closure_to_buffer(_closure, buf, + index, _sz, true, worker_i); - void** buf = nd->buf; - size_t index = nd->index; - delete nd; if (b) { deallocate_buffer(buf); return true; // In normal case, go on to next buffer. } else { - enqueue_complete_buffer(buf, index, true); + enqueue_complete_buffer(buf, index); return false; } } else { @@ -203,32 +204,33 @@ bool during_pause) { assert(!during_pause || stop_at == 0, "Should not leave any completed buffers during a pause"); - CompletedBufferNode* nd = get_completed_buffer(stop_at); + BufferNode* nd = get_completed_buffer(stop_at); bool res = apply_closure_to_completed_buffer_helper(worker_i, nd); if (res) Atomic::inc(&_processed_buffers_rs_thread); return res; } void DirtyCardQueueSet::apply_closure_to_all_completed_buffers() { - CompletedBufferNode* nd = _completed_buffers_head; + BufferNode* nd = _completed_buffers_head; while (nd != NULL) { bool b = - DirtyCardQueue::apply_closure_to_buffer(_closure, nd->buf, 0, _sz, - false); + DirtyCardQueue::apply_closure_to_buffer(_closure, + BufferNode::make_buffer_from_node(nd), + 0, _sz, false); guarantee(b, "Should not stop early."); - nd = nd->next; + nd = nd->next(); } } void DirtyCardQueueSet::abandon_logs() { assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); - CompletedBufferNode* buffers_to_delete = NULL; + BufferNode* buffers_to_delete = NULL; { MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); while (_completed_buffers_head != NULL) { - CompletedBufferNode* nd = _completed_buffers_head; - _completed_buffers_head = nd->next; - nd->next = buffers_to_delete; + BufferNode* nd = _completed_buffers_head; + _completed_buffers_head = nd->next(); + nd->set_next(buffers_to_delete); buffers_to_delete = nd; } _n_completed_buffers = 0; @@ -236,10 +238,9 @@ debug_only(assert_completed_buffer_list_len_correct_locked()); } while (buffers_to_delete != NULL) { - CompletedBufferNode* nd = buffers_to_delete; - buffers_to_delete = nd->next; - deallocate_buffer(nd->buf); - delete nd; + BufferNode* nd = buffers_to_delete; + buffers_to_delete = nd->next(); + deallocate_buffer(BufferNode::make_buffer_from_node(nd)); } // Since abandon is done only at safepoints, we can safely manipulate // these queues.
--- a/src/share/vm/gc_implementation/g1/dirtyCardQueue.hpp Fri Dec 11 09:30:48 2009 -0800 +++ b/src/share/vm/gc_implementation/g1/dirtyCardQueue.hpp Wed Dec 16 15:12:51 2009 -0800 @@ -84,11 +84,12 @@ jint _processed_buffers_rs_thread; public: - DirtyCardQueueSet(); + DirtyCardQueueSet(bool notify_when_complete = true); void initialize(Monitor* cbl_mon, Mutex* fl_lock, - int max_completed_queue = 0, - Mutex* lock = NULL, PtrQueueSet* fl_owner = NULL); + int process_completed_threshold, + int max_completed_queue, + Mutex* lock, PtrQueueSet* fl_owner = NULL); // The number of parallel ids that can be claimed to allow collector or // mutator threads to do card-processing work. @@ -123,9 +124,9 @@ bool during_pause = false); bool apply_closure_to_completed_buffer_helper(int worker_i, - CompletedBufferNode* nd); + BufferNode* nd); - CompletedBufferNode* get_completed_buffer(int stop_at); + BufferNode* get_completed_buffer(int stop_at); // Applies the current closure to all completed buffers, // non-consumptively.
--- a/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Fri Dec 11 09:30:48 2009 -0800 +++ b/src/share/vm/gc_implementation/g1/g1CollectedHeap.cpp Wed Dec 16 15:12:51 2009 -0800 @@ -1375,6 +1375,7 @@ G1CollectedHeap::G1CollectedHeap(G1CollectorPolicy* policy_) : SharedHeap(policy_), _g1_policy(policy_), + _dirty_card_queue_set(false), _ref_processor(NULL), _process_strong_tasks(new SubTasksDone(G1H_PS_NumElements)), _bot_shared(NULL), @@ -1460,8 +1461,6 @@ Universe::check_alignment(init_byte_size, HeapRegion::GrainBytes, "g1 heap"); Universe::check_alignment(max_byte_size, HeapRegion::GrainBytes, "g1 heap"); - // We allocate this in any case, but only do no work if the command line - // param is off. _cg1r = new ConcurrentG1Refine(); // Reserve the maximum. @@ -1594,18 +1593,20 @@ JavaThread::satb_mark_queue_set().initialize(SATB_Q_CBL_mon, SATB_Q_FL_lock, - 0, + G1SATBProcessCompletedThreshold, Shared_SATB_Q_lock); JavaThread::dirty_card_queue_set().initialize(DirtyCardQ_CBL_mon, DirtyCardQ_FL_lock, - G1UpdateBufferQueueMaxLength, + concurrent_g1_refine()->yellow_zone(), + concurrent_g1_refine()->red_zone(), Shared_DirtyCardQ_lock); if (G1DeferredRSUpdate) { dirty_card_queue_set().initialize(DirtyCardQ_CBL_mon, DirtyCardQ_FL_lock, - 0, + -1, // never trigger processing + -1, // no limit on length Shared_DirtyCardQ_lock, &JavaThread::dirty_card_queue_set()); } @@ -4239,10 +4240,11 @@ RedirtyLoggedCardTableEntryFastClosure redirty; dirty_card_queue_set().set_closure(&redirty); dirty_card_queue_set().apply_closure_to_all_completed_buffers(); - JavaThread::dirty_card_queue_set().merge_bufferlists(&dirty_card_queue_set()); + + DirtyCardQueueSet& dcq = JavaThread::dirty_card_queue_set(); + dcq.merge_bufferlists(&dirty_card_queue_set()); assert(dirty_card_queue_set().completed_buffers_num() == 0, "All should be consumed"); } - COMPILER2_PRESENT(DerivedPointerTable::update_pointers()); }
--- a/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp Fri Dec 11 09:30:48 2009 -0800 +++ b/src/share/vm/gc_implementation/g1/g1CollectorPolicy.cpp Wed Dec 16 15:12:51 2009 -0800 @@ -1914,6 +1914,10 @@ calculate_young_list_min_length(); calculate_young_list_target_config(); + // Note that _mmu_tracker->max_gc_time() returns the time in seconds. + double update_rs_time_goal_ms = _mmu_tracker->max_gc_time() * MILLIUNITS * G1RSUpdatePauseFractionPercent / 100.0; + adjust_concurrent_refinement(update_rs_time, update_rs_processed_buffers, update_rs_time_goal_ms); + // </NEW PREDICTION> _target_pause_time_ms = -1.0; @@ -1921,6 +1925,47 @@ // <NEW PREDICTION> +void G1CollectorPolicy::adjust_concurrent_refinement(double update_rs_time, + double update_rs_processed_buffers, + double goal_ms) { + DirtyCardQueueSet& dcqs = JavaThread::dirty_card_queue_set(); + ConcurrentG1Refine *cg1r = G1CollectedHeap::heap()->concurrent_g1_refine(); + + if (G1AdaptiveConcRefine) { + const int k_gy = 3, k_gr = 6; + const double inc_k = 1.1, dec_k = 0.9; + + int g = cg1r->green_zone(); + if (update_rs_time > goal_ms) { + g = (int)(g * dec_k); // Can become 0, that's OK. That would mean a mutator-only processing. + } else { + if (update_rs_time < goal_ms && update_rs_processed_buffers > g) { + g = (int)MAX2(g * inc_k, g + 1.0); + } + } + // Change the refinement threads params + cg1r->set_green_zone(g); + cg1r->set_yellow_zone(g * k_gy); + cg1r->set_red_zone(g * k_gr); + cg1r->reinitialize_threads(); + + int processing_threshold_delta = MAX2((int)(cg1r->green_zone() * sigma()), 1); + int processing_threshold = MIN2(cg1r->green_zone() + processing_threshold_delta, + cg1r->yellow_zone()); + // Change the barrier params + dcqs.set_process_completed_threshold(processing_threshold); + dcqs.set_max_completed_queue(cg1r->red_zone()); + } + + int curr_queue_size = dcqs.completed_buffers_num(); + if (curr_queue_size >= cg1r->yellow_zone()) { + dcqs.set_completed_queue_padding(curr_queue_size); + } else { + dcqs.set_completed_queue_padding(0); + } + dcqs.notify_if_necessary(); +} + double G1CollectorPolicy:: predict_young_collection_elapsed_time_ms(size_t adjustment) {
--- a/src/share/vm/gc_implementation/g1/g1CollectorPolicy.hpp Fri Dec 11 09:30:48 2009 -0800 +++ b/src/share/vm/gc_implementation/g1/g1CollectorPolicy.hpp Wed Dec 16 15:12:51 2009 -0800 @@ -316,6 +316,10 @@ bool verify_young_ages(HeapRegion* head, SurvRateGroup *surv_rate_group); #endif // PRODUCT + void adjust_concurrent_refinement(double update_rs_time, + double update_rs_processed_buffers, + double goal_ms); + protected: double _pause_time_target_ms; double _recorded_young_cset_choice_time_ms;
--- a/src/share/vm/gc_implementation/g1/g1_globals.hpp Fri Dec 11 09:30:48 2009 -0800 +++ b/src/share/vm/gc_implementation/g1/g1_globals.hpp Wed Dec 16 15:12:51 2009 -0800 @@ -85,7 +85,7 @@ diagnostic(bool, G1SummarizeZFStats, false, \ "Summarize zero-filling info") \ \ - develop(bool, G1TraceConcurrentRefinement, false, \ + diagnostic(bool, G1TraceConcurrentRefinement, false, \ "Trace G1 concurrent refinement") \ \ product(intx, G1MarkStackSize, 2 * 1024 * 1024, \ @@ -94,19 +94,6 @@ product(intx, G1MarkRegionStackSize, 1024 * 1024, \ "Size of the region stack for concurrent marking.") \ \ - develop(bool, G1ConcRefine, true, \ - "If true, run concurrent rem set refinement for G1") \ - \ - develop(intx, G1ConcRefineTargTraversals, 4, \ - "Number of concurrent refinement we try to achieve") \ - \ - develop(intx, G1ConcRefineInitialDelta, 4, \ - "Number of heap regions of alloc ahead of starting collection " \ - "pause to start concurrent refinement (initially)") \ - \ - develop(bool, G1SmoothConcRefine, true, \ - "Attempts to smooth out the overhead of concurrent refinement") \ - \ develop(bool, G1ConcZeroFill, true, \ "If true, run concurrent zero-filling thread") \ \ @@ -178,13 +165,38 @@ product(intx, G1UpdateBufferSize, 256, \ "Size of an update buffer") \ \ - product(intx, G1UpdateBufferQueueProcessingThreshold, 5, \ + product(intx, G1ConcRefineYellowZone, 0, \ "Number of enqueued update buffers that will " \ - "trigger concurrent processing") \ + "trigger concurrent processing. Will be selected ergonomically " \ + "by default.") \ + \ + product(intx, G1ConcRefineRedZone, 0, \ + "Maximum number of enqueued update buffers before mutator " \ + "threads start processing new ones instead of enqueueing them. " \ + "Will be selected ergonomically by default. Zero will disable " \ + "concurrent processing.") \ + \ + product(intx, G1ConcRefineGreenZone, 0, \ + "The number of update buffers that are left in the queue by the " \ + "concurrent processing threads. Will be selected ergonomically " \ + "by default.") \ \ - product(intx, G1UpdateBufferQueueMaxLength, 30, \ - "Maximum number of enqueued update buffers before mutator " \ - "threads start processing new ones instead of enqueueing them") \ + product(intx, G1ConcRefineServiceInterval, 300, \ + "The last concurrent refinement thread wakes up every " \ + "specified number of milliseconds to do miscellaneous work.") \ + \ + product(intx, G1ConcRefineThresholdStep, 0, \ + "Each time the rset update queue increases by this amount " \ + "activate the next refinement thread if available. " \ + "Will be selected ergonomically by default.") \ + \ + product(intx, G1RSUpdatePauseFractionPercent, 10, \ + "A target percentage of time that is allowed to be spend on " \ + "process RS update buffers during the collection pause.") \ + \ + product(bool, G1AdaptiveConcRefine, true, \ + "Select green, yellow and red zones adaptively to meet the " \ + "the pause requirements.") \ \ develop(intx, G1ConcRSLogCacheSize, 10, \ "Log base 2 of the length of conc RS hot-card cache.") \
--- a/src/share/vm/gc_implementation/g1/ptrQueue.cpp Fri Dec 11 09:30:48 2009 -0800 +++ b/src/share/vm/gc_implementation/g1/ptrQueue.cpp Wed Dec 16 15:12:51 2009 -0800 @@ -64,8 +64,8 @@ while (_index == 0) { handle_zero_index(); } + assert(_index > 0, "postcondition"); - _index -= oopSize; _buf[byte_index_to_index((int)_index)] = ptr; assert(0 <= _index && _index <= _sz, "Invariant."); @@ -99,95 +99,110 @@ assert(_sz > 0, "Didn't set a buffer size."); MutexLockerEx x(_fl_owner->_fl_lock, Mutex::_no_safepoint_check_flag); if (_fl_owner->_buf_free_list != NULL) { - void** res = _fl_owner->_buf_free_list; - _fl_owner->_buf_free_list = (void**)_fl_owner->_buf_free_list[0]; + void** res = BufferNode::make_buffer_from_node(_fl_owner->_buf_free_list); + _fl_owner->_buf_free_list = _fl_owner->_buf_free_list->next(); _fl_owner->_buf_free_list_sz--; - // Just override the next pointer with NULL, just in case we scan this part - // of the buffer. - res[0] = NULL; return res; } else { - return (void**) NEW_C_HEAP_ARRAY(char, _sz); + // Allocate space for the BufferNode in front of the buffer. + char *b = NEW_C_HEAP_ARRAY(char, _sz + BufferNode::aligned_size()); + return BufferNode::make_buffer_from_block(b); } } void PtrQueueSet::deallocate_buffer(void** buf) { assert(_sz > 0, "Didn't set a buffer size."); MutexLockerEx x(_fl_owner->_fl_lock, Mutex::_no_safepoint_check_flag); - buf[0] = (void*)_fl_owner->_buf_free_list; - _fl_owner->_buf_free_list = buf; + BufferNode *node = BufferNode::make_node_from_buffer(buf); + node->set_next(_fl_owner->_buf_free_list); + _fl_owner->_buf_free_list = node; _fl_owner->_buf_free_list_sz++; } void PtrQueueSet::reduce_free_list() { + assert(_fl_owner == this, "Free list reduction is allowed only for the owner"); // For now we'll adopt the strategy of deleting half. MutexLockerEx x(_fl_lock, Mutex::_no_safepoint_check_flag); size_t n = _buf_free_list_sz / 2; while (n > 0) { assert(_buf_free_list != NULL, "_buf_free_list_sz must be wrong."); - void** head = _buf_free_list; - _buf_free_list = (void**)_buf_free_list[0]; - FREE_C_HEAP_ARRAY(char, head); + void* b = BufferNode::make_block_from_node(_buf_free_list); + _buf_free_list = _buf_free_list->next(); + FREE_C_HEAP_ARRAY(char, b); _buf_free_list_sz --; n--; } } -void PtrQueueSet::enqueue_complete_buffer(void** buf, size_t index, bool ignore_max_completed) { - // I use explicit locking here because there's a bailout in the middle. - _cbl_mon->lock_without_safepoint_check(); - - Thread* thread = Thread::current(); - assert( ignore_max_completed || - thread->is_Java_thread() || - SafepointSynchronize::is_at_safepoint(), - "invariant" ); - ignore_max_completed = ignore_max_completed || !thread->is_Java_thread(); +void PtrQueue::handle_zero_index() { + assert(0 == _index, "Precondition."); + // This thread records the full buffer and allocates a new one (while + // holding the lock if there is one). + if (_buf != NULL) { + if (_lock) { + locking_enqueue_completed_buffer(_buf); + } else { + if (qset()->process_or_enqueue_complete_buffer(_buf)) { + // Recycle the buffer. No allocation. + _sz = qset()->buffer_size(); + _index = _sz; + return; + } + } + } + // Reallocate the buffer + _buf = qset()->allocate_buffer(); + _sz = qset()->buffer_size(); + _index = _sz; + assert(0 <= _index && _index <= _sz, "Invariant."); +} - if (!ignore_max_completed && _max_completed_queue > 0 && - _n_completed_buffers >= (size_t) _max_completed_queue) { - _cbl_mon->unlock(); - bool b = mut_process_buffer(buf); - if (b) { - deallocate_buffer(buf); - return; +bool PtrQueueSet::process_or_enqueue_complete_buffer(void** buf) { + if (Thread::current()->is_Java_thread()) { + // We don't lock. It is fine to be epsilon-precise here. + if (_max_completed_queue == 0 || _max_completed_queue > 0 && + _n_completed_buffers >= _max_completed_queue + _completed_queue_padding) { + bool b = mut_process_buffer(buf); + if (b) { + // True here means that the buffer hasn't been deallocated and the caller may reuse it. + return true; + } } + } + // The buffer will be enqueued. The caller will have to get a new one. + enqueue_complete_buffer(buf); + return false; +} - // Otherwise, go ahead and enqueue the buffer. Must reaquire the lock. - _cbl_mon->lock_without_safepoint_check(); - } - - // Here we still hold the _cbl_mon. - CompletedBufferNode* cbn = new CompletedBufferNode; - cbn->buf = buf; - cbn->next = NULL; - cbn->index = index; +void PtrQueueSet::enqueue_complete_buffer(void** buf, size_t index) { + MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); + BufferNode* cbn = BufferNode::new_from_buffer(buf); + cbn->set_index(index); if (_completed_buffers_tail == NULL) { assert(_completed_buffers_head == NULL, "Well-formedness"); _completed_buffers_head = cbn; _completed_buffers_tail = cbn; } else { - _completed_buffers_tail->next = cbn; + _completed_buffers_tail->set_next(cbn); _completed_buffers_tail = cbn; } _n_completed_buffers++; - if (!_process_completed && + if (!_process_completed && _process_completed_threshold >= 0 && _n_completed_buffers >= _process_completed_threshold) { _process_completed = true; if (_notify_when_complete) - _cbl_mon->notify_all(); + _cbl_mon->notify(); } debug_only(assert_completed_buffer_list_len_correct_locked()); - _cbl_mon->unlock(); } int PtrQueueSet::completed_buffers_list_length() { int n = 0; - CompletedBufferNode* cbn = _completed_buffers_head; + BufferNode* cbn = _completed_buffers_head; while (cbn != NULL) { n++; - cbn = cbn->next; + cbn = cbn->next(); } return n; } @@ -198,7 +213,7 @@ } void PtrQueueSet::assert_completed_buffer_list_len_correct_locked() { - guarantee((size_t)completed_buffers_list_length() == _n_completed_buffers, + guarantee(completed_buffers_list_length() == _n_completed_buffers, "Completed buffer length is wrong."); } @@ -207,12 +222,8 @@ _sz = sz * oopSize; } -void PtrQueueSet::set_process_completed_threshold(size_t sz) { - _process_completed_threshold = sz; -} - -// Merge lists of buffers. Notify waiting threads if the length of the list -// exceeds threshold. The source queue is emptied as a result. The queues +// Merge lists of buffers. Notify the processing threads. +// The source queue is emptied as a result. The queues // must share the monitor. void PtrQueueSet::merge_bufferlists(PtrQueueSet *src) { assert(_cbl_mon == src->_cbl_mon, "Should share the same lock"); @@ -224,7 +235,7 @@ } else { assert(_completed_buffers_head != NULL, "Well formedness"); if (src->_completed_buffers_head != NULL) { - _completed_buffers_tail->next = src->_completed_buffers_head; + _completed_buffers_tail->set_next(src->_completed_buffers_head); _completed_buffers_tail = src->_completed_buffers_tail; } } @@ -237,31 +248,13 @@ assert(_completed_buffers_head == NULL && _completed_buffers_tail == NULL || _completed_buffers_head != NULL && _completed_buffers_tail != NULL, "Sanity"); - - if (!_process_completed && - _n_completed_buffers >= _process_completed_threshold) { - _process_completed = true; - if (_notify_when_complete) - _cbl_mon->notify_all(); - } } -// Merge free lists of the two queues. The free list of the source -// queue is emptied as a result. The queues must share the same -// mutex that guards free lists. -void PtrQueueSet::merge_freelists(PtrQueueSet* src) { - assert(_fl_lock == src->_fl_lock, "Should share the same lock"); - MutexLockerEx x(_fl_lock, Mutex::_no_safepoint_check_flag); - if (_buf_free_list != NULL) { - void **p = _buf_free_list; - while (*p != NULL) { - p = (void**)*p; - } - *p = src->_buf_free_list; - } else { - _buf_free_list = src->_buf_free_list; +void PtrQueueSet::notify_if_necessary() { + MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); + if (_n_completed_buffers >= _process_completed_threshold || _max_completed_queue == 0) { + _process_completed = true; + if (_notify_when_complete) + _cbl_mon->notify(); } - _buf_free_list_sz += src->_buf_free_list_sz; - src->_buf_free_list = NULL; - src->_buf_free_list_sz = 0; }
--- a/src/share/vm/gc_implementation/g1/ptrQueue.hpp Fri Dec 11 09:30:48 2009 -0800 +++ b/src/share/vm/gc_implementation/g1/ptrQueue.hpp Wed Dec 16 15:12:51 2009 -0800 @@ -27,8 +27,10 @@ // the addresses of modified old-generation objects. This type supports // this operation. +// The definition of placement operator new(size_t, void*) in the <new>. +#include <new> + class PtrQueueSet; - class PtrQueue VALUE_OBJ_CLASS_SPEC { protected: @@ -77,7 +79,7 @@ else enqueue_known_active(ptr); } - inline void handle_zero_index(); + void handle_zero_index(); void locking_enqueue_completed_buffer(void** buf); void enqueue_known_active(void* ptr); @@ -126,34 +128,65 @@ }; +class BufferNode { + size_t _index; + BufferNode* _next; +public: + BufferNode() : _index(0), _next(NULL) { } + BufferNode* next() const { return _next; } + void set_next(BufferNode* n) { _next = n; } + size_t index() const { return _index; } + void set_index(size_t i) { _index = i; } + + // Align the size of the structure to the size of the pointer + static size_t aligned_size() { + static const size_t alignment = round_to(sizeof(BufferNode), sizeof(void*)); + return alignment; + } + + // BufferNode is allocated before the buffer. + // The chunk of memory that holds both of them is a block. + + // Produce a new BufferNode given a buffer. + static BufferNode* new_from_buffer(void** buf) { + return new (make_block_from_buffer(buf)) BufferNode; + } + + // The following are the required conversion routines: + static BufferNode* make_node_from_buffer(void** buf) { + return (BufferNode*)make_block_from_buffer(buf); + } + static void** make_buffer_from_node(BufferNode *node) { + return make_buffer_from_block(node); + } + static void* make_block_from_node(BufferNode *node) { + return (void*)node; + } + static void** make_buffer_from_block(void* p) { + return (void**)((char*)p + aligned_size()); + } + static void* make_block_from_buffer(void** p) { + return (void*)((char*)p - aligned_size()); + } +}; + // A PtrQueueSet represents resources common to a set of pointer queues. // In particular, the individual queues allocate buffers from this shared // set, and return completed buffers to the set. // All these variables are are protected by the TLOQ_CBL_mon. XXX ??? class PtrQueueSet VALUE_OBJ_CLASS_SPEC { - protected: - - class CompletedBufferNode: public CHeapObj { - public: - void** buf; - size_t index; - CompletedBufferNode* next; - CompletedBufferNode() : buf(NULL), - index(0), next(NULL){ } - }; - Monitor* _cbl_mon; // Protects the fields below. - CompletedBufferNode* _completed_buffers_head; - CompletedBufferNode* _completed_buffers_tail; - size_t _n_completed_buffers; - size_t _process_completed_threshold; + BufferNode* _completed_buffers_head; + BufferNode* _completed_buffers_tail; + int _n_completed_buffers; + int _process_completed_threshold; volatile bool _process_completed; // This (and the interpretation of the first element as a "next" // pointer) are protected by the TLOQ_FL_lock. Mutex* _fl_lock; - void** _buf_free_list; + BufferNode* _buf_free_list; size_t _buf_free_list_sz; // Queue set can share a freelist. The _fl_owner variable // specifies the owner. It is set to "this" by default. @@ -170,6 +203,7 @@ // Maximum number of elements allowed on completed queue: after that, // enqueuer does the work itself. Zero indicates no maximum. int _max_completed_queue; + int _completed_queue_padding; int completed_buffers_list_length(); void assert_completed_buffer_list_len_correct_locked(); @@ -191,9 +225,12 @@ // Because of init-order concerns, we can't pass these as constructor // arguments. void initialize(Monitor* cbl_mon, Mutex* fl_lock, - int max_completed_queue = 0, + int process_completed_threshold, + int max_completed_queue, PtrQueueSet *fl_owner = NULL) { _max_completed_queue = max_completed_queue; + _process_completed_threshold = process_completed_threshold; + _completed_queue_padding = 0; assert(cbl_mon != NULL && fl_lock != NULL, "Init order issue?"); _cbl_mon = cbl_mon; _fl_lock = fl_lock; @@ -208,14 +245,17 @@ void deallocate_buffer(void** buf); // Declares that "buf" is a complete buffer. - void enqueue_complete_buffer(void** buf, size_t index = 0, - bool ignore_max_completed = false); + void enqueue_complete_buffer(void** buf, size_t index = 0); + + // To be invoked by the mutator. + bool process_or_enqueue_complete_buffer(void** buf); bool completed_buffers_exist_dirty() { return _n_completed_buffers > 0; } bool process_completed_buffers() { return _process_completed; } + void set_process_completed(bool x) { _process_completed = x; } bool active() { return _all_active; } @@ -226,15 +266,24 @@ // Get the buffer size. size_t buffer_size() { return _sz; } - // Set the number of completed buffers that triggers log processing. - void set_process_completed_threshold(size_t sz); + // Get/Set the number of completed buffers that triggers log processing. + void set_process_completed_threshold(int sz) { _process_completed_threshold = sz; } + int process_completed_threshold() const { return _process_completed_threshold; } // Must only be called at a safe point. Indicates that the buffer free // list size may be reduced, if that is deemed desirable. void reduce_free_list(); - size_t completed_buffers_num() { return _n_completed_buffers; } + int completed_buffers_num() { return _n_completed_buffers; } void merge_bufferlists(PtrQueueSet* src); - void merge_freelists(PtrQueueSet* src); + + void set_max_completed_queue(int m) { _max_completed_queue = m; } + int max_completed_queue() { return _max_completed_queue; } + + void set_completed_queue_padding(int padding) { _completed_queue_padding = padding; } + int completed_queue_padding() { return _completed_queue_padding; } + + // Notify the consumer if the number of buffers crossed the threshold + void notify_if_necessary(); };
--- a/src/share/vm/gc_implementation/g1/satbQueue.cpp Fri Dec 11 09:30:48 2009 -0800 +++ b/src/share/vm/gc_implementation/g1/satbQueue.cpp Wed Dec 16 15:12:51 2009 -0800 @@ -67,9 +67,9 @@ {} void SATBMarkQueueSet::initialize(Monitor* cbl_mon, Mutex* fl_lock, - int max_completed_queue, + int process_completed_threshold, Mutex* lock) { - PtrQueueSet::initialize(cbl_mon, fl_lock, max_completed_queue); + PtrQueueSet::initialize(cbl_mon, fl_lock, process_completed_threshold, -1); _shared_satb_queue.set_lock(lock); if (ParallelGCThreads > 0) { _par_closures = NEW_C_HEAP_ARRAY(ObjectClosure*, ParallelGCThreads); @@ -122,12 +122,12 @@ bool SATBMarkQueueSet::apply_closure_to_completed_buffer_work(bool par, int worker) { - CompletedBufferNode* nd = NULL; + BufferNode* nd = NULL; { MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); if (_completed_buffers_head != NULL) { nd = _completed_buffers_head; - _completed_buffers_head = nd->next; + _completed_buffers_head = nd->next(); if (_completed_buffers_head == NULL) _completed_buffers_tail = NULL; _n_completed_buffers--; if (_n_completed_buffers == 0) _process_completed = false; @@ -135,9 +135,9 @@ } ObjectClosure* cl = (par ? _par_closures[worker] : _closure); if (nd != NULL) { - ObjPtrQueue::apply_closure_to_buffer(cl, nd->buf, 0, _sz); - deallocate_buffer(nd->buf); - delete nd; + void **buf = BufferNode::make_buffer_from_node(nd); + ObjPtrQueue::apply_closure_to_buffer(cl, buf, 0, _sz); + deallocate_buffer(buf); return true; } else { return false; @@ -145,13 +145,13 @@ } void SATBMarkQueueSet::abandon_partial_marking() { - CompletedBufferNode* buffers_to_delete = NULL; + BufferNode* buffers_to_delete = NULL; { MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); while (_completed_buffers_head != NULL) { - CompletedBufferNode* nd = _completed_buffers_head; - _completed_buffers_head = nd->next; - nd->next = buffers_to_delete; + BufferNode* nd = _completed_buffers_head; + _completed_buffers_head = nd->next(); + nd->set_next(buffers_to_delete); buffers_to_delete = nd; } _completed_buffers_tail = NULL; @@ -159,10 +159,9 @@ DEBUG_ONLY(assert_completed_buffer_list_len_correct_locked()); } while (buffers_to_delete != NULL) { - CompletedBufferNode* nd = buffers_to_delete; - buffers_to_delete = nd->next; - deallocate_buffer(nd->buf); - delete nd; + BufferNode* nd = buffers_to_delete; + buffers_to_delete = nd->next(); + deallocate_buffer(BufferNode::make_buffer_from_node(nd)); } assert(SafepointSynchronize::is_at_safepoint(), "Must be at safepoint."); // So we can safely manipulate these queues.
--- a/src/share/vm/gc_implementation/g1/satbQueue.hpp Fri Dec 11 09:30:48 2009 -0800 +++ b/src/share/vm/gc_implementation/g1/satbQueue.hpp Wed Dec 16 15:12:51 2009 -0800 @@ -60,8 +60,8 @@ SATBMarkQueueSet(); void initialize(Monitor* cbl_mon, Mutex* fl_lock, - int max_completed_queue = 0, - Mutex* lock = NULL); + int process_completed_threshold, + Mutex* lock); static void handle_zero_index_for_thread(JavaThread* t);
--- a/src/share/vm/gc_implementation/includeDB_gc_g1 Fri Dec 11 09:30:48 2009 -0800 +++ b/src/share/vm/gc_implementation/includeDB_gc_g1 Wed Dec 16 15:12:51 2009 -0800 @@ -109,7 +109,6 @@ dirtyCardQueue.cpp dirtyCardQueue.hpp dirtyCardQueue.cpp heapRegionRemSet.hpp dirtyCardQueue.cpp mutexLocker.hpp -dirtyCardQueue.cpp ptrQueue.inline.hpp dirtyCardQueue.cpp safepoint.hpp dirtyCardQueue.cpp thread.hpp dirtyCardQueue.cpp thread_<os_family>.inline.hpp @@ -319,7 +318,6 @@ ptrQueue.cpp mutex.hpp ptrQueue.cpp mutexLocker.hpp ptrQueue.cpp ptrQueue.hpp -ptrQueue.cpp ptrQueue.inline.hpp ptrQueue.cpp thread_<os_family>.inline.hpp ptrQueue.hpp allocation.hpp @@ -329,7 +327,6 @@ satbQueue.cpp allocation.inline.hpp satbQueue.cpp mutexLocker.hpp -satbQueue.cpp ptrQueue.inline.hpp satbQueue.cpp satbQueue.hpp satbQueue.cpp sharedHeap.hpp satbQueue.cpp thread.hpp