Mercurial > hg > truffle
annotate src/share/vm/gc_implementation/g1/ptrQueue.cpp @ 889:15c5903cf9e1
6865703: G1: Parallelize hot card cache cleanup
Summary: Have the GC worker threads clear the hot card cache in parallel by having each worker thread claim a chunk of the card cache and process the cards in that chunk. The size of the chunks that each thread will claim is determined at VM initialization from the size of the card cache and the number of worker threads.
Reviewed-by: jmasa, tonyp
| author | johnc |
|---|---|
| date | Mon, 03 Aug 2009 12:59:30 -0700 |
| parents | bd02caa94611 |
| children | 5f932a151fd4 |
| rev | line source |
|---|---|
| 342 | 1 /* |
| 844 | 2 * Copyright 2001-2009 Sun Microsystems, Inc. All Rights Reserved. |
| 342 | 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| 4 * | |
| 5 * This code is free software; you can redistribute it and/or modify it | |
| 6 * under the terms of the GNU General Public License version 2 only, as | |
| 7 * published by the Free Software Foundation. | |
| 8 * | |
| 9 * This code is distributed in the hope that it will be useful, but WITHOUT | |
| 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
| 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
| 12 * version 2 for more details (a copy is included in the LICENSE file that | |
| 13 * accompanied this code). | |
| 14 * | |
| 15 * You should have received a copy of the GNU General Public License version | |
| 16 * 2 along with this work; if not, write to the Free Software Foundation, | |
| 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |
| 18 * | |
| 19 * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara, | |
| 20 * CA 95054 USA or visit www.sun.com if you need additional information or | |
| 21 * have any questions. | |
| 22 * | |
| 23 */ | |
| 24 | |
| 25 # include "incls/_precompiled.incl" | |
| 26 # include "incls/_ptrQueue.cpp.incl" | |
| 27 | |
| 28 PtrQueue::PtrQueue(PtrQueueSet* qset_, bool perm) : | |
| 29 _qset(qset_), _buf(NULL), _index(0), _active(false), | |
| 30 _perm(perm), _lock(NULL) | |
| 31 {} | |
| 32 | |
|
441
da9cb4e97a5f
6770608: G1: Mutator thread can flush barrier and satb queues during safepoint
iveresov
parents:
342
diff
changeset
|
33 void PtrQueue::flush() { |
| 342 | 34 if (!_perm && _buf != NULL) { |
| 35 if (_index == _sz) { | |
| 36 // No work to do. | |
| 37 qset()->deallocate_buffer(_buf); | |
| 38 } else { | |
| 39 // We must NULL out the unused entries, then enqueue. | |
| 40 for (size_t i = 0; i < _index; i += oopSize) { | |
| 41 _buf[byte_index_to_index((int)i)] = NULL; | |
| 42 } | |
| 43 qset()->enqueue_complete_buffer(_buf); | |
| 44 } | |
|
441
da9cb4e97a5f
6770608: G1: Mutator thread can flush barrier and satb queues during safepoint
iveresov
parents:
342
diff
changeset
|
45 _buf = NULL; |
|
da9cb4e97a5f
6770608: G1: Mutator thread can flush barrier and satb queues during safepoint
iveresov
parents:
342
diff
changeset
|
46 _index = 0; |
| 342 | 47 } |
| 48 } | |
| 49 | |
| 50 | |
| 51 static int byte_index_to_index(int ind) { | |
| 52 assert((ind % oopSize) == 0, "Invariant."); | |
| 53 return ind / oopSize; | |
| 54 } | |
| 55 | |
| 56 static int index_to_byte_index(int byte_ind) { | |
| 57 return byte_ind * oopSize; | |
| 58 } | |
| 59 | |
| 60 void PtrQueue::enqueue_known_active(void* ptr) { | |
| 61 assert(0 <= _index && _index <= _sz, "Invariant."); | |
| 62 assert(_index == 0 || _buf != NULL, "invariant"); | |
| 63 | |
| 64 while (_index == 0) { | |
| 65 handle_zero_index(); | |
| 66 } | |
| 67 assert(_index > 0, "postcondition"); | |
| 68 | |
| 69 _index -= oopSize; | |
| 70 _buf[byte_index_to_index((int)_index)] = ptr; | |
| 71 assert(0 <= _index && _index <= _sz, "Invariant."); | |
| 72 } | |
| 73 | |
| 74 void PtrQueue::locking_enqueue_completed_buffer(void** buf) { | |
| 75 assert(_lock->owned_by_self(), "Required."); | |
| 76 _lock->unlock(); | |
| 77 qset()->enqueue_complete_buffer(buf); | |
| 78 // We must relock only because the caller will unlock, for the normal | |
| 79 // case. | |
| 80 _lock->lock_without_safepoint_check(); | |
| 81 } | |
| 82 | |
| 83 | |
| 84 PtrQueueSet::PtrQueueSet(bool notify_when_complete) : | |
| 85 _max_completed_queue(0), | |
| 86 _cbl_mon(NULL), _fl_lock(NULL), | |
| 87 _notify_when_complete(notify_when_complete), | |
| 88 _sz(0), | |
| 89 _completed_buffers_head(NULL), | |
| 90 _completed_buffers_tail(NULL), | |
| 91 _n_completed_buffers(0), | |
| 92 _process_completed_threshold(0), _process_completed(false), | |
| 93 _buf_free_list(NULL), _buf_free_list_sz(0) | |
|
616
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
94 { |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
95 _fl_owner = this; |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
96 } |
| 342 | 97 |
| 98 void** PtrQueueSet::allocate_buffer() { | |
| 99 assert(_sz > 0, "Didn't set a buffer size."); | |
|
616
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
100 MutexLockerEx x(_fl_owner->_fl_lock, Mutex::_no_safepoint_check_flag); |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
101 if (_fl_owner->_buf_free_list != NULL) { |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
102 void** res = _fl_owner->_buf_free_list; |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
103 _fl_owner->_buf_free_list = (void**)_fl_owner->_buf_free_list[0]; |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
104 _fl_owner->_buf_free_list_sz--; |
| 342 | 105 // Just override the next pointer with NULL, just in case we scan this part |
| 106 // of the buffer. | |
| 107 res[0] = NULL; | |
| 108 return res; | |
| 109 } else { | |
| 110 return NEW_C_HEAP_ARRAY(void*, _sz); | |
| 111 } | |
| 112 } | |
| 113 | |
| 114 void PtrQueueSet::deallocate_buffer(void** buf) { | |
| 115 assert(_sz > 0, "Didn't set a buffer size."); | |
|
616
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
116 MutexLockerEx x(_fl_owner->_fl_lock, Mutex::_no_safepoint_check_flag); |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
117 buf[0] = (void*)_fl_owner->_buf_free_list; |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
118 _fl_owner->_buf_free_list = buf; |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
119 _fl_owner->_buf_free_list_sz++; |
| 342 | 120 } |
| 121 | |
| 122 void PtrQueueSet::reduce_free_list() { | |
| 123 // For now we'll adopt the strategy of deleting half. | |
| 124 MutexLockerEx x(_fl_lock, Mutex::_no_safepoint_check_flag); | |
| 125 size_t n = _buf_free_list_sz / 2; | |
| 126 while (n > 0) { | |
| 127 assert(_buf_free_list != NULL, "_buf_free_list_sz must be wrong."); | |
| 128 void** head = _buf_free_list; | |
| 129 _buf_free_list = (void**)_buf_free_list[0]; | |
| 130 FREE_C_HEAP_ARRAY(void*,head); | |
| 131 n--; | |
| 132 } | |
| 133 } | |
| 134 | |
| 135 void PtrQueueSet::enqueue_complete_buffer(void** buf, size_t index, bool ignore_max_completed) { | |
| 136 // I use explicit locking here because there's a bailout in the middle. | |
| 137 _cbl_mon->lock_without_safepoint_check(); | |
| 138 | |
| 139 Thread* thread = Thread::current(); | |
| 140 assert( ignore_max_completed || | |
| 141 thread->is_Java_thread() || | |
| 142 SafepointSynchronize::is_at_safepoint(), | |
| 143 "invariant" ); | |
| 144 ignore_max_completed = ignore_max_completed || !thread->is_Java_thread(); | |
| 145 | |
| 146 if (!ignore_max_completed && _max_completed_queue > 0 && | |
| 147 _n_completed_buffers >= (size_t) _max_completed_queue) { | |
| 148 _cbl_mon->unlock(); | |
| 149 bool b = mut_process_buffer(buf); | |
| 150 if (b) { | |
| 151 deallocate_buffer(buf); | |
| 152 return; | |
| 153 } | |
| 154 | |
| 155 // Otherwise, go ahead and enqueue the buffer. Must reaquire the lock. | |
| 156 _cbl_mon->lock_without_safepoint_check(); | |
| 157 } | |
| 158 | |
| 159 // Here we still hold the _cbl_mon. | |
| 160 CompletedBufferNode* cbn = new CompletedBufferNode; | |
| 161 cbn->buf = buf; | |
| 162 cbn->next = NULL; | |
| 163 cbn->index = index; | |
| 164 if (_completed_buffers_tail == NULL) { | |
| 165 assert(_completed_buffers_head == NULL, "Well-formedness"); | |
| 166 _completed_buffers_head = cbn; | |
| 167 _completed_buffers_tail = cbn; | |
| 168 } else { | |
| 169 _completed_buffers_tail->next = cbn; | |
| 170 _completed_buffers_tail = cbn; | |
| 171 } | |
| 172 _n_completed_buffers++; | |
| 173 | |
| 174 if (!_process_completed && | |
| 794 | 175 _n_completed_buffers >= _process_completed_threshold) { |
| 342 | 176 _process_completed = true; |
| 177 if (_notify_when_complete) | |
| 178 _cbl_mon->notify_all(); | |
| 179 } | |
| 180 debug_only(assert_completed_buffer_list_len_correct_locked()); | |
| 181 _cbl_mon->unlock(); | |
| 182 } | |
| 183 | |
| 184 int PtrQueueSet::completed_buffers_list_length() { | |
| 185 int n = 0; | |
| 186 CompletedBufferNode* cbn = _completed_buffers_head; | |
| 187 while (cbn != NULL) { | |
| 188 n++; | |
| 189 cbn = cbn->next; | |
| 190 } | |
| 191 return n; | |
| 192 } | |
| 193 | |
| 194 void PtrQueueSet::assert_completed_buffer_list_len_correct() { | |
| 195 MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); | |
| 196 assert_completed_buffer_list_len_correct_locked(); | |
| 197 } | |
| 198 | |
| 199 void PtrQueueSet::assert_completed_buffer_list_len_correct_locked() { | |
| 200 guarantee((size_t)completed_buffers_list_length() == _n_completed_buffers, | |
| 201 "Completed buffer length is wrong."); | |
| 202 } | |
| 203 | |
| 204 void PtrQueueSet::set_buffer_size(size_t sz) { | |
| 205 assert(_sz == 0 && sz > 0, "Should be called only once."); | |
| 206 _sz = sz * oopSize; | |
| 207 } | |
| 208 | |
| 209 void PtrQueueSet::set_process_completed_threshold(size_t sz) { | |
| 210 _process_completed_threshold = sz; | |
| 211 } | |
|
616
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
212 |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
213 // Merge lists of buffers. Notify waiting threads if the length of the list |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
214 // exceeds threshold. The source queue is emptied as a result. The queues |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
215 // must share the monitor. |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
216 void PtrQueueSet::merge_bufferlists(PtrQueueSet *src) { |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
217 assert(_cbl_mon == src->_cbl_mon, "Should share the same lock"); |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
218 MutexLockerEx x(_cbl_mon, Mutex::_no_safepoint_check_flag); |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
219 if (_completed_buffers_tail == NULL) { |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
220 assert(_completed_buffers_head == NULL, "Well-formedness"); |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
221 _completed_buffers_head = src->_completed_buffers_head; |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
222 _completed_buffers_tail = src->_completed_buffers_tail; |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
223 } else { |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
224 assert(_completed_buffers_head != NULL, "Well formedness"); |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
225 if (src->_completed_buffers_head != NULL) { |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
226 _completed_buffers_tail->next = src->_completed_buffers_head; |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
227 _completed_buffers_tail = src->_completed_buffers_tail; |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
228 } |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
229 } |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
230 _n_completed_buffers += src->_n_completed_buffers; |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
231 |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
232 src->_n_completed_buffers = 0; |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
233 src->_completed_buffers_head = NULL; |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
234 src->_completed_buffers_tail = NULL; |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
235 |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
236 assert(_completed_buffers_head == NULL && _completed_buffers_tail == NULL || |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
237 _completed_buffers_head != NULL && _completed_buffers_tail != NULL, |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
238 "Sanity"); |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
239 |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
240 if (!_process_completed && |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
241 _n_completed_buffers >= _process_completed_threshold) { |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
242 _process_completed = true; |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
243 if (_notify_when_complete) |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
244 _cbl_mon->notify_all(); |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
245 } |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
246 } |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
247 |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
248 // Merge free lists of the two queues. The free list of the source |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
249 // queue is emptied as a result. The queues must share the same |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
250 // mutex that guards free lists. |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
251 void PtrQueueSet::merge_freelists(PtrQueueSet* src) { |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
252 assert(_fl_lock == src->_fl_lock, "Should share the same lock"); |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
253 MutexLockerEx x(_fl_lock, Mutex::_no_safepoint_check_flag); |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
254 if (_buf_free_list != NULL) { |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
255 void **p = _buf_free_list; |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
256 while (*p != NULL) { |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
257 p = (void**)*p; |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
258 } |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
259 *p = src->_buf_free_list; |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
260 } else { |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
261 _buf_free_list = src->_buf_free_list; |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
262 } |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
263 _buf_free_list_sz += src->_buf_free_list_sz; |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
264 src->_buf_free_list = NULL; |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
265 src->_buf_free_list_sz = 0; |
|
4f360ec815ba
6720309: G1: don't synchronously update RSet during evacuation pauses
iveresov
parents:
470
diff
changeset
|
266 } |