Mercurial > hg > truffle
annotate src/share/vm/gc_implementation/g1/concurrentMark.cpp @ 2173:97ba643ea3ed
7014261: G1: RSet-related failures
Summary: A race between the concurrent cleanup thread and the VM thread while it is processing the "expanded sparse table list" causes both threads to try to free the same sparse table entry and either causes one of the threads to fail or leaves the entry in an inconsistent state. The solution is purge all entries on the expanded list that correspond go regions that are being cleaned up.
Reviewed-by: brutisso, johnc
author | tonyp |
---|---|
date | Tue, 25 Jan 2011 17:58:19 -0500 |
parents | 0fa27f37d4d4 |
children | 81668b1f4877 |
rev | line source |
---|---|
342 | 1 /* |
2149 | 2 * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. |
342 | 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
4 * | |
5 * This code is free software; you can redistribute it and/or modify it | |
6 * under the terms of the GNU General Public License version 2 only, as | |
7 * published by the Free Software Foundation. | |
8 * | |
9 * This code is distributed in the hope that it will be useful, but WITHOUT | |
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
12 * version 2 for more details (a copy is included in the LICENSE file that | |
13 * accompanied this code). | |
14 * | |
15 * You should have received a copy of the GNU General Public License version | |
16 * 2 along with this work; if not, write to the Free Software Foundation, | |
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |
18 * | |
1552
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1547
diff
changeset
|
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1547
diff
changeset
|
20 * or visit www.oracle.com if you need additional information or have any |
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1547
diff
changeset
|
21 * questions. |
342 | 22 * |
23 */ | |
24 | |
1972 | 25 #include "precompiled.hpp" |
26 #include "classfile/symbolTable.hpp" | |
27 #include "gc_implementation/g1/concurrentMark.hpp" | |
28 #include "gc_implementation/g1/concurrentMarkThread.inline.hpp" | |
29 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp" | |
30 #include "gc_implementation/g1/g1CollectorPolicy.hpp" | |
31 #include "gc_implementation/g1/g1RemSet.hpp" | |
32 #include "gc_implementation/g1/heapRegionRemSet.hpp" | |
33 #include "gc_implementation/g1/heapRegionSeq.inline.hpp" | |
2125
7246a374a9f2
6458402: 3 jvmti tests fail with CMS and +ExplicitGCInvokesConcurrent
kamg
parents:
2037
diff
changeset
|
34 #include "gc_implementation/shared/vmGCOperations.hpp" |
1972 | 35 #include "memory/genOopClosures.inline.hpp" |
36 #include "memory/referencePolicy.hpp" | |
37 #include "memory/resourceArea.hpp" | |
38 #include "oops/oop.inline.hpp" | |
39 #include "runtime/handles.inline.hpp" | |
40 #include "runtime/java.hpp" | |
342 | 41 |
42 // | |
43 // CMS Bit Map Wrapper | |
44 | |
45 CMBitMapRO::CMBitMapRO(ReservedSpace rs, int shifter): | |
46 _bm((uintptr_t*)NULL,0), | |
47 _shifter(shifter) { | |
48 _bmStartWord = (HeapWord*)(rs.base()); | |
49 _bmWordSize = rs.size()/HeapWordSize; // rs.size() is in bytes | |
50 ReservedSpace brs(ReservedSpace::allocation_align_size_up( | |
51 (_bmWordSize >> (_shifter + LogBitsPerByte)) + 1)); | |
52 | |
53 guarantee(brs.is_reserved(), "couldn't allocate CMS bit map"); | |
54 // For now we'll just commit all of the bit map up fromt. | |
55 // Later on we'll try to be more parsimonious with swap. | |
56 guarantee(_virtual_space.initialize(brs, brs.size()), | |
57 "couldn't reseve backing store for CMS bit map"); | |
58 assert(_virtual_space.committed_size() == brs.size(), | |
59 "didn't reserve backing store for all of CMS bit map?"); | |
60 _bm.set_map((uintptr_t*)_virtual_space.low()); | |
61 assert(_virtual_space.committed_size() << (_shifter + LogBitsPerByte) >= | |
62 _bmWordSize, "inconsistency in bit map sizing"); | |
63 _bm.set_size(_bmWordSize >> _shifter); | |
64 } | |
65 | |
66 HeapWord* CMBitMapRO::getNextMarkedWordAddress(HeapWord* addr, | |
67 HeapWord* limit) const { | |
68 // First we must round addr *up* to a possible object boundary. | |
69 addr = (HeapWord*)align_size_up((intptr_t)addr, | |
70 HeapWordSize << _shifter); | |
71 size_t addrOffset = heapWordToOffset(addr); | |
72 if (limit == NULL) limit = _bmStartWord + _bmWordSize; | |
73 size_t limitOffset = heapWordToOffset(limit); | |
74 size_t nextOffset = _bm.get_next_one_offset(addrOffset, limitOffset); | |
75 HeapWord* nextAddr = offsetToHeapWord(nextOffset); | |
76 assert(nextAddr >= addr, "get_next_one postcondition"); | |
77 assert(nextAddr == limit || isMarked(nextAddr), | |
78 "get_next_one postcondition"); | |
79 return nextAddr; | |
80 } | |
81 | |
82 HeapWord* CMBitMapRO::getNextUnmarkedWordAddress(HeapWord* addr, | |
83 HeapWord* limit) const { | |
84 size_t addrOffset = heapWordToOffset(addr); | |
85 if (limit == NULL) limit = _bmStartWord + _bmWordSize; | |
86 size_t limitOffset = heapWordToOffset(limit); | |
87 size_t nextOffset = _bm.get_next_zero_offset(addrOffset, limitOffset); | |
88 HeapWord* nextAddr = offsetToHeapWord(nextOffset); | |
89 assert(nextAddr >= addr, "get_next_one postcondition"); | |
90 assert(nextAddr == limit || !isMarked(nextAddr), | |
91 "get_next_one postcondition"); | |
92 return nextAddr; | |
93 } | |
94 | |
95 int CMBitMapRO::heapWordDiffToOffsetDiff(size_t diff) const { | |
96 assert((diff & ((1 << _shifter) - 1)) == 0, "argument check"); | |
97 return (int) (diff >> _shifter); | |
98 } | |
99 | |
100 bool CMBitMapRO::iterate(BitMapClosure* cl, MemRegion mr) { | |
101 HeapWord* left = MAX2(_bmStartWord, mr.start()); | |
102 HeapWord* right = MIN2(_bmStartWord + _bmWordSize, mr.end()); | |
103 if (right > left) { | |
104 // Right-open interval [leftOffset, rightOffset). | |
105 return _bm.iterate(cl, heapWordToOffset(left), heapWordToOffset(right)); | |
106 } else { | |
107 return true; | |
108 } | |
109 } | |
110 | |
111 void CMBitMapRO::mostly_disjoint_range_union(BitMap* from_bitmap, | |
112 size_t from_start_index, | |
113 HeapWord* to_start_word, | |
114 size_t word_num) { | |
115 _bm.mostly_disjoint_range_union(from_bitmap, | |
116 from_start_index, | |
117 heapWordToOffset(to_start_word), | |
118 word_num); | |
119 } | |
120 | |
121 #ifndef PRODUCT | |
122 bool CMBitMapRO::covers(ReservedSpace rs) const { | |
123 // assert(_bm.map() == _virtual_space.low(), "map inconsistency"); | |
645
c3a720eefe82
6816308: Changes to allow builds with latest Windows SDK 6.1 on 64bit Windows 2003
kvn
parents:
619
diff
changeset
|
124 assert(((size_t)_bm.size() * (size_t)(1 << _shifter)) == _bmWordSize, |
342 | 125 "size inconsistency"); |
126 return _bmStartWord == (HeapWord*)(rs.base()) && | |
127 _bmWordSize == rs.size()>>LogHeapWordSize; | |
128 } | |
129 #endif | |
130 | |
131 void CMBitMap::clearAll() { | |
132 _bm.clear(); | |
133 return; | |
134 } | |
135 | |
136 void CMBitMap::markRange(MemRegion mr) { | |
137 mr.intersection(MemRegion(_bmStartWord, _bmWordSize)); | |
138 assert(!mr.is_empty(), "unexpected empty region"); | |
139 assert((offsetToHeapWord(heapWordToOffset(mr.end())) == | |
140 ((HeapWord *) mr.end())), | |
141 "markRange memory region end is not card aligned"); | |
142 // convert address range into offset range | |
143 _bm.at_put_range(heapWordToOffset(mr.start()), | |
144 heapWordToOffset(mr.end()), true); | |
145 } | |
146 | |
147 void CMBitMap::clearRange(MemRegion mr) { | |
148 mr.intersection(MemRegion(_bmStartWord, _bmWordSize)); | |
149 assert(!mr.is_empty(), "unexpected empty region"); | |
150 // convert address range into offset range | |
151 _bm.at_put_range(heapWordToOffset(mr.start()), | |
152 heapWordToOffset(mr.end()), false); | |
153 } | |
154 | |
155 MemRegion CMBitMap::getAndClearMarkedRegion(HeapWord* addr, | |
156 HeapWord* end_addr) { | |
157 HeapWord* start = getNextMarkedWordAddress(addr); | |
158 start = MIN2(start, end_addr); | |
159 HeapWord* end = getNextUnmarkedWordAddress(start); | |
160 end = MIN2(end, end_addr); | |
161 assert(start <= end, "Consistency check"); | |
162 MemRegion mr(start, end); | |
163 if (!mr.is_empty()) { | |
164 clearRange(mr); | |
165 } | |
166 return mr; | |
167 } | |
168 | |
169 CMMarkStack::CMMarkStack(ConcurrentMark* cm) : | |
170 _base(NULL), _cm(cm) | |
171 #ifdef ASSERT | |
172 , _drain_in_progress(false) | |
173 , _drain_in_progress_yields(false) | |
174 #endif | |
175 {} | |
176 | |
177 void CMMarkStack::allocate(size_t size) { | |
178 _base = NEW_C_HEAP_ARRAY(oop, size); | |
179 if (_base == NULL) | |
180 vm_exit_during_initialization("Failed to allocate " | |
181 "CM region mark stack"); | |
182 _index = 0; | |
183 // QQQQ cast ... | |
184 _capacity = (jint) size; | |
185 _oops_do_bound = -1; | |
186 NOT_PRODUCT(_max_depth = 0); | |
187 } | |
188 | |
189 CMMarkStack::~CMMarkStack() { | |
190 if (_base != NULL) FREE_C_HEAP_ARRAY(oop, _base); | |
191 } | |
192 | |
193 void CMMarkStack::par_push(oop ptr) { | |
194 while (true) { | |
195 if (isFull()) { | |
196 _overflow = true; | |
197 return; | |
198 } | |
199 // Otherwise... | |
200 jint index = _index; | |
201 jint next_index = index+1; | |
202 jint res = Atomic::cmpxchg(next_index, &_index, index); | |
203 if (res == index) { | |
204 _base[index] = ptr; | |
205 // Note that we don't maintain this atomically. We could, but it | |
206 // doesn't seem necessary. | |
207 NOT_PRODUCT(_max_depth = MAX2(_max_depth, next_index)); | |
208 return; | |
209 } | |
210 // Otherwise, we need to try again. | |
211 } | |
212 } | |
213 | |
214 void CMMarkStack::par_adjoin_arr(oop* ptr_arr, int n) { | |
215 while (true) { | |
216 if (isFull()) { | |
217 _overflow = true; | |
218 return; | |
219 } | |
220 // Otherwise... | |
221 jint index = _index; | |
222 jint next_index = index + n; | |
223 if (next_index > _capacity) { | |
224 _overflow = true; | |
225 return; | |
226 } | |
227 jint res = Atomic::cmpxchg(next_index, &_index, index); | |
228 if (res == index) { | |
229 for (int i = 0; i < n; i++) { | |
230 int ind = index + i; | |
231 assert(ind < _capacity, "By overflow test above."); | |
232 _base[ind] = ptr_arr[i]; | |
233 } | |
234 NOT_PRODUCT(_max_depth = MAX2(_max_depth, next_index)); | |
235 return; | |
236 } | |
237 // Otherwise, we need to try again. | |
238 } | |
239 } | |
240 | |
241 | |
242 void CMMarkStack::par_push_arr(oop* ptr_arr, int n) { | |
243 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); | |
244 jint start = _index; | |
245 jint next_index = start + n; | |
246 if (next_index > _capacity) { | |
247 _overflow = true; | |
248 return; | |
249 } | |
250 // Otherwise. | |
251 _index = next_index; | |
252 for (int i = 0; i < n; i++) { | |
253 int ind = start + i; | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
254 assert(ind < _capacity, "By overflow test above."); |
342 | 255 _base[ind] = ptr_arr[i]; |
256 } | |
257 } | |
258 | |
259 | |
260 bool CMMarkStack::par_pop_arr(oop* ptr_arr, int max, int* n) { | |
261 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); | |
262 jint index = _index; | |
263 if (index == 0) { | |
264 *n = 0; | |
265 return false; | |
266 } else { | |
267 int k = MIN2(max, index); | |
268 jint new_ind = index - k; | |
269 for (int j = 0; j < k; j++) { | |
270 ptr_arr[j] = _base[new_ind + j]; | |
271 } | |
272 _index = new_ind; | |
273 *n = k; | |
274 return true; | |
275 } | |
276 } | |
277 | |
278 | |
279 CMRegionStack::CMRegionStack() : _base(NULL) {} | |
280 | |
281 void CMRegionStack::allocate(size_t size) { | |
282 _base = NEW_C_HEAP_ARRAY(MemRegion, size); | |
283 if (_base == NULL) | |
284 vm_exit_during_initialization("Failed to allocate " | |
285 "CM region mark stack"); | |
286 _index = 0; | |
287 // QQQQ cast ... | |
288 _capacity = (jint) size; | |
289 } | |
290 | |
291 CMRegionStack::~CMRegionStack() { | |
292 if (_base != NULL) FREE_C_HEAP_ARRAY(oop, _base); | |
293 } | |
294 | |
1835
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
295 void CMRegionStack::push_lock_free(MemRegion mr) { |
342 | 296 assert(mr.word_size() > 0, "Precondition"); |
297 while (true) { | |
1835
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
298 jint index = _index; |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
299 |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
300 if (index >= _capacity) { |
342 | 301 _overflow = true; |
302 return; | |
303 } | |
304 // Otherwise... | |
305 jint next_index = index+1; | |
306 jint res = Atomic::cmpxchg(next_index, &_index, index); | |
307 if (res == index) { | |
308 _base[index] = mr; | |
309 return; | |
310 } | |
311 // Otherwise, we need to try again. | |
312 } | |
313 } | |
314 | |
1835
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
315 // Lock-free pop of the region stack. Called during the concurrent |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
316 // marking / remark phases. Should only be called in tandem with |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
317 // other lock-free pops. |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
318 MemRegion CMRegionStack::pop_lock_free() { |
342 | 319 while (true) { |
320 jint index = _index; | |
321 | |
322 if (index == 0) { | |
323 return MemRegion(); | |
324 } | |
1835
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
325 // Otherwise... |
342 | 326 jint next_index = index-1; |
327 jint res = Atomic::cmpxchg(next_index, &_index, index); | |
328 if (res == index) { | |
329 MemRegion mr = _base[next_index]; | |
330 if (mr.start() != NULL) { | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
331 assert(mr.end() != NULL, "invariant"); |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
332 assert(mr.word_size() > 0, "invariant"); |
342 | 333 return mr; |
334 } else { | |
335 // that entry was invalidated... let's skip it | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
336 assert(mr.end() == NULL, "invariant"); |
342 | 337 } |
338 } | |
339 // Otherwise, we need to try again. | |
340 } | |
341 } | |
1835
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
342 |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
343 #if 0 |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
344 // The routines that manipulate the region stack with a lock are |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
345 // not currently used. They should be retained, however, as a |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
346 // diagnostic aid. |
1358
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
347 |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
348 void CMRegionStack::push_with_lock(MemRegion mr) { |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
349 assert(mr.word_size() > 0, "Precondition"); |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
350 MutexLockerEx x(CMRegionStack_lock, Mutex::_no_safepoint_check_flag); |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
351 |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
352 if (isFull()) { |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
353 _overflow = true; |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
354 return; |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
355 } |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
356 |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
357 _base[_index] = mr; |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
358 _index += 1; |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
359 } |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
360 |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
361 MemRegion CMRegionStack::pop_with_lock() { |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
362 MutexLockerEx x(CMRegionStack_lock, Mutex::_no_safepoint_check_flag); |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
363 |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
364 while (true) { |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
365 if (_index == 0) { |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
366 return MemRegion(); |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
367 } |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
368 _index -= 1; |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
369 |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
370 MemRegion mr = _base[_index]; |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
371 if (mr.start() != NULL) { |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
372 assert(mr.end() != NULL, "invariant"); |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
373 assert(mr.word_size() > 0, "invariant"); |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
374 return mr; |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
375 } else { |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
376 // that entry was invalidated... let's skip it |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
377 assert(mr.end() == NULL, "invariant"); |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
378 } |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
379 } |
72f725c5a7be
6940310: G1: MT-unsafe calls to CM::region_stack_push() / CM::region_stack_pop()
tonyp
parents:
1317
diff
changeset
|
380 } |
1835
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
381 #endif |
342 | 382 |
383 bool CMRegionStack::invalidate_entries_into_cset() { | |
384 bool result = false; | |
385 G1CollectedHeap* g1h = G1CollectedHeap::heap(); | |
386 for (int i = 0; i < _oops_do_bound; ++i) { | |
387 MemRegion mr = _base[i]; | |
388 if (mr.start() != NULL) { | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
389 assert(mr.end() != NULL, "invariant"); |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
390 assert(mr.word_size() > 0, "invariant"); |
342 | 391 HeapRegion* hr = g1h->heap_region_containing(mr.start()); |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
392 assert(hr != NULL, "invariant"); |
342 | 393 if (hr->in_collection_set()) { |
394 // The region points into the collection set | |
395 _base[i] = MemRegion(); | |
396 result = true; | |
397 } | |
398 } else { | |
399 // that entry was invalidated... let's skip it | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
400 assert(mr.end() == NULL, "invariant"); |
342 | 401 } |
402 } | |
403 return result; | |
404 } | |
405 | |
406 template<class OopClosureClass> | |
407 bool CMMarkStack::drain(OopClosureClass* cl, CMBitMap* bm, bool yield_after) { | |
408 assert(!_drain_in_progress || !_drain_in_progress_yields || yield_after | |
409 || SafepointSynchronize::is_at_safepoint(), | |
410 "Drain recursion must be yield-safe."); | |
411 bool res = true; | |
412 debug_only(_drain_in_progress = true); | |
413 debug_only(_drain_in_progress_yields = yield_after); | |
414 while (!isEmpty()) { | |
415 oop newOop = pop(); | |
416 assert(G1CollectedHeap::heap()->is_in_reserved(newOop), "Bad pop"); | |
417 assert(newOop->is_oop(), "Expected an oop"); | |
418 assert(bm == NULL || bm->isMarked((HeapWord*)newOop), | |
419 "only grey objects on this stack"); | |
420 // iterate over the oops in this oop, marking and pushing | |
421 // the ones in CMS generation. | |
422 newOop->oop_iterate(cl); | |
423 if (yield_after && _cm->do_yield_check()) { | |
424 res = false; break; | |
425 } | |
426 } | |
427 debug_only(_drain_in_progress = false); | |
428 return res; | |
429 } | |
430 | |
431 void CMMarkStack::oops_do(OopClosure* f) { | |
432 if (_index == 0) return; | |
433 assert(_oops_do_bound != -1 && _oops_do_bound <= _index, | |
434 "Bound must be set."); | |
435 for (int i = 0; i < _oops_do_bound; i++) { | |
436 f->do_oop(&_base[i]); | |
437 } | |
438 _oops_do_bound = -1; | |
439 } | |
440 | |
441 bool ConcurrentMark::not_yet_marked(oop obj) const { | |
442 return (_g1h->is_obj_ill(obj) | |
443 || (_g1h->is_in_permanent(obj) | |
444 && !nextMarkBitMap()->isMarked((HeapWord*)obj))); | |
445 } | |
446 | |
447 #ifdef _MSC_VER // the use of 'this' below gets a warning, make it go away | |
448 #pragma warning( disable:4355 ) // 'this' : used in base member initializer list | |
449 #endif // _MSC_VER | |
450 | |
451 ConcurrentMark::ConcurrentMark(ReservedSpace rs, | |
452 int max_regions) : | |
453 _markBitMap1(rs, MinObjAlignment - 1), | |
454 _markBitMap2(rs, MinObjAlignment - 1), | |
455 | |
456 _parallel_marking_threads(0), | |
457 _sleep_factor(0.0), | |
458 _marking_task_overhead(1.0), | |
459 _cleanup_sleep_factor(0.0), | |
460 _cleanup_task_overhead(1.0), | |
2152 | 461 _cleanup_list("Cleanup List"), |
342 | 462 _region_bm(max_regions, false /* in_resource_area*/), |
463 _card_bm((rs.size() + CardTableModRefBS::card_size - 1) >> | |
464 CardTableModRefBS::card_shift, | |
465 false /* in_resource_area*/), | |
466 _prevMarkBitMap(&_markBitMap1), | |
467 _nextMarkBitMap(&_markBitMap2), | |
468 _at_least_one_mark_complete(false), | |
469 | |
470 _markStack(this), | |
471 _regionStack(), | |
472 // _finger set in set_non_marking_state | |
473 | |
474 _max_task_num(MAX2(ParallelGCThreads, (size_t)1)), | |
475 // _active_tasks set in set_non_marking_state | |
476 // _tasks set inside the constructor | |
477 _task_queues(new CMTaskQueueSet((int) _max_task_num)), | |
478 _terminator(ParallelTaskTerminator((int) _max_task_num, _task_queues)), | |
479 | |
480 _has_overflown(false), | |
481 _concurrent(false), | |
619
7ea5ca260b28
6814467: G1: small fixes related to concurrent marking verboseness
tonyp
parents:
470
diff
changeset
|
482 _has_aborted(false), |
7ea5ca260b28
6814467: G1: small fixes related to concurrent marking verboseness
tonyp
parents:
470
diff
changeset
|
483 _restart_for_overflow(false), |
7ea5ca260b28
6814467: G1: small fixes related to concurrent marking verboseness
tonyp
parents:
470
diff
changeset
|
484 _concurrent_marking_in_progress(false), |
7ea5ca260b28
6814467: G1: small fixes related to concurrent marking verboseness
tonyp
parents:
470
diff
changeset
|
485 _should_gray_objects(false), |
342 | 486 |
487 // _verbose_level set below | |
488 | |
489 _init_times(), | |
490 _remark_times(), _remark_mark_times(), _remark_weak_ref_times(), | |
491 _cleanup_times(), | |
492 _total_counting_time(0.0), | |
493 _total_rs_scrub_time(0.0), | |
494 | |
936 | 495 _parallel_workers(NULL) |
342 | 496 { |
497 CMVerboseLevel verbose_level = | |
498 (CMVerboseLevel) G1MarkingVerboseLevel; | |
499 if (verbose_level < no_verbose) | |
500 verbose_level = no_verbose; | |
501 if (verbose_level > high_verbose) | |
502 verbose_level = high_verbose; | |
503 _verbose_level = verbose_level; | |
504 | |
505 if (verbose_low()) | |
506 gclog_or_tty->print_cr("[global] init, heap start = "PTR_FORMAT", " | |
507 "heap end = "PTR_FORMAT, _heap_start, _heap_end); | |
508 | |
1284 | 509 _markStack.allocate(MarkStackSize); |
751 | 510 _regionStack.allocate(G1MarkRegionStackSize); |
342 | 511 |
512 // Create & start a ConcurrentMark thread. | |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
513 _cmThread = new ConcurrentMarkThread(this); |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
514 assert(cmThread() != NULL, "CM Thread should have been created"); |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
515 assert(cmThread()->cm() != NULL, "CM Thread should refer to this cm"); |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
516 |
342 | 517 _g1h = G1CollectedHeap::heap(); |
518 assert(CGC_lock != NULL, "Where's the CGC_lock?"); | |
519 assert(_markBitMap1.covers(rs), "_markBitMap1 inconsistency"); | |
520 assert(_markBitMap2.covers(rs), "_markBitMap2 inconsistency"); | |
521 | |
522 SATBMarkQueueSet& satb_qs = JavaThread::satb_mark_queue_set(); | |
1282 | 523 satb_qs.set_buffer_size(G1SATBBufferSize); |
342 | 524 |
525 _tasks = NEW_C_HEAP_ARRAY(CMTask*, _max_task_num); | |
526 _accum_task_vtime = NEW_C_HEAP_ARRAY(double, _max_task_num); | |
527 | |
528 // so that the assertion in MarkingTaskQueue::task_queue doesn't fail | |
529 _active_tasks = _max_task_num; | |
530 for (int i = 0; i < (int) _max_task_num; ++i) { | |
531 CMTaskQueue* task_queue = new CMTaskQueue(); | |
532 task_queue->initialize(); | |
533 _task_queues->register_queue(i, task_queue); | |
534 | |
535 _tasks[i] = new CMTask(i, this, task_queue, _task_queues); | |
536 _accum_task_vtime[i] = 0.0; | |
537 } | |
538 | |
1284 | 539 if (ConcGCThreads > ParallelGCThreads) { |
540 vm_exit_during_initialization("Can't have more ConcGCThreads " | |
342 | 541 "than ParallelGCThreads."); |
542 } | |
543 if (ParallelGCThreads == 0) { | |
544 // if we are not running with any parallel GC threads we will not | |
545 // spawn any marking threads either | |
546 _parallel_marking_threads = 0; | |
547 _sleep_factor = 0.0; | |
548 _marking_task_overhead = 1.0; | |
549 } else { | |
1284 | 550 if (ConcGCThreads > 0) { |
551 // notice that ConcGCThreads overwrites G1MarkingOverheadPercent | |
342 | 552 // if both are set |
553 | |
1284 | 554 _parallel_marking_threads = ConcGCThreads; |
342 | 555 _sleep_factor = 0.0; |
556 _marking_task_overhead = 1.0; | |
751 | 557 } else if (G1MarkingOverheadPercent > 0) { |
342 | 558 // we will calculate the number of parallel marking threads |
559 // based on a target overhead with respect to the soft real-time | |
560 // goal | |
561 | |
751 | 562 double marking_overhead = (double) G1MarkingOverheadPercent / 100.0; |
342 | 563 double overall_cm_overhead = |
751 | 564 (double) MaxGCPauseMillis * marking_overhead / |
565 (double) GCPauseIntervalMillis; | |
342 | 566 double cpu_ratio = 1.0 / (double) os::processor_count(); |
567 double marking_thread_num = ceil(overall_cm_overhead / cpu_ratio); | |
568 double marking_task_overhead = | |
569 overall_cm_overhead / marking_thread_num * | |
570 (double) os::processor_count(); | |
571 double sleep_factor = | |
572 (1.0 - marking_task_overhead) / marking_task_overhead; | |
573 | |
574 _parallel_marking_threads = (size_t) marking_thread_num; | |
575 _sleep_factor = sleep_factor; | |
576 _marking_task_overhead = marking_task_overhead; | |
577 } else { | |
578 _parallel_marking_threads = MAX2((ParallelGCThreads + 2) / 4, (size_t)1); | |
579 _sleep_factor = 0.0; | |
580 _marking_task_overhead = 1.0; | |
581 } | |
582 | |
583 if (parallel_marking_threads() > 1) | |
584 _cleanup_task_overhead = 1.0; | |
585 else | |
586 _cleanup_task_overhead = marking_task_overhead(); | |
587 _cleanup_sleep_factor = | |
588 (1.0 - cleanup_task_overhead()) / cleanup_task_overhead(); | |
589 | |
590 #if 0 | |
591 gclog_or_tty->print_cr("Marking Threads %d", parallel_marking_threads()); | |
592 gclog_or_tty->print_cr("CM Marking Task Overhead %1.4lf", marking_task_overhead()); | |
593 gclog_or_tty->print_cr("CM Sleep Factor %1.4lf", sleep_factor()); | |
594 gclog_or_tty->print_cr("CL Marking Task Overhead %1.4lf", cleanup_task_overhead()); | |
595 gclog_or_tty->print_cr("CL Sleep Factor %1.4lf", cleanup_sleep_factor()); | |
596 #endif | |
597 | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
598 guarantee(parallel_marking_threads() > 0, "peace of mind"); |
1833
8b10f48633dc
6984287: Regularize how GC parallel workers are specified.
jmasa
parents:
1719
diff
changeset
|
599 _parallel_workers = new FlexibleWorkGang("G1 Parallel Marking Threads", |
8b10f48633dc
6984287: Regularize how GC parallel workers are specified.
jmasa
parents:
1719
diff
changeset
|
600 (int) _parallel_marking_threads, false, true); |
8b10f48633dc
6984287: Regularize how GC parallel workers are specified.
jmasa
parents:
1719
diff
changeset
|
601 if (_parallel_workers == NULL) { |
342 | 602 vm_exit_during_initialization("Failed necessary allocation."); |
1833
8b10f48633dc
6984287: Regularize how GC parallel workers are specified.
jmasa
parents:
1719
diff
changeset
|
603 } else { |
8b10f48633dc
6984287: Regularize how GC parallel workers are specified.
jmasa
parents:
1719
diff
changeset
|
604 _parallel_workers->initialize_workers(); |
8b10f48633dc
6984287: Regularize how GC parallel workers are specified.
jmasa
parents:
1719
diff
changeset
|
605 } |
342 | 606 } |
607 | |
608 // so that the call below can read a sensible value | |
609 _heap_start = (HeapWord*) rs.base(); | |
610 set_non_marking_state(); | |
611 } | |
612 | |
613 void ConcurrentMark::update_g1_committed(bool force) { | |
614 // If concurrent marking is not in progress, then we do not need to | |
615 // update _heap_end. This has a subtle and important | |
616 // side-effect. Imagine that two evacuation pauses happen between | |
617 // marking completion and remark. The first one can grow the | |
618 // heap (hence now the finger is below the heap end). Then, the | |
619 // second one could unnecessarily push regions on the region | |
620 // stack. This causes the invariant that the region stack is empty | |
621 // at the beginning of remark to be false. By ensuring that we do | |
622 // not observe heap expansions after marking is complete, then we do | |
623 // not have this problem. | |
624 if (!concurrent_marking_in_progress() && !force) | |
625 return; | |
626 | |
627 MemRegion committed = _g1h->g1_committed(); | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
628 assert(committed.start() == _heap_start, "start shouldn't change"); |
342 | 629 HeapWord* new_end = committed.end(); |
630 if (new_end > _heap_end) { | |
631 // The heap has been expanded. | |
632 | |
633 _heap_end = new_end; | |
634 } | |
635 // Notice that the heap can also shrink. However, this only happens | |
636 // during a Full GC (at least currently) and the entire marking | |
637 // phase will bail out and the task will not be restarted. So, let's | |
638 // do nothing. | |
639 } | |
640 | |
641 void ConcurrentMark::reset() { | |
642 // Starting values for these two. This should be called in a STW | |
643 // phase. CM will be notified of any future g1_committed expansions | |
644 // will be at the end of evacuation pauses, when tasks are | |
645 // inactive. | |
646 MemRegion committed = _g1h->g1_committed(); | |
647 _heap_start = committed.start(); | |
648 _heap_end = committed.end(); | |
649 | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
650 // Separated the asserts so that we know which one fires. |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
651 assert(_heap_start != NULL, "heap bounds should look ok"); |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
652 assert(_heap_end != NULL, "heap bounds should look ok"); |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
653 assert(_heap_start < _heap_end, "heap bounds should look ok"); |
342 | 654 |
655 // reset all the marking data structures and any necessary flags | |
656 clear_marking_state(); | |
657 | |
658 if (verbose_low()) | |
659 gclog_or_tty->print_cr("[global] resetting"); | |
660 | |
661 // We do reset all of them, since different phases will use | |
662 // different number of active threads. So, it's easiest to have all | |
663 // of them ready. | |
1835
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
664 for (int i = 0; i < (int) _max_task_num; ++i) { |
342 | 665 _tasks[i]->reset(_nextMarkBitMap); |
1835
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
666 } |
342 | 667 |
668 // we need this to make sure that the flag is on during the evac | |
669 // pause with initial mark piggy-backed | |
670 set_concurrent_marking_in_progress(); | |
671 } | |
672 | |
673 void ConcurrentMark::set_phase(size_t active_tasks, bool concurrent) { | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
674 assert(active_tasks <= _max_task_num, "we should not have more"); |
342 | 675 |
676 _active_tasks = active_tasks; | |
677 // Need to update the three data structures below according to the | |
678 // number of active threads for this phase. | |
679 _terminator = ParallelTaskTerminator((int) active_tasks, _task_queues); | |
680 _first_overflow_barrier_sync.set_n_workers((int) active_tasks); | |
681 _second_overflow_barrier_sync.set_n_workers((int) active_tasks); | |
682 | |
683 _concurrent = concurrent; | |
684 // We propagate this to all tasks, not just the active ones. | |
685 for (int i = 0; i < (int) _max_task_num; ++i) | |
686 _tasks[i]->set_concurrent(concurrent); | |
687 | |
688 if (concurrent) { | |
689 set_concurrent_marking_in_progress(); | |
690 } else { | |
691 // We currently assume that the concurrent flag has been set to | |
692 // false before we start remark. At this point we should also be | |
693 // in a STW phase. | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
694 assert(!concurrent_marking_in_progress(), "invariant"); |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
695 assert(_finger == _heap_end, "only way to get here"); |
342 | 696 update_g1_committed(true); |
697 } | |
698 } | |
699 | |
700 void ConcurrentMark::set_non_marking_state() { | |
701 // We set the global marking state to some default values when we're | |
702 // not doing marking. | |
703 clear_marking_state(); | |
704 _active_tasks = 0; | |
705 clear_concurrent_marking_in_progress(); | |
706 } | |
707 | |
708 ConcurrentMark::~ConcurrentMark() { | |
709 for (int i = 0; i < (int) _max_task_num; ++i) { | |
710 delete _task_queues->queue(i); | |
711 delete _tasks[i]; | |
712 } | |
713 delete _task_queues; | |
714 FREE_C_HEAP_ARRAY(CMTask*, _max_task_num); | |
715 } | |
716 | |
717 // This closure is used to mark refs into the g1 generation | |
718 // from external roots in the CMS bit map. | |
719 // Called at the first checkpoint. | |
720 // | |
721 | |
722 void ConcurrentMark::clearNextBitmap() { | |
1359
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
723 G1CollectedHeap* g1h = G1CollectedHeap::heap(); |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
724 G1CollectorPolicy* g1p = g1h->g1_policy(); |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
725 |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
726 // Make sure that the concurrent mark thread looks to still be in |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
727 // the current cycle. |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
728 guarantee(cmThread()->during_cycle(), "invariant"); |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
729 |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
730 // We are finishing up the current cycle by clearing the next |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
731 // marking bitmap and getting it ready for the next cycle. During |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
732 // this time no other cycle can start. So, let's make sure that this |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
733 // is the case. |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
734 guarantee(!g1h->mark_in_progress(), "invariant"); |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
735 |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
736 // clear the mark bitmap (no grey objects to start with). |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
737 // We need to do this in chunks and offer to yield in between |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
738 // each chunk. |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
739 HeapWord* start = _nextMarkBitMap->startWord(); |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
740 HeapWord* end = _nextMarkBitMap->endWord(); |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
741 HeapWord* cur = start; |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
742 size_t chunkSize = M; |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
743 while (cur < end) { |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
744 HeapWord* next = cur + chunkSize; |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
745 if (next > end) |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
746 next = end; |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
747 MemRegion mr(cur,next); |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
748 _nextMarkBitMap->clearRange(mr); |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
749 cur = next; |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
750 do_yield_check(); |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
751 |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
752 // Repeat the asserts from above. We'll do them as asserts here to |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
753 // minimize their overhead on the product. However, we'll have |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
754 // them as guarantees at the beginning / end of the bitmap |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
755 // clearing to get some checking in the product. |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
756 assert(cmThread()->during_cycle(), "invariant"); |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
757 assert(!g1h->mark_in_progress(), "invariant"); |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
758 } |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
759 |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
760 // Repeat the asserts from above. |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
761 guarantee(cmThread()->during_cycle(), "invariant"); |
23b1b27ac76c
6909756: G1: guarantee(G1CollectedHeap::heap()->mark_in_progress(),"Precondition.")
tonyp
parents:
1358
diff
changeset
|
762 guarantee(!g1h->mark_in_progress(), "invariant"); |
342 | 763 } |
764 | |
765 class NoteStartOfMarkHRClosure: public HeapRegionClosure { | |
766 public: | |
767 bool doHeapRegion(HeapRegion* r) { | |
768 if (!r->continuesHumongous()) { | |
769 r->note_start_of_marking(true); | |
770 } | |
771 return false; | |
772 } | |
773 }; | |
774 | |
775 void ConcurrentMark::checkpointRootsInitialPre() { | |
776 G1CollectedHeap* g1h = G1CollectedHeap::heap(); | |
777 G1CollectorPolicy* g1p = g1h->g1_policy(); | |
778 | |
779 _has_aborted = false; | |
780 | |
1547
fb1a39993f69
6951319: enable solaris builds using Sun Studio 12 update 1
jcoomes
parents:
1388
diff
changeset
|
781 #ifndef PRODUCT |
1044 | 782 if (G1PrintReachableAtInitialMark) { |
1388 | 783 print_reachable("at-cycle-start", |
784 true /* use_prev_marking */, true /* all */); | |
1044 | 785 } |
1547
fb1a39993f69
6951319: enable solaris builds using Sun Studio 12 update 1
jcoomes
parents:
1388
diff
changeset
|
786 #endif |
342 | 787 |
788 // Initialise marking structures. This has to be done in a STW phase. | |
789 reset(); | |
790 } | |
791 | |
792 class CMMarkRootsClosure: public OopsInGenClosure { | |
793 private: | |
794 ConcurrentMark* _cm; | |
795 G1CollectedHeap* _g1h; | |
796 bool _do_barrier; | |
797 | |
798 public: | |
799 CMMarkRootsClosure(ConcurrentMark* cm, | |
800 G1CollectedHeap* g1h, | |
801 bool do_barrier) : _cm(cm), _g1h(g1h), | |
802 _do_barrier(do_barrier) { } | |
803 | |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
804 virtual void do_oop(narrowOop* p) { do_oop_work(p); } |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
805 virtual void do_oop( oop* p) { do_oop_work(p); } |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
806 |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
807 template <class T> void do_oop_work(T* p) { |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
808 T heap_oop = oopDesc::load_heap_oop(p); |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
809 if (!oopDesc::is_null(heap_oop)) { |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
810 oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
811 assert(obj->is_oop() || obj->mark() == NULL, |
342 | 812 "expected an oop, possibly with mark word displaced"); |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
813 HeapWord* addr = (HeapWord*)obj; |
342 | 814 if (_g1h->is_in_g1_reserved(addr)) { |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
815 _cm->grayRoot(obj); |
342 | 816 } |
817 } | |
818 if (_do_barrier) { | |
819 assert(!_g1h->is_in_g1_reserved(p), | |
820 "Should be called on external roots"); | |
821 do_barrier(p); | |
822 } | |
823 } | |
824 }; | |
825 | |
826 void ConcurrentMark::checkpointRootsInitialPost() { | |
827 G1CollectedHeap* g1h = G1CollectedHeap::heap(); | |
828 | |
829 // For each region note start of marking. | |
830 NoteStartOfMarkHRClosure startcl; | |
831 g1h->heap_region_iterate(&startcl); | |
832 | |
833 // Start weak-reference discovery. | |
834 ReferenceProcessor* rp = g1h->ref_processor(); | |
835 rp->verify_no_references_recorded(); | |
836 rp->enable_discovery(); // enable ("weak") refs discovery | |
457
27a80744a83b
6778647: snap(), snap_policy() should be renamed setup(), setup_policy()
ysr
parents:
453
diff
changeset
|
837 rp->setup_policy(false); // snapshot the soft ref policy to be used in this cycle |
342 | 838 |
839 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); | |
1317
d4197f8d516a
6935821: G1: threads created during marking do not active their SATB queues
tonyp
parents:
1314
diff
changeset
|
840 // This is the start of the marking cycle, we're expected all |
d4197f8d516a
6935821: G1: threads created during marking do not active their SATB queues
tonyp
parents:
1314
diff
changeset
|
841 // threads to have SATB queues with active set to false. |
d4197f8d516a
6935821: G1: threads created during marking do not active their SATB queues
tonyp
parents:
1314
diff
changeset
|
842 satb_mq_set.set_active_all_threads(true, /* new active value */ |
d4197f8d516a
6935821: G1: threads created during marking do not active their SATB queues
tonyp
parents:
1314
diff
changeset
|
843 false /* expected_active */); |
342 | 844 |
845 // update_g1_committed() will be called at the end of an evac pause | |
846 // when marking is on. So, it's also called at the end of the | |
847 // initial-mark pause to update the heap end, if the heap expands | |
848 // during it. No need to call it here. | |
849 } | |
850 | |
851 // Checkpoint the roots into this generation from outside | |
852 // this generation. [Note this initial checkpoint need only | |
853 // be approximate -- we'll do a catch up phase subsequently.] | |
854 void ConcurrentMark::checkpointRootsInitial() { | |
855 assert(SafepointSynchronize::is_at_safepoint(), "world should be stopped"); | |
856 G1CollectedHeap* g1h = G1CollectedHeap::heap(); | |
857 | |
858 double start = os::elapsedTime(); | |
859 | |
860 G1CollectorPolicy* g1p = G1CollectedHeap::heap()->g1_policy(); | |
861 g1p->record_concurrent_mark_init_start(); | |
862 checkpointRootsInitialPre(); | |
863 | |
864 // YSR: when concurrent precleaning is in place, we'll | |
865 // need to clear the cached card table here | |
866 | |
867 ResourceMark rm; | |
868 HandleMark hm; | |
869 | |
870 g1h->ensure_parsability(false); | |
871 g1h->perm_gen()->save_marks(); | |
872 | |
873 CMMarkRootsClosure notOlder(this, g1h, false); | |
874 CMMarkRootsClosure older(this, g1h, true); | |
875 | |
876 g1h->set_marking_started(); | |
877 g1h->rem_set()->prepare_for_younger_refs_iterate(false); | |
878 | |
989
148e5441d916
6863023: need non-perm oops in code cache for JSR 292
jrose
parents:
912
diff
changeset
|
879 g1h->process_strong_roots(true, // activate StrongRootsScope |
148e5441d916
6863023: need non-perm oops in code cache for JSR 292
jrose
parents:
912
diff
changeset
|
880 false, // fake perm gen collection |
342 | 881 SharedHeap::SO_AllClasses, |
882 ¬Older, // Regular roots | |
989
148e5441d916
6863023: need non-perm oops in code cache for JSR 292
jrose
parents:
912
diff
changeset
|
883 NULL, // do not visit active blobs |
342 | 884 &older // Perm Gen Roots |
885 ); | |
886 checkpointRootsInitialPost(); | |
887 | |
888 // Statistics. | |
889 double end = os::elapsedTime(); | |
890 _init_times.add((end - start) * 1000.0); | |
891 | |
892 g1p->record_concurrent_mark_init_end(); | |
893 } | |
894 | |
895 /* | |
896 Notice that in the next two methods, we actually leave the STS | |
897 during the barrier sync and join it immediately afterwards. If we | |
898 do not do this, this then the following deadlock can occur: one | |
899 thread could be in the barrier sync code, waiting for the other | |
900 thread to also sync up, whereas another one could be trying to | |
901 yield, while also waiting for the other threads to sync up too. | |
902 | |
903 Because the thread that does the sync barrier has left the STS, it | |
904 is possible to be suspended for a Full GC or an evacuation pause | |
905 could occur. This is actually safe, since the entering the sync | |
906 barrier is one of the last things do_marking_step() does, and it | |
907 doesn't manipulate any data structures afterwards. | |
908 */ | |
909 | |
910 void ConcurrentMark::enter_first_sync_barrier(int task_num) { | |
911 if (verbose_low()) | |
912 gclog_or_tty->print_cr("[%d] entering first barrier", task_num); | |
913 | |
914 ConcurrentGCThread::stsLeave(); | |
915 _first_overflow_barrier_sync.enter(); | |
916 ConcurrentGCThread::stsJoin(); | |
917 // at this point everyone should have synced up and not be doing any | |
918 // more work | |
919 | |
920 if (verbose_low()) | |
921 gclog_or_tty->print_cr("[%d] leaving first barrier", task_num); | |
922 | |
923 // let task 0 do this | |
924 if (task_num == 0) { | |
925 // task 0 is responsible for clearing the global data structures | |
926 clear_marking_state(); | |
927 | |
928 if (PrintGC) { | |
929 gclog_or_tty->date_stamp(PrintGCDateStamps); | |
930 gclog_or_tty->stamp(PrintGCTimeStamps); | |
931 gclog_or_tty->print_cr("[GC concurrent-mark-reset-for-overflow]"); | |
932 } | |
933 } | |
934 | |
935 // after this, each task should reset its own data structures then | |
936 // then go into the second barrier | |
937 } | |
938 | |
939 void ConcurrentMark::enter_second_sync_barrier(int task_num) { | |
940 if (verbose_low()) | |
941 gclog_or_tty->print_cr("[%d] entering second barrier", task_num); | |
942 | |
943 ConcurrentGCThread::stsLeave(); | |
944 _second_overflow_barrier_sync.enter(); | |
945 ConcurrentGCThread::stsJoin(); | |
946 // at this point everything should be re-initialised and ready to go | |
947 | |
948 if (verbose_low()) | |
949 gclog_or_tty->print_cr("[%d] leaving second barrier", task_num); | |
950 } | |
951 | |
952 void ConcurrentMark::grayRoot(oop p) { | |
953 HeapWord* addr = (HeapWord*) p; | |
954 // We can't really check against _heap_start and _heap_end, since it | |
955 // is possible during an evacuation pause with piggy-backed | |
956 // initial-mark that the committed space is expanded during the | |
957 // pause without CM observing this change. So the assertions below | |
958 // is a bit conservative; but better than nothing. | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
959 assert(_g1h->g1_committed().contains(addr), |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
960 "address should be within the heap bounds"); |
342 | 961 |
962 if (!_nextMarkBitMap->isMarked(addr)) | |
963 _nextMarkBitMap->parMark(addr); | |
964 } | |
965 | |
966 void ConcurrentMark::grayRegionIfNecessary(MemRegion mr) { | |
967 // The objects on the region have already been marked "in bulk" by | |
968 // the caller. We only need to decide whether to push the region on | |
969 // the region stack or not. | |
970 | |
971 if (!concurrent_marking_in_progress() || !_should_gray_objects) | |
972 // We're done with marking and waiting for remark. We do not need to | |
973 // push anything else on the region stack. | |
974 return; | |
975 | |
976 HeapWord* finger = _finger; | |
977 | |
978 if (verbose_low()) | |
979 gclog_or_tty->print_cr("[global] attempting to push " | |
980 "region ["PTR_FORMAT", "PTR_FORMAT"), finger is at " | |
981 PTR_FORMAT, mr.start(), mr.end(), finger); | |
982 | |
983 if (mr.start() < finger) { | |
984 // The finger is always heap region aligned and it is not possible | |
985 // for mr to span heap regions. | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
986 assert(mr.end() <= finger, "invariant"); |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
987 |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
988 // Separated the asserts so that we know which one fires. |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
989 assert(mr.start() <= mr.end(), |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
990 "region boundaries should fall within the committed space"); |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
991 assert(_heap_start <= mr.start(), |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
992 "region boundaries should fall within the committed space"); |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
993 assert(mr.end() <= _heap_end, |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
994 "region boundaries should fall within the committed space"); |
342 | 995 if (verbose_low()) |
996 gclog_or_tty->print_cr("[global] region ["PTR_FORMAT", "PTR_FORMAT") " | |
997 "below the finger, pushing it", | |
998 mr.start(), mr.end()); | |
999 | |
1835
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
1000 if (!region_stack_push_lock_free(mr)) { |
342 | 1001 if (verbose_low()) |
1002 gclog_or_tty->print_cr("[global] region stack has overflown."); | |
1003 } | |
1004 } | |
1005 } | |
1006 | |
1007 void ConcurrentMark::markAndGrayObjectIfNecessary(oop p) { | |
1008 // The object is not marked by the caller. We need to at least mark | |
1009 // it and maybe push in on the stack. | |
1010 | |
1011 HeapWord* addr = (HeapWord*)p; | |
1012 if (!_nextMarkBitMap->isMarked(addr)) { | |
1013 // We definitely need to mark it, irrespective whether we bail out | |
1014 // because we're done with marking. | |
1015 if (_nextMarkBitMap->parMark(addr)) { | |
1016 if (!concurrent_marking_in_progress() || !_should_gray_objects) | |
1017 // If we're done with concurrent marking and we're waiting for | |
1018 // remark, then we're not pushing anything on the stack. | |
1019 return; | |
1020 | |
1021 // No OrderAccess:store_load() is needed. It is implicit in the | |
1022 // CAS done in parMark(addr) above | |
1023 HeapWord* finger = _finger; | |
1024 | |
1025 if (addr < finger) { | |
1026 if (!mark_stack_push(oop(addr))) { | |
1027 if (verbose_low()) | |
1028 gclog_or_tty->print_cr("[global] global stack overflow " | |
1029 "during parMark"); | |
1030 } | |
1031 } | |
1032 } | |
1033 } | |
1034 } | |
1035 | |
1036 class CMConcurrentMarkingTask: public AbstractGangTask { | |
1037 private: | |
1038 ConcurrentMark* _cm; | |
1039 ConcurrentMarkThread* _cmt; | |
1040 | |
1041 public: | |
1042 void work(int worker_i) { | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
1043 assert(Thread::current()->is_ConcurrentGC_thread(), |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
1044 "this should only be done by a conc GC thread"); |
1974
fd1d227ef1b9
6983204: G1: Nightly test nsk/regression/b4958615 failing with +ExplicitGCInvokesConcurrent
johnc
parents:
1972
diff
changeset
|
1045 ResourceMark rm; |
342 | 1046 |
1047 double start_vtime = os::elapsedVTime(); | |
1048 | |
1049 ConcurrentGCThread::stsJoin(); | |
1050 | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
1051 assert((size_t) worker_i < _cm->active_tasks(), "invariant"); |
342 | 1052 CMTask* the_task = _cm->task(worker_i); |
1053 the_task->record_start_time(); | |
1054 if (!_cm->has_aborted()) { | |
1055 do { | |
1056 double start_vtime_sec = os::elapsedVTime(); | |
1057 double start_time_sec = os::elapsedTime(); | |
1058 the_task->do_marking_step(10.0); | |
1059 double end_time_sec = os::elapsedTime(); | |
1060 double end_vtime_sec = os::elapsedVTime(); | |
1061 double elapsed_vtime_sec = end_vtime_sec - start_vtime_sec; | |
1062 double elapsed_time_sec = end_time_sec - start_time_sec; | |
1063 _cm->clear_has_overflown(); | |
1064 | |
1065 bool ret = _cm->do_yield_check(worker_i); | |
1066 | |
1067 jlong sleep_time_ms; | |
1068 if (!_cm->has_aborted() && the_task->has_aborted()) { | |
1069 sleep_time_ms = | |
1070 (jlong) (elapsed_vtime_sec * _cm->sleep_factor() * 1000.0); | |
1071 ConcurrentGCThread::stsLeave(); | |
1072 os::sleep(Thread::current(), sleep_time_ms, false); | |
1073 ConcurrentGCThread::stsJoin(); | |
1074 } | |
1075 double end_time2_sec = os::elapsedTime(); | |
1076 double elapsed_time2_sec = end_time2_sec - start_time_sec; | |
1077 | |
1078 #if 0 | |
1079 gclog_or_tty->print_cr("CM: elapsed %1.4lf ms, sleep %1.4lf ms, " | |
1080 "overhead %1.4lf", | |
1081 elapsed_vtime_sec * 1000.0, (double) sleep_time_ms, | |
1082 the_task->conc_overhead(os::elapsedTime()) * 8.0); | |
1083 gclog_or_tty->print_cr("elapsed time %1.4lf ms, time 2: %1.4lf ms", | |
1084 elapsed_time_sec * 1000.0, elapsed_time2_sec * 1000.0); | |
1085 #endif | |
1086 } while (!_cm->has_aborted() && the_task->has_aborted()); | |
1087 } | |
1088 the_task->record_end_time(); | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
1089 guarantee(!the_task->has_aborted() || _cm->has_aborted(), "invariant"); |
342 | 1090 |
1091 ConcurrentGCThread::stsLeave(); | |
1092 | |
1093 double end_vtime = os::elapsedVTime(); | |
1094 _cm->update_accum_task_vtime(worker_i, end_vtime - start_vtime); | |
1095 } | |
1096 | |
1097 CMConcurrentMarkingTask(ConcurrentMark* cm, | |
1098 ConcurrentMarkThread* cmt) : | |
1099 AbstractGangTask("Concurrent Mark"), _cm(cm), _cmt(cmt) { } | |
1100 | |
1101 ~CMConcurrentMarkingTask() { } | |
1102 }; | |
1103 | |
1104 void ConcurrentMark::markFromRoots() { | |
1105 // we might be tempted to assert that: | |
1106 // assert(asynch == !SafepointSynchronize::is_at_safepoint(), | |
1107 // "inconsistent argument?"); | |
1108 // However that wouldn't be right, because it's possible that | |
1109 // a safepoint is indeed in progress as a younger generation | |
1110 // stop-the-world GC happens even as we mark in this generation. | |
1111 | |
1112 _restart_for_overflow = false; | |
1113 | |
1114 set_phase(MAX2((size_t) 1, parallel_marking_threads()), true); | |
1115 | |
1116 CMConcurrentMarkingTask markingTask(this, cmThread()); | |
1117 if (parallel_marking_threads() > 0) | |
1118 _parallel_workers->run_task(&markingTask); | |
1119 else | |
1120 markingTask.work(0); | |
1121 print_stats(); | |
1122 } | |
1123 | |
1124 void ConcurrentMark::checkpointRootsFinal(bool clear_all_soft_refs) { | |
1125 // world is stopped at this checkpoint | |
1126 assert(SafepointSynchronize::is_at_safepoint(), | |
1127 "world should be stopped"); | |
1128 G1CollectedHeap* g1h = G1CollectedHeap::heap(); | |
1129 | |
1130 // If a full collection has happened, we shouldn't do this. | |
1131 if (has_aborted()) { | |
1132 g1h->set_marking_complete(); // So bitmap clearing isn't confused | |
1133 return; | |
1134 } | |
1135 | |
2125
7246a374a9f2
6458402: 3 jvmti tests fail with CMS and +ExplicitGCInvokesConcurrent
kamg
parents:
2037
diff
changeset
|
1136 SvcGCMarker sgcm(SvcGCMarker::OTHER); |
7246a374a9f2
6458402: 3 jvmti tests fail with CMS and +ExplicitGCInvokesConcurrent
kamg
parents:
2037
diff
changeset
|
1137 |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1138 if (VerifyDuringGC) { |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1139 HandleMark hm; // handle scope |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1140 gclog_or_tty->print(" VerifyDuringGC:(before)"); |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1141 Universe::heap()->prepare_for_verify(); |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1142 Universe::verify(true, false, true); |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1143 } |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1144 |
342 | 1145 G1CollectorPolicy* g1p = g1h->g1_policy(); |
1146 g1p->record_concurrent_mark_remark_start(); | |
1147 | |
1148 double start = os::elapsedTime(); | |
1149 | |
1150 checkpointRootsFinalWork(); | |
1151 | |
1152 double mark_work_end = os::elapsedTime(); | |
1153 | |
1154 weakRefsWork(clear_all_soft_refs); | |
1155 | |
1156 if (has_overflown()) { | |
1157 // Oops. We overflowed. Restart concurrent marking. | |
1158 _restart_for_overflow = true; | |
1159 // Clear the flag. We do not need it any more. | |
1160 clear_has_overflown(); | |
1161 if (G1TraceMarkStackOverflow) | |
1162 gclog_or_tty->print_cr("\nRemark led to restart for overflow."); | |
1163 } else { | |
2149 | 1164 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); |
342 | 1165 // We're done with marking. |
1317
d4197f8d516a
6935821: G1: threads created during marking do not active their SATB queues
tonyp
parents:
1314
diff
changeset
|
1166 // This is the end of the marking cycle, we're expected all |
d4197f8d516a
6935821: G1: threads created during marking do not active their SATB queues
tonyp
parents:
1314
diff
changeset
|
1167 // threads to have SATB queues with active set to true. |
2149 | 1168 satb_mq_set.set_active_all_threads(false, /* new active value */ |
1169 true /* expected_active */); | |
811 | 1170 |
1171 if (VerifyDuringGC) { | |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1172 HandleMark hm; // handle scope |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1173 gclog_or_tty->print(" VerifyDuringGC:(after)"); |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1174 Universe::heap()->prepare_for_verify(); |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1175 Universe::heap()->verify(/* allow_dirty */ true, |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1176 /* silent */ false, |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1177 /* use_prev_marking */ false); |
811 | 1178 } |
342 | 1179 } |
1180 | |
1181 #if VERIFY_OBJS_PROCESSED | |
1182 _scan_obj_cl.objs_processed = 0; | |
1183 ThreadLocalObjQueue::objs_enqueued = 0; | |
1184 #endif | |
1185 | |
1186 // Statistics | |
1187 double now = os::elapsedTime(); | |
1188 _remark_mark_times.add((mark_work_end - start) * 1000.0); | |
1189 _remark_weak_ref_times.add((now - mark_work_end) * 1000.0); | |
1190 _remark_times.add((now - start) * 1000.0); | |
1191 | |
1192 g1p->record_concurrent_mark_remark_end(); | |
1193 } | |
1194 | |
1195 | |
1196 #define CARD_BM_TEST_MODE 0 | |
1197 | |
1198 class CalcLiveObjectsClosure: public HeapRegionClosure { | |
1199 | |
1200 CMBitMapRO* _bm; | |
1201 ConcurrentMark* _cm; | |
1202 bool _changed; | |
1203 bool _yield; | |
1204 size_t _words_done; | |
1205 size_t _tot_live; | |
1206 size_t _tot_used; | |
1207 size_t _regions_done; | |
1208 double _start_vtime_sec; | |
1209 | |
1210 BitMap* _region_bm; | |
1211 BitMap* _card_bm; | |
1212 intptr_t _bottom_card_num; | |
1213 bool _final; | |
1214 | |
1215 void mark_card_num_range(intptr_t start_card_num, intptr_t last_card_num) { | |
1216 for (intptr_t i = start_card_num; i <= last_card_num; i++) { | |
1217 #if CARD_BM_TEST_MODE | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
1218 guarantee(_card_bm->at(i - _bottom_card_num), "Should already be set."); |
342 | 1219 #else |
1220 _card_bm->par_at_put(i - _bottom_card_num, 1); | |
1221 #endif | |
1222 } | |
1223 } | |
1224 | |
1225 public: | |
1226 CalcLiveObjectsClosure(bool final, | |
1227 CMBitMapRO *bm, ConcurrentMark *cm, | |
936 | 1228 BitMap* region_bm, BitMap* card_bm) : |
342 | 1229 _bm(bm), _cm(cm), _changed(false), _yield(true), |
1230 _words_done(0), _tot_live(0), _tot_used(0), | |
936 | 1231 _region_bm(region_bm), _card_bm(card_bm),_final(final), |
342 | 1232 _regions_done(0), _start_vtime_sec(0.0) |
1233 { | |
1234 _bottom_card_num = | |
1235 intptr_t(uintptr_t(G1CollectedHeap::heap()->reserved_region().start()) >> | |
1236 CardTableModRefBS::card_shift); | |
1237 } | |
1238 | |
829 | 1239 // It takes a region that's not empty (i.e., it has at least one |
1240 // live object in it and sets its corresponding bit on the region | |
1241 // bitmap to 1. If the region is "starts humongous" it will also set | |
1242 // to 1 the bits on the region bitmap that correspond to its | |
1243 // associated "continues humongous" regions. | |
1244 void set_bit_for_region(HeapRegion* hr) { | |
1245 assert(!hr->continuesHumongous(), "should have filtered those out"); | |
1246 | |
1247 size_t index = hr->hrs_index(); | |
1248 if (!hr->startsHumongous()) { | |
1249 // Normal (non-humongous) case: just set the bit. | |
1250 _region_bm->par_at_put((BitMap::idx_t) index, true); | |
1251 } else { | |
1252 // Starts humongous case: calculate how many regions are part of | |
1253 // this humongous region and then set the bit range. It might | |
1254 // have been a bit more efficient to look at the object that | |
1255 // spans these humongous regions to calculate their number from | |
1256 // the object's size. However, it's a good idea to calculate | |
1257 // this based on the metadata itself, and not the region | |
1258 // contents, so that this code is not aware of what goes into | |
1259 // the humongous regions (in case this changes in the future). | |
1260 G1CollectedHeap* g1h = G1CollectedHeap::heap(); | |
1261 size_t end_index = index + 1; | |
831 | 1262 while (end_index < g1h->n_regions()) { |
1263 HeapRegion* chr = g1h->region_at(end_index); | |
829 | 1264 if (!chr->continuesHumongous()) { |
1265 break; | |
1266 } | |
1267 end_index += 1; | |
1268 } | |
1269 _region_bm->par_at_put_range((BitMap::idx_t) index, | |
1270 (BitMap::idx_t) end_index, true); | |
1271 } | |
1272 } | |
1273 | |
342 | 1274 bool doHeapRegion(HeapRegion* hr) { |
1275 if (!_final && _regions_done == 0) | |
1276 _start_vtime_sec = os::elapsedVTime(); | |
1277 | |
639 | 1278 if (hr->continuesHumongous()) { |
829 | 1279 // We will ignore these here and process them when their |
1280 // associated "starts humongous" region is processed (see | |
1281 // set_bit_for_heap_region()). Note that we cannot rely on their | |
1282 // associated "starts humongous" region to have their bit set to | |
1283 // 1 since, due to the region chunking in the parallel region | |
1284 // iteration, a "continues humongous" region might be visited | |
1285 // before its associated "starts humongous". | |
639 | 1286 return false; |
1287 } | |
342 | 1288 |
1289 HeapWord* nextTop = hr->next_top_at_mark_start(); | |
1290 HeapWord* start = hr->top_at_conc_mark_count(); | |
1291 assert(hr->bottom() <= start && start <= hr->end() && | |
1292 hr->bottom() <= nextTop && nextTop <= hr->end() && | |
1293 start <= nextTop, | |
1294 "Preconditions."); | |
1295 // Otherwise, record the number of word's we'll examine. | |
1296 size_t words_done = (nextTop - start); | |
1297 // Find the first marked object at or after "start". | |
1298 start = _bm->getNextMarkedWordAddress(start, nextTop); | |
1299 size_t marked_bytes = 0; | |
1300 | |
1301 // Below, the term "card num" means the result of shifting an address | |
1302 // by the card shift -- address 0 corresponds to card number 0. One | |
1303 // must subtract the card num of the bottom of the heap to obtain a | |
1304 // card table index. | |
1305 // The first card num of the sequence of live cards currently being | |
1306 // constructed. -1 ==> no sequence. | |
1307 intptr_t start_card_num = -1; | |
1308 // The last card num of the sequence of live cards currently being | |
1309 // constructed. -1 ==> no sequence. | |
1310 intptr_t last_card_num = -1; | |
1311 | |
1312 while (start < nextTop) { | |
1313 if (_yield && _cm->do_yield_check()) { | |
1314 // We yielded. It might be for a full collection, in which case | |
1315 // all bets are off; terminate the traversal. | |
1316 if (_cm->has_aborted()) { | |
1317 _changed = false; | |
1318 return true; | |
1319 } else { | |
1320 // Otherwise, it might be a collection pause, and the region | |
1321 // we're looking at might be in the collection set. We'll | |
1322 // abandon this region. | |
1323 return false; | |
1324 } | |
1325 } | |
1326 oop obj = oop(start); | |
1327 int obj_sz = obj->size(); | |
1328 // The card num of the start of the current object. | |
1329 intptr_t obj_card_num = | |
1330 intptr_t(uintptr_t(start) >> CardTableModRefBS::card_shift); | |
1331 | |
1332 HeapWord* obj_last = start + obj_sz - 1; | |
1333 intptr_t obj_last_card_num = | |
1334 intptr_t(uintptr_t(obj_last) >> CardTableModRefBS::card_shift); | |
1335 | |
1336 if (obj_card_num != last_card_num) { | |
1337 if (start_card_num == -1) { | |
1338 assert(last_card_num == -1, "Both or neither."); | |
1339 start_card_num = obj_card_num; | |
1340 } else { | |
1341 assert(last_card_num != -1, "Both or neither."); | |
1342 assert(obj_card_num >= last_card_num, "Inv"); | |
1343 if ((obj_card_num - last_card_num) > 1) { | |
1344 // Mark the last run, and start a new one. | |
1345 mark_card_num_range(start_card_num, last_card_num); | |
1346 start_card_num = obj_card_num; | |
1347 } | |
1348 } | |
1349 #if CARD_BM_TEST_MODE | |
1350 /* | |
1351 gclog_or_tty->print_cr("Setting bits from %d/%d.", | |
1352 obj_card_num - _bottom_card_num, | |
1353 obj_last_card_num - _bottom_card_num); | |
1354 */ | |
1355 for (intptr_t j = obj_card_num; j <= obj_last_card_num; j++) { | |
1356 _card_bm->par_at_put(j - _bottom_card_num, 1); | |
1357 } | |
1358 #endif | |
1359 } | |
1360 // In any case, we set the last card num. | |
1361 last_card_num = obj_last_card_num; | |
1362 | |
1030
dfdaf65c3423
6858886: G1: guarantee(_next_marked_bytes <= used(),"invariant") at heapRegion.hpp:359
apetrusenko
parents:
1023
diff
changeset
|
1363 marked_bytes += (size_t)obj_sz * HeapWordSize; |
342 | 1364 // Find the next marked object after this one. |
1365 start = _bm->getNextMarkedWordAddress(start + 1, nextTop); | |
1366 _changed = true; | |
1367 } | |
1368 // Handle the last range, if any. | |
1369 if (start_card_num != -1) | |
1370 mark_card_num_range(start_card_num, last_card_num); | |
1371 if (_final) { | |
1372 // Mark the allocated-since-marking portion... | |
1373 HeapWord* tp = hr->top(); | |
1374 if (nextTop < tp) { | |
1375 start_card_num = | |
1376 intptr_t(uintptr_t(nextTop) >> CardTableModRefBS::card_shift); | |
1377 last_card_num = | |
1378 intptr_t(uintptr_t(tp) >> CardTableModRefBS::card_shift); | |
1379 mark_card_num_range(start_card_num, last_card_num); | |
1380 // This definitely means the region has live objects. | |
829 | 1381 set_bit_for_region(hr); |
342 | 1382 } |
1383 } | |
1384 | |
1385 hr->add_to_marked_bytes(marked_bytes); | |
1386 // Update the live region bitmap. | |
1387 if (marked_bytes > 0) { | |
829 | 1388 set_bit_for_region(hr); |
342 | 1389 } |
1390 hr->set_top_at_conc_mark_count(nextTop); | |
1391 _tot_live += hr->next_live_bytes(); | |
1392 _tot_used += hr->used(); | |
1393 _words_done = words_done; | |
1394 | |
1395 if (!_final) { | |
1396 ++_regions_done; | |
1397 if (_regions_done % 10 == 0) { | |
1398 double end_vtime_sec = os::elapsedVTime(); | |
1399 double elapsed_vtime_sec = end_vtime_sec - _start_vtime_sec; | |
1400 if (elapsed_vtime_sec > (10.0 / 1000.0)) { | |
1401 jlong sleep_time_ms = | |
1402 (jlong) (elapsed_vtime_sec * _cm->cleanup_sleep_factor() * 1000.0); | |
1403 os::sleep(Thread::current(), sleep_time_ms, false); | |
1404 _start_vtime_sec = end_vtime_sec; | |
1405 } | |
1406 } | |
1407 } | |
1408 | |
1409 return false; | |
1410 } | |
1411 | |
1412 bool changed() { return _changed; } | |
1413 void reset() { _changed = false; _words_done = 0; } | |
1414 void no_yield() { _yield = false; } | |
1415 size_t words_done() { return _words_done; } | |
1416 size_t tot_live() { return _tot_live; } | |
1417 size_t tot_used() { return _tot_used; } | |
1418 }; | |
1419 | |
1420 | |
1421 void ConcurrentMark::calcDesiredRegions() { | |
1422 _region_bm.clear(); | |
1423 _card_bm.clear(); | |
1424 CalcLiveObjectsClosure calccl(false /*final*/, | |
1425 nextMarkBitMap(), this, | |
936 | 1426 &_region_bm, &_card_bm); |
342 | 1427 G1CollectedHeap *g1h = G1CollectedHeap::heap(); |
1428 g1h->heap_region_iterate(&calccl); | |
1429 | |
1430 do { | |
1431 calccl.reset(); | |
1432 g1h->heap_region_iterate(&calccl); | |
1433 } while (calccl.changed()); | |
1434 } | |
1435 | |
1436 class G1ParFinalCountTask: public AbstractGangTask { | |
1437 protected: | |
1438 G1CollectedHeap* _g1h; | |
1439 CMBitMap* _bm; | |
1440 size_t _n_workers; | |
1441 size_t *_live_bytes; | |
1442 size_t *_used_bytes; | |
1443 BitMap* _region_bm; | |
1444 BitMap* _card_bm; | |
1445 public: | |
1446 G1ParFinalCountTask(G1CollectedHeap* g1h, CMBitMap* bm, | |
1447 BitMap* region_bm, BitMap* card_bm) : | |
1448 AbstractGangTask("G1 final counting"), _g1h(g1h), | |
1449 _bm(bm), _region_bm(region_bm), _card_bm(card_bm) | |
1450 { | |
1451 if (ParallelGCThreads > 0) | |
1452 _n_workers = _g1h->workers()->total_workers(); | |
1453 else | |
1454 _n_workers = 1; | |
1455 _live_bytes = NEW_C_HEAP_ARRAY(size_t, _n_workers); | |
1456 _used_bytes = NEW_C_HEAP_ARRAY(size_t, _n_workers); | |
1457 } | |
1458 | |
1459 ~G1ParFinalCountTask() { | |
1460 FREE_C_HEAP_ARRAY(size_t, _live_bytes); | |
1461 FREE_C_HEAP_ARRAY(size_t, _used_bytes); | |
1462 } | |
1463 | |
1464 void work(int i) { | |
1465 CalcLiveObjectsClosure calccl(true /*final*/, | |
1466 _bm, _g1h->concurrent_mark(), | |
936 | 1467 _region_bm, _card_bm); |
342 | 1468 calccl.no_yield(); |
1833
8b10f48633dc
6984287: Regularize how GC parallel workers are specified.
jmasa
parents:
1719
diff
changeset
|
1469 if (G1CollectedHeap::use_parallel_gc_threads()) { |
355 | 1470 _g1h->heap_region_par_iterate_chunked(&calccl, i, |
1471 HeapRegion::FinalCountClaimValue); | |
342 | 1472 } else { |
1473 _g1h->heap_region_iterate(&calccl); | |
1474 } | |
1475 assert(calccl.complete(), "Shouldn't have yielded!"); | |
1476 | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
1477 assert((size_t) i < _n_workers, "invariant"); |
342 | 1478 _live_bytes[i] = calccl.tot_live(); |
1479 _used_bytes[i] = calccl.tot_used(); | |
1480 } | |
1481 size_t live_bytes() { | |
1482 size_t live_bytes = 0; | |
1483 for (size_t i = 0; i < _n_workers; ++i) | |
1484 live_bytes += _live_bytes[i]; | |
1485 return live_bytes; | |
1486 } | |
1487 size_t used_bytes() { | |
1488 size_t used_bytes = 0; | |
1489 for (size_t i = 0; i < _n_workers; ++i) | |
1490 used_bytes += _used_bytes[i]; | |
1491 return used_bytes; | |
1492 } | |
1493 }; | |
1494 | |
1495 class G1ParNoteEndTask; | |
1496 | |
1497 class G1NoteEndOfConcMarkClosure : public HeapRegionClosure { | |
1498 G1CollectedHeap* _g1; | |
1499 int _worker_num; | |
1500 size_t _max_live_bytes; | |
1501 size_t _regions_claimed; | |
1502 size_t _freed_bytes; | |
2173 | 1503 FreeRegionList* _local_cleanup_list; |
1504 HumongousRegionSet* _humongous_proxy_set; | |
1505 HRRSCleanupTask* _hrrs_cleanup_task; | |
342 | 1506 double _claimed_region_time; |
1507 double _max_region_time; | |
1508 | |
1509 public: | |
1510 G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1, | |
2173 | 1511 int worker_num, |
1512 FreeRegionList* local_cleanup_list, | |
1513 HumongousRegionSet* humongous_proxy_set, | |
1514 HRRSCleanupTask* hrrs_cleanup_task); | |
342 | 1515 size_t freed_bytes() { return _freed_bytes; } |
1516 | |
1517 bool doHeapRegion(HeapRegion *r); | |
1518 | |
1519 size_t max_live_bytes() { return _max_live_bytes; } | |
1520 size_t regions_claimed() { return _regions_claimed; } | |
1521 double claimed_region_time_sec() { return _claimed_region_time; } | |
1522 double max_region_time_sec() { return _max_region_time; } | |
1523 }; | |
1524 | |
1525 class G1ParNoteEndTask: public AbstractGangTask { | |
1526 friend class G1NoteEndOfConcMarkClosure; | |
2152 | 1527 |
342 | 1528 protected: |
1529 G1CollectedHeap* _g1h; | |
1530 size_t _max_live_bytes; | |
1531 size_t _freed_bytes; | |
2152 | 1532 FreeRegionList* _cleanup_list; |
1533 | |
342 | 1534 public: |
1535 G1ParNoteEndTask(G1CollectedHeap* g1h, | |
2152 | 1536 FreeRegionList* cleanup_list) : |
342 | 1537 AbstractGangTask("G1 note end"), _g1h(g1h), |
2152 | 1538 _max_live_bytes(0), _freed_bytes(0), _cleanup_list(cleanup_list) { } |
342 | 1539 |
1540 void work(int i) { | |
1541 double start = os::elapsedTime(); | |
2173 | 1542 FreeRegionList local_cleanup_list("Local Cleanup List"); |
1543 HumongousRegionSet humongous_proxy_set("Local Cleanup Humongous Proxy Set"); | |
1544 HRRSCleanupTask hrrs_cleanup_task; | |
1545 G1NoteEndOfConcMarkClosure g1_note_end(_g1h, i, &local_cleanup_list, | |
1546 &humongous_proxy_set, | |
1547 &hrrs_cleanup_task); | |
1833
8b10f48633dc
6984287: Regularize how GC parallel workers are specified.
jmasa
parents:
1719
diff
changeset
|
1548 if (G1CollectedHeap::use_parallel_gc_threads()) { |
355 | 1549 _g1h->heap_region_par_iterate_chunked(&g1_note_end, i, |
1550 HeapRegion::NoteEndClaimValue); | |
342 | 1551 } else { |
1552 _g1h->heap_region_iterate(&g1_note_end); | |
1553 } | |
1554 assert(g1_note_end.complete(), "Shouldn't have yielded!"); | |
1555 | |
2152 | 1556 // Now update the lists |
1557 _g1h->update_sets_after_freeing_regions(g1_note_end.freed_bytes(), | |
1558 NULL /* free_list */, | |
2173 | 1559 &humongous_proxy_set, |
2152 | 1560 true /* par */); |
342 | 1561 { |
1562 MutexLockerEx x(ParGCRareEvent_lock, Mutex::_no_safepoint_check_flag); | |
1563 _max_live_bytes += g1_note_end.max_live_bytes(); | |
1564 _freed_bytes += g1_note_end.freed_bytes(); | |
2152 | 1565 |
2173 | 1566 _cleanup_list->add_as_tail(&local_cleanup_list); |
1567 assert(local_cleanup_list.is_empty(), "post-condition"); | |
1568 | |
1569 HeapRegionRemSet::finish_cleanup_task(&hrrs_cleanup_task); | |
342 | 1570 } |
1571 double end = os::elapsedTime(); | |
1572 if (G1PrintParCleanupStats) { | |
1573 gclog_or_tty->print(" Worker thread %d [%8.3f..%8.3f = %8.3f ms] " | |
1574 "claimed %d regions (tot = %8.3f ms, max = %8.3f ms).\n", | |
1575 i, start, end, (end-start)*1000.0, | |
1576 g1_note_end.regions_claimed(), | |
1577 g1_note_end.claimed_region_time_sec()*1000.0, | |
1578 g1_note_end.max_region_time_sec()*1000.0); | |
1579 } | |
1580 } | |
1581 size_t max_live_bytes() { return _max_live_bytes; } | |
1582 size_t freed_bytes() { return _freed_bytes; } | |
1583 }; | |
1584 | |
1585 class G1ParScrubRemSetTask: public AbstractGangTask { | |
1586 protected: | |
1587 G1RemSet* _g1rs; | |
1588 BitMap* _region_bm; | |
1589 BitMap* _card_bm; | |
1590 public: | |
1591 G1ParScrubRemSetTask(G1CollectedHeap* g1h, | |
1592 BitMap* region_bm, BitMap* card_bm) : | |
1593 AbstractGangTask("G1 ScrubRS"), _g1rs(g1h->g1_rem_set()), | |
1594 _region_bm(region_bm), _card_bm(card_bm) | |
1595 {} | |
1596 | |
1597 void work(int i) { | |
1833
8b10f48633dc
6984287: Regularize how GC parallel workers are specified.
jmasa
parents:
1719
diff
changeset
|
1598 if (G1CollectedHeap::use_parallel_gc_threads()) { |
355 | 1599 _g1rs->scrub_par(_region_bm, _card_bm, i, |
1600 HeapRegion::ScrubRemSetClaimValue); | |
342 | 1601 } else { |
1602 _g1rs->scrub(_region_bm, _card_bm); | |
1603 } | |
1604 } | |
1605 | |
1606 }; | |
1607 | |
1608 G1NoteEndOfConcMarkClosure:: | |
1609 G1NoteEndOfConcMarkClosure(G1CollectedHeap* g1, | |
2173 | 1610 int worker_num, |
1611 FreeRegionList* local_cleanup_list, | |
1612 HumongousRegionSet* humongous_proxy_set, | |
1613 HRRSCleanupTask* hrrs_cleanup_task) | |
342 | 1614 : _g1(g1), _worker_num(worker_num), |
1615 _max_live_bytes(0), _regions_claimed(0), | |
2152 | 1616 _freed_bytes(0), |
342 | 1617 _claimed_region_time(0.0), _max_region_time(0.0), |
2173 | 1618 _local_cleanup_list(local_cleanup_list), |
1619 _humongous_proxy_set(humongous_proxy_set), | |
1620 _hrrs_cleanup_task(hrrs_cleanup_task) { } | |
2152 | 1621 |
1622 bool G1NoteEndOfConcMarkClosure::doHeapRegion(HeapRegion *hr) { | |
342 | 1623 // We use a claim value of zero here because all regions |
1624 // were claimed with value 1 in the FinalCount task. | |
2152 | 1625 hr->reset_gc_time_stamp(); |
1626 if (!hr->continuesHumongous()) { | |
342 | 1627 double start = os::elapsedTime(); |
1628 _regions_claimed++; | |
2152 | 1629 hr->note_end_of_marking(); |
1630 _max_live_bytes += hr->max_live_bytes(); | |
2173 | 1631 _g1->free_region_if_empty(hr, |
1632 &_freed_bytes, | |
1633 _local_cleanup_list, | |
1634 _humongous_proxy_set, | |
1635 _hrrs_cleanup_task, | |
1636 true /* par */); | |
342 | 1637 double region_time = (os::elapsedTime() - start); |
1638 _claimed_region_time += region_time; | |
1639 if (region_time > _max_region_time) _max_region_time = region_time; | |
1640 } | |
1641 return false; | |
1642 } | |
1643 | |
1644 void ConcurrentMark::cleanup() { | |
1645 // world is stopped at this checkpoint | |
1646 assert(SafepointSynchronize::is_at_safepoint(), | |
1647 "world should be stopped"); | |
1648 G1CollectedHeap* g1h = G1CollectedHeap::heap(); | |
1649 | |
1650 // If a full collection has happened, we shouldn't do this. | |
1651 if (has_aborted()) { | |
1652 g1h->set_marking_complete(); // So bitmap clearing isn't confused | |
1653 return; | |
1654 } | |
1655 | |
2152 | 1656 g1h->verify_region_sets_optional(); |
1657 | |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1658 if (VerifyDuringGC) { |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1659 HandleMark hm; // handle scope |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1660 gclog_or_tty->print(" VerifyDuringGC:(before)"); |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1661 Universe::heap()->prepare_for_verify(); |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1662 Universe::verify(/* allow dirty */ true, |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1663 /* silent */ false, |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1664 /* prev marking */ true); |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1665 } |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1666 |
342 | 1667 G1CollectorPolicy* g1p = G1CollectedHeap::heap()->g1_policy(); |
1668 g1p->record_concurrent_mark_cleanup_start(); | |
1669 | |
1670 double start = os::elapsedTime(); | |
1671 | |
2173 | 1672 HeapRegionRemSet::reset_for_cleanup_tasks(); |
1673 | |
342 | 1674 // Do counting once more with the world stopped for good measure. |
1675 G1ParFinalCountTask g1_par_count_task(g1h, nextMarkBitMap(), | |
1676 &_region_bm, &_card_bm); | |
1833
8b10f48633dc
6984287: Regularize how GC parallel workers are specified.
jmasa
parents:
1719
diff
changeset
|
1677 if (G1CollectedHeap::use_parallel_gc_threads()) { |
355 | 1678 assert(g1h->check_heap_region_claim_values( |
1679 HeapRegion::InitialClaimValue), | |
1680 "sanity check"); | |
1681 | |
342 | 1682 int n_workers = g1h->workers()->total_workers(); |
1683 g1h->set_par_threads(n_workers); | |
1684 g1h->workers()->run_task(&g1_par_count_task); | |
1685 g1h->set_par_threads(0); | |
355 | 1686 |
1687 assert(g1h->check_heap_region_claim_values( | |
1688 HeapRegion::FinalCountClaimValue), | |
1689 "sanity check"); | |
342 | 1690 } else { |
1691 g1_par_count_task.work(0); | |
1692 } | |
1693 | |
1694 size_t known_garbage_bytes = | |
1695 g1_par_count_task.used_bytes() - g1_par_count_task.live_bytes(); | |
1696 #if 0 | |
1697 gclog_or_tty->print_cr("used %1.2lf, live %1.2lf, garbage %1.2lf", | |
1698 (double) g1_par_count_task.used_bytes() / (double) (1024 * 1024), | |
1699 (double) g1_par_count_task.live_bytes() / (double) (1024 * 1024), | |
1700 (double) known_garbage_bytes / (double) (1024 * 1024)); | |
1701 #endif // 0 | |
1702 g1p->set_known_garbage_bytes(known_garbage_bytes); | |
1703 | |
1704 size_t start_used_bytes = g1h->used(); | |
1705 _at_least_one_mark_complete = true; | |
1706 g1h->set_marking_complete(); | |
1707 | |
1708 double count_end = os::elapsedTime(); | |
1709 double this_final_counting_time = (count_end - start); | |
1710 if (G1PrintParCleanupStats) { | |
1711 gclog_or_tty->print_cr("Cleanup:"); | |
1712 gclog_or_tty->print_cr(" Finalize counting: %8.3f ms", | |
1713 this_final_counting_time*1000.0); | |
1714 } | |
1715 _total_counting_time += this_final_counting_time; | |
1716 | |
1717 // Install newly created mark bitMap as "prev". | |
1718 swapMarkBitMaps(); | |
1719 | |
1720 g1h->reset_gc_time_stamp(); | |
1721 | |
1722 // Note end of marking in all heap regions. | |
1723 double note_end_start = os::elapsedTime(); | |
2152 | 1724 G1ParNoteEndTask g1_par_note_end_task(g1h, &_cleanup_list); |
1833
8b10f48633dc
6984287: Regularize how GC parallel workers are specified.
jmasa
parents:
1719
diff
changeset
|
1725 if (G1CollectedHeap::use_parallel_gc_threads()) { |
342 | 1726 int n_workers = g1h->workers()->total_workers(); |
1727 g1h->set_par_threads(n_workers); | |
1728 g1h->workers()->run_task(&g1_par_note_end_task); | |
1729 g1h->set_par_threads(0); | |
355 | 1730 |
1731 assert(g1h->check_heap_region_claim_values(HeapRegion::NoteEndClaimValue), | |
1732 "sanity check"); | |
342 | 1733 } else { |
1734 g1_par_note_end_task.work(0); | |
1735 } | |
2152 | 1736 |
1737 if (!cleanup_list_is_empty()) { | |
1738 // The cleanup list is not empty, so we'll have to process it | |
1739 // concurrently. Notify anyone else that might be wanting free | |
1740 // regions that there will be more free regions coming soon. | |
1741 g1h->set_free_regions_coming(); | |
1742 } | |
342 | 1743 double note_end_end = os::elapsedTime(); |
1744 if (G1PrintParCleanupStats) { | |
1745 gclog_or_tty->print_cr(" note end of marking: %8.3f ms.", | |
1746 (note_end_end - note_end_start)*1000.0); | |
1747 } | |
1748 | |
355 | 1749 |
342 | 1750 // call below, since it affects the metric by which we sort the heap |
1751 // regions. | |
1752 if (G1ScrubRemSets) { | |
1753 double rs_scrub_start = os::elapsedTime(); | |
1754 G1ParScrubRemSetTask g1_par_scrub_rs_task(g1h, &_region_bm, &_card_bm); | |
1833
8b10f48633dc
6984287: Regularize how GC parallel workers are specified.
jmasa
parents:
1719
diff
changeset
|
1755 if (G1CollectedHeap::use_parallel_gc_threads()) { |
342 | 1756 int n_workers = g1h->workers()->total_workers(); |
1757 g1h->set_par_threads(n_workers); | |
1758 g1h->workers()->run_task(&g1_par_scrub_rs_task); | |
1759 g1h->set_par_threads(0); | |
355 | 1760 |
1761 assert(g1h->check_heap_region_claim_values( | |
1762 HeapRegion::ScrubRemSetClaimValue), | |
1763 "sanity check"); | |
342 | 1764 } else { |
1765 g1_par_scrub_rs_task.work(0); | |
1766 } | |
1767 | |
1768 double rs_scrub_end = os::elapsedTime(); | |
1769 double this_rs_scrub_time = (rs_scrub_end - rs_scrub_start); | |
1770 _total_rs_scrub_time += this_rs_scrub_time; | |
1771 } | |
1772 | |
1773 // this will also free any regions totally full of garbage objects, | |
1774 // and sort the regions. | |
1775 g1h->g1_policy()->record_concurrent_mark_cleanup_end( | |
1776 g1_par_note_end_task.freed_bytes(), | |
1777 g1_par_note_end_task.max_live_bytes()); | |
1778 | |
1779 // Statistics. | |
1780 double end = os::elapsedTime(); | |
1781 _cleanup_times.add((end - start) * 1000.0); | |
1782 | |
1783 // G1CollectedHeap::heap()->print(); | |
1784 // gclog_or_tty->print_cr("HEAP GC TIME STAMP : %d", | |
1785 // G1CollectedHeap::heap()->get_gc_time_stamp()); | |
1786 | |
1787 if (PrintGC || PrintGCDetails) { | |
1788 g1h->print_size_transition(gclog_or_tty, | |
1789 start_used_bytes, | |
1790 g1h->used(), | |
1791 g1h->capacity()); | |
1792 } | |
1793 | |
1794 size_t cleaned_up_bytes = start_used_bytes - g1h->used(); | |
1795 g1p->decrease_known_garbage_bytes(cleaned_up_bytes); | |
1796 | |
1797 // We need to make this be a "collection" so any collection pause that | |
1798 // races with it goes around and waits for completeCleanup to finish. | |
1799 g1h->increment_total_collections(); | |
1800 | |
751 | 1801 if (VerifyDuringGC) { |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1802 HandleMark hm; // handle scope |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1803 gclog_or_tty->print(" VerifyDuringGC:(after)"); |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1804 Universe::heap()->prepare_for_verify(); |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1805 Universe::verify(/* allow dirty */ true, |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1806 /* silent */ false, |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1807 /* prev marking */ true); |
342 | 1808 } |
2152 | 1809 |
1810 g1h->verify_region_sets_optional(); | |
342 | 1811 } |
1812 | |
1813 void ConcurrentMark::completeCleanup() { | |
1814 if (has_aborted()) return; | |
1815 | |
2152 | 1816 G1CollectedHeap* g1h = G1CollectedHeap::heap(); |
1817 | |
1818 _cleanup_list.verify_optional(); | |
1819 FreeRegionList local_free_list("Local Cleanup List"); | |
1820 | |
1821 if (G1ConcRegionFreeingVerbose) { | |
1822 gclog_or_tty->print_cr("G1ConcRegionFreeing [complete cleanup] : " | |
1823 "cleanup list has "SIZE_FORMAT" entries", | |
1824 _cleanup_list.length()); | |
1825 } | |
1826 | |
1827 // Noone else should be accessing the _cleanup_list at this point, | |
1828 // so it's not necessary to take any locks | |
1829 while (!_cleanup_list.is_empty()) { | |
1830 HeapRegion* hr = _cleanup_list.remove_head(); | |
1831 assert(hr != NULL, "the list was not empty"); | |
1832 hr->rem_set()->clear(); | |
1833 local_free_list.add_as_tail(hr); | |
1834 | |
1835 // Instead of adding one region at a time to the secondary_free_list, | |
1836 // we accumulate them in the local list and move them a few at a | |
1837 // time. This also cuts down on the number of notify_all() calls | |
1838 // we do during this process. We'll also append the local list when | |
1839 // _cleanup_list is empty (which means we just removed the last | |
1840 // region from the _cleanup_list). | |
1841 if ((local_free_list.length() % G1SecondaryFreeListAppendLength == 0) || | |
1842 _cleanup_list.is_empty()) { | |
1843 if (G1ConcRegionFreeingVerbose) { | |
1844 gclog_or_tty->print_cr("G1ConcRegionFreeing [complete cleanup] : " | |
1845 "appending "SIZE_FORMAT" entries to the " | |
1846 "secondary_free_list, clean list still has " | |
1847 SIZE_FORMAT" entries", | |
1848 local_free_list.length(), | |
1849 _cleanup_list.length()); | |
342 | 1850 } |
2152 | 1851 |
1852 { | |
1853 MutexLockerEx x(SecondaryFreeList_lock, Mutex::_no_safepoint_check_flag); | |
1854 g1h->secondary_free_list_add_as_tail(&local_free_list); | |
1855 SecondaryFreeList_lock->notify_all(); | |
1856 } | |
1857 | |
1858 if (G1StressConcRegionFreeing) { | |
1859 for (uintx i = 0; i < G1StressConcRegionFreeingDelayMillis; ++i) { | |
1860 os::sleep(Thread::current(), (jlong) 1, false); | |
1861 } | |
1862 } | |
342 | 1863 } |
1864 } | |
2152 | 1865 assert(local_free_list.is_empty(), "post-condition"); |
342 | 1866 } |
1867 | |
2037
b03260081e9b
7006113: G1: Initialize ReferenceProcessor::_is_alive_non_header field
johnc
parents:
1974
diff
changeset
|
1868 bool G1CMIsAliveClosure::do_object_b(oop obj) { |
b03260081e9b
7006113: G1: Initialize ReferenceProcessor::_is_alive_non_header field
johnc
parents:
1974
diff
changeset
|
1869 HeapWord* addr = (HeapWord*)obj; |
b03260081e9b
7006113: G1: Initialize ReferenceProcessor::_is_alive_non_header field
johnc
parents:
1974
diff
changeset
|
1870 return addr != NULL && |
b03260081e9b
7006113: G1: Initialize ReferenceProcessor::_is_alive_non_header field
johnc
parents:
1974
diff
changeset
|
1871 (!_g1->is_in_g1_reserved(addr) || !_g1->is_obj_ill(obj)); |
b03260081e9b
7006113: G1: Initialize ReferenceProcessor::_is_alive_non_header field
johnc
parents:
1974
diff
changeset
|
1872 } |
342 | 1873 |
1874 class G1CMKeepAliveClosure: public OopClosure { | |
1875 G1CollectedHeap* _g1; | |
1876 ConcurrentMark* _cm; | |
1877 CMBitMap* _bitMap; | |
1878 public: | |
1879 G1CMKeepAliveClosure(G1CollectedHeap* g1, ConcurrentMark* cm, | |
1880 CMBitMap* bitMap) : | |
1881 _g1(g1), _cm(cm), | |
1882 _bitMap(bitMap) {} | |
1883 | |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1884 virtual void do_oop(narrowOop* p) { do_oop_work(p); } |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1885 virtual void do_oop( oop* p) { do_oop_work(p); } |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1886 |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1887 template <class T> void do_oop_work(T* p) { |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
1888 oop thisOop = oopDesc::load_decode_heap_oop(p); |
342 | 1889 HeapWord* addr = (HeapWord*)thisOop; |
1890 if (_g1->is_in_g1_reserved(addr) && _g1->is_obj_ill(thisOop)) { | |
1891 _bitMap->mark(addr); | |
1892 _cm->mark_stack_push(thisOop); | |
1893 } | |
1894 } | |
1895 }; | |
1896 | |
1897 class G1CMDrainMarkingStackClosure: public VoidClosure { | |
1898 CMMarkStack* _markStack; | |
1899 CMBitMap* _bitMap; | |
1900 G1CMKeepAliveClosure* _oopClosure; | |
1901 public: | |
1902 G1CMDrainMarkingStackClosure(CMBitMap* bitMap, CMMarkStack* markStack, | |
1903 G1CMKeepAliveClosure* oopClosure) : | |
1904 _bitMap(bitMap), | |
1905 _markStack(markStack), | |
1906 _oopClosure(oopClosure) | |
1907 {} | |
1908 | |
1909 void do_void() { | |
1910 _markStack->drain((OopClosure*)_oopClosure, _bitMap, false); | |
1911 } | |
1912 }; | |
1913 | |
1914 void ConcurrentMark::weakRefsWork(bool clear_all_soft_refs) { | |
1915 ResourceMark rm; | |
1916 HandleMark hm; | |
453
c96030fff130
6684579: SoftReference processing can be made more efficient
ysr
parents:
355
diff
changeset
|
1917 G1CollectedHeap* g1h = G1CollectedHeap::heap(); |
c96030fff130
6684579: SoftReference processing can be made more efficient
ysr
parents:
355
diff
changeset
|
1918 ReferenceProcessor* rp = g1h->ref_processor(); |
342 | 1919 |
1974
fd1d227ef1b9
6983204: G1: Nightly test nsk/regression/b4958615 failing with +ExplicitGCInvokesConcurrent
johnc
parents:
1972
diff
changeset
|
1920 // See the comment in G1CollectedHeap::ref_processing_init() |
fd1d227ef1b9
6983204: G1: Nightly test nsk/regression/b4958615 failing with +ExplicitGCInvokesConcurrent
johnc
parents:
1972
diff
changeset
|
1921 // about how reference processing currently works in G1. |
fd1d227ef1b9
6983204: G1: Nightly test nsk/regression/b4958615 failing with +ExplicitGCInvokesConcurrent
johnc
parents:
1972
diff
changeset
|
1922 |
342 | 1923 // Process weak references. |
457
27a80744a83b
6778647: snap(), snap_policy() should be renamed setup(), setup_policy()
ysr
parents:
453
diff
changeset
|
1924 rp->setup_policy(clear_all_soft_refs); |
342 | 1925 assert(_markStack.isEmpty(), "mark stack should be empty"); |
1926 | |
2037
b03260081e9b
7006113: G1: Initialize ReferenceProcessor::_is_alive_non_header field
johnc
parents:
1974
diff
changeset
|
1927 G1CMIsAliveClosure g1_is_alive(g1h); |
b03260081e9b
7006113: G1: Initialize ReferenceProcessor::_is_alive_non_header field
johnc
parents:
1974
diff
changeset
|
1928 G1CMKeepAliveClosure g1_keep_alive(g1h, this, nextMarkBitMap()); |
342 | 1929 G1CMDrainMarkingStackClosure |
2037
b03260081e9b
7006113: G1: Initialize ReferenceProcessor::_is_alive_non_header field
johnc
parents:
1974
diff
changeset
|
1930 g1_drain_mark_stack(nextMarkBitMap(), &_markStack, &g1_keep_alive); |
342 | 1931 |
1932 // XXXYYY Also: copy the parallel ref processing code from CMS. | |
2037
b03260081e9b
7006113: G1: Initialize ReferenceProcessor::_is_alive_non_header field
johnc
parents:
1974
diff
changeset
|
1933 rp->process_discovered_references(&g1_is_alive, |
b03260081e9b
7006113: G1: Initialize ReferenceProcessor::_is_alive_non_header field
johnc
parents:
1974
diff
changeset
|
1934 &g1_keep_alive, |
b03260081e9b
7006113: G1: Initialize ReferenceProcessor::_is_alive_non_header field
johnc
parents:
1974
diff
changeset
|
1935 &g1_drain_mark_stack, |
342 | 1936 NULL); |
1937 assert(_markStack.overflow() || _markStack.isEmpty(), | |
1938 "mark stack should be empty (unless it overflowed)"); | |
1939 if (_markStack.overflow()) { | |
1940 set_has_overflown(); | |
1941 } | |
1942 | |
1943 rp->enqueue_discovered_references(); | |
1944 rp->verify_no_references_recorded(); | |
1945 assert(!rp->discovery_enabled(), "should have been disabled"); | |
1946 | |
1947 // Now clean up stale oops in SymbolTable and StringTable | |
2037
b03260081e9b
7006113: G1: Initialize ReferenceProcessor::_is_alive_non_header field
johnc
parents:
1974
diff
changeset
|
1948 SymbolTable::unlink(&g1_is_alive); |
b03260081e9b
7006113: G1: Initialize ReferenceProcessor::_is_alive_non_header field
johnc
parents:
1974
diff
changeset
|
1949 StringTable::unlink(&g1_is_alive); |
342 | 1950 } |
1951 | |
1952 void ConcurrentMark::swapMarkBitMaps() { | |
1953 CMBitMapRO* temp = _prevMarkBitMap; | |
1954 _prevMarkBitMap = (CMBitMapRO*)_nextMarkBitMap; | |
1955 _nextMarkBitMap = (CMBitMap*) temp; | |
1956 } | |
1957 | |
1958 class CMRemarkTask: public AbstractGangTask { | |
1959 private: | |
1960 ConcurrentMark *_cm; | |
1961 | |
1962 public: | |
1963 void work(int worker_i) { | |
1964 // Since all available tasks are actually started, we should | |
1965 // only proceed if we're supposed to be actived. | |
1966 if ((size_t)worker_i < _cm->active_tasks()) { | |
1967 CMTask* task = _cm->task(worker_i); | |
1968 task->record_start_time(); | |
1969 do { | |
1970 task->do_marking_step(1000000000.0 /* something very large */); | |
1971 } while (task->has_aborted() && !_cm->has_overflown()); | |
1972 // If we overflow, then we do not want to restart. We instead | |
1973 // want to abort remark and do concurrent marking again. | |
1974 task->record_end_time(); | |
1975 } | |
1976 } | |
1977 | |
1978 CMRemarkTask(ConcurrentMark* cm) : | |
1979 AbstractGangTask("Par Remark"), _cm(cm) { } | |
1980 }; | |
1981 | |
1982 void ConcurrentMark::checkpointRootsFinalWork() { | |
1983 ResourceMark rm; | |
1984 HandleMark hm; | |
1985 G1CollectedHeap* g1h = G1CollectedHeap::heap(); | |
1986 | |
1987 g1h->ensure_parsability(false); | |
1988 | |
1833
8b10f48633dc
6984287: Regularize how GC parallel workers are specified.
jmasa
parents:
1719
diff
changeset
|
1989 if (G1CollectedHeap::use_parallel_gc_threads()) { |
989
148e5441d916
6863023: need non-perm oops in code cache for JSR 292
jrose
parents:
912
diff
changeset
|
1990 G1CollectedHeap::StrongRootsScope srs(g1h); |
342 | 1991 // this is remark, so we'll use up all available threads |
1992 int active_workers = ParallelGCThreads; | |
1993 set_phase(active_workers, false); | |
1994 | |
1995 CMRemarkTask remarkTask(this); | |
1996 // We will start all available threads, even if we decide that the | |
1997 // active_workers will be fewer. The extra ones will just bail out | |
1998 // immediately. | |
1999 int n_workers = g1h->workers()->total_workers(); | |
2000 g1h->set_par_threads(n_workers); | |
2001 g1h->workers()->run_task(&remarkTask); | |
2002 g1h->set_par_threads(0); | |
2003 } else { | |
989
148e5441d916
6863023: need non-perm oops in code cache for JSR 292
jrose
parents:
912
diff
changeset
|
2004 G1CollectedHeap::StrongRootsScope srs(g1h); |
342 | 2005 // this is remark, so we'll use up all available threads |
2006 int active_workers = 1; | |
2007 set_phase(active_workers, false); | |
2008 | |
2009 CMRemarkTask remarkTask(this); | |
2010 // We will start all available threads, even if we decide that the | |
2011 // active_workers will be fewer. The extra ones will just bail out | |
2012 // immediately. | |
2013 remarkTask.work(0); | |
2014 } | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
2015 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
2016 guarantee(satb_mq_set.completed_buffers_num() == 0, "invariant"); |
342 | 2017 |
2018 print_stats(); | |
2019 | |
2020 if (!restart_for_overflow()) | |
2021 set_non_marking_state(); | |
2022 | |
2023 #if VERIFY_OBJS_PROCESSED | |
2024 if (_scan_obj_cl.objs_processed != ThreadLocalObjQueue::objs_enqueued) { | |
2025 gclog_or_tty->print_cr("Processed = %d, enqueued = %d.", | |
2026 _scan_obj_cl.objs_processed, | |
2027 ThreadLocalObjQueue::objs_enqueued); | |
2028 guarantee(_scan_obj_cl.objs_processed == | |
2029 ThreadLocalObjQueue::objs_enqueued, | |
2030 "Different number of objs processed and enqueued."); | |
2031 } | |
2032 #endif | |
2033 } | |
2034 | |
1044 | 2035 #ifndef PRODUCT |
2036 | |
1388 | 2037 class PrintReachableOopClosure: public OopClosure { |
342 | 2038 private: |
2039 G1CollectedHeap* _g1h; | |
2040 CMBitMapRO* _bitmap; | |
2041 outputStream* _out; | |
1044 | 2042 bool _use_prev_marking; |
1388 | 2043 bool _all; |
342 | 2044 |
2045 public: | |
1388 | 2046 PrintReachableOopClosure(CMBitMapRO* bitmap, |
2047 outputStream* out, | |
2048 bool use_prev_marking, | |
2049 bool all) : | |
1044 | 2050 _g1h(G1CollectedHeap::heap()), |
1388 | 2051 _bitmap(bitmap), _out(out), _use_prev_marking(use_prev_marking), _all(all) { } |
342 | 2052 |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
2053 void do_oop(narrowOop* p) { do_oop_work(p); } |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
2054 void do_oop( oop* p) { do_oop_work(p); } |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
2055 |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
2056 template <class T> void do_oop_work(T* p) { |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
2057 oop obj = oopDesc::load_decode_heap_oop(p); |
342 | 2058 const char* str = NULL; |
2059 const char* str2 = ""; | |
2060 | |
1388 | 2061 if (obj == NULL) { |
2062 str = ""; | |
2063 } else if (!_g1h->is_in_g1_reserved(obj)) { | |
2064 str = " O"; | |
2065 } else { | |
342 | 2066 HeapRegion* hr = _g1h->heap_region_containing(obj); |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
2067 guarantee(hr != NULL, "invariant"); |
1044 | 2068 bool over_tams = false; |
2069 if (_use_prev_marking) { | |
2070 over_tams = hr->obj_allocated_since_prev_marking(obj); | |
2071 } else { | |
2072 over_tams = hr->obj_allocated_since_next_marking(obj); | |
2073 } | |
1388 | 2074 bool marked = _bitmap->isMarked((HeapWord*) obj); |
1044 | 2075 |
2076 if (over_tams) { | |
1388 | 2077 str = " >"; |
2078 if (marked) { | |
342 | 2079 str2 = " AND MARKED"; |
1044 | 2080 } |
1388 | 2081 } else if (marked) { |
2082 str = " M"; | |
1044 | 2083 } else { |
1388 | 2084 str = " NOT"; |
1044 | 2085 } |
342 | 2086 } |
2087 | |
1388 | 2088 _out->print_cr(" "PTR_FORMAT": "PTR_FORMAT"%s%s", |
342 | 2089 p, (void*) obj, str, str2); |
2090 } | |
2091 }; | |
2092 | |
1388 | 2093 class PrintReachableObjectClosure : public ObjectClosure { |
342 | 2094 private: |
1044 | 2095 CMBitMapRO* _bitmap; |
342 | 2096 outputStream* _out; |
1044 | 2097 bool _use_prev_marking; |
1388 | 2098 bool _all; |
2099 HeapRegion* _hr; | |
342 | 2100 |
2101 public: | |
1388 | 2102 PrintReachableObjectClosure(CMBitMapRO* bitmap, |
2103 outputStream* out, | |
2104 bool use_prev_marking, | |
2105 bool all, | |
2106 HeapRegion* hr) : | |
2107 _bitmap(bitmap), _out(out), | |
2108 _use_prev_marking(use_prev_marking), _all(all), _hr(hr) { } | |
2109 | |
2110 void do_object(oop o) { | |
2111 bool over_tams; | |
2112 if (_use_prev_marking) { | |
2113 over_tams = _hr->obj_allocated_since_prev_marking(o); | |
2114 } else { | |
2115 over_tams = _hr->obj_allocated_since_next_marking(o); | |
2116 } | |
2117 bool marked = _bitmap->isMarked((HeapWord*) o); | |
2118 bool print_it = _all || over_tams || marked; | |
2119 | |
2120 if (print_it) { | |
2121 _out->print_cr(" "PTR_FORMAT"%s", | |
2122 o, (over_tams) ? " >" : (marked) ? " M" : ""); | |
2123 PrintReachableOopClosure oopCl(_bitmap, _out, _use_prev_marking, _all); | |
2124 o->oop_iterate(&oopCl); | |
2125 } | |
342 | 2126 } |
2127 }; | |
2128 | |
1388 | 2129 class PrintReachableRegionClosure : public HeapRegionClosure { |
342 | 2130 private: |
1044 | 2131 CMBitMapRO* _bitmap; |
342 | 2132 outputStream* _out; |
1044 | 2133 bool _use_prev_marking; |
1388 | 2134 bool _all; |
342 | 2135 |
2136 public: | |
2137 bool doHeapRegion(HeapRegion* hr) { | |
2138 HeapWord* b = hr->bottom(); | |
2139 HeapWord* e = hr->end(); | |
2140 HeapWord* t = hr->top(); | |
1044 | 2141 HeapWord* p = NULL; |
2142 if (_use_prev_marking) { | |
2143 p = hr->prev_top_at_mark_start(); | |
2144 } else { | |
2145 p = hr->next_top_at_mark_start(); | |
2146 } | |
342 | 2147 _out->print_cr("** ["PTR_FORMAT", "PTR_FORMAT"] top: "PTR_FORMAT" " |
1044 | 2148 "TAMS: "PTR_FORMAT, b, e, t, p); |
1388 | 2149 _out->cr(); |
2150 | |
2151 HeapWord* from = b; | |
2152 HeapWord* to = t; | |
2153 | |
2154 if (to > from) { | |
2155 _out->print_cr("Objects in ["PTR_FORMAT", "PTR_FORMAT"]", from, to); | |
2156 _out->cr(); | |
2157 PrintReachableObjectClosure ocl(_bitmap, _out, | |
2158 _use_prev_marking, _all, hr); | |
2159 hr->object_iterate_mem_careful(MemRegion(from, to), &ocl); | |
2160 _out->cr(); | |
2161 } | |
342 | 2162 |
2163 return false; | |
2164 } | |
2165 | |
1388 | 2166 PrintReachableRegionClosure(CMBitMapRO* bitmap, |
2167 outputStream* out, | |
2168 bool use_prev_marking, | |
2169 bool all) : | |
2170 _bitmap(bitmap), _out(out), _use_prev_marking(use_prev_marking), _all(all) { } | |
342 | 2171 }; |
2172 | |
1388 | 2173 void ConcurrentMark::print_reachable(const char* str, |
2174 bool use_prev_marking, | |
2175 bool all) { | |
2176 gclog_or_tty->cr(); | |
2177 gclog_or_tty->print_cr("== Doing heap dump... "); | |
1044 | 2178 |
2179 if (G1PrintReachableBaseFile == NULL) { | |
2180 gclog_or_tty->print_cr(" #### error: no base file defined"); | |
2181 return; | |
2182 } | |
2183 | |
2184 if (strlen(G1PrintReachableBaseFile) + 1 + strlen(str) > | |
2185 (JVM_MAXPATHLEN - 1)) { | |
2186 gclog_or_tty->print_cr(" #### error: file name too long"); | |
2187 return; | |
2188 } | |
2189 | |
2190 char file_name[JVM_MAXPATHLEN]; | |
2191 sprintf(file_name, "%s.%s", G1PrintReachableBaseFile, str); | |
2192 gclog_or_tty->print_cr(" dumping to file %s", file_name); | |
2193 | |
2194 fileStream fout(file_name); | |
2195 if (!fout.is_open()) { | |
2196 gclog_or_tty->print_cr(" #### error: could not open file"); | |
2197 return; | |
2198 } | |
2199 | |
2200 outputStream* out = &fout; | |
2201 | |
2202 CMBitMapRO* bitmap = NULL; | |
2203 if (use_prev_marking) { | |
2204 bitmap = _prevMarkBitMap; | |
2205 } else { | |
2206 bitmap = _nextMarkBitMap; | |
2207 } | |
2208 | |
2209 out->print_cr("-- USING %s", (use_prev_marking) ? "PTAMS" : "NTAMS"); | |
2210 out->cr(); | |
2211 | |
1388 | 2212 out->print_cr("--- ITERATING OVER REGIONS"); |
1044 | 2213 out->cr(); |
1388 | 2214 PrintReachableRegionClosure rcl(bitmap, out, use_prev_marking, all); |
2215 _g1h->heap_region_iterate(&rcl); | |
1044 | 2216 out->cr(); |
2217 | |
2218 gclog_or_tty->print_cr(" done"); | |
1388 | 2219 gclog_or_tty->flush(); |
342 | 2220 } |
2221 | |
1044 | 2222 #endif // PRODUCT |
2223 | |
342 | 2224 // This note is for drainAllSATBBuffers and the code in between. |
2225 // In the future we could reuse a task to do this work during an | |
2226 // evacuation pause (since now tasks are not active and can be claimed | |
2227 // during an evacuation pause). This was a late change to the code and | |
2228 // is currently not being taken advantage of. | |
2229 | |
2230 class CMGlobalObjectClosure : public ObjectClosure { | |
2231 private: | |
2232 ConcurrentMark* _cm; | |
2233 | |
2234 public: | |
2235 void do_object(oop obj) { | |
2236 _cm->deal_with_reference(obj); | |
2237 } | |
2238 | |
2239 CMGlobalObjectClosure(ConcurrentMark* cm) : _cm(cm) { } | |
2240 }; | |
2241 | |
2242 void ConcurrentMark::deal_with_reference(oop obj) { | |
2243 if (verbose_high()) | |
2244 gclog_or_tty->print_cr("[global] we're dealing with reference "PTR_FORMAT, | |
2245 (void*) obj); | |
2246 | |
2247 | |
2248 HeapWord* objAddr = (HeapWord*) obj; | |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
2249 assert(obj->is_oop_or_null(true /* ignore mark word */), "Error"); |
342 | 2250 if (_g1h->is_in_g1_reserved(objAddr)) { |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
2251 assert(obj != NULL, "is_in_g1_reserved should ensure this"); |
342 | 2252 HeapRegion* hr = _g1h->heap_region_containing(obj); |
2253 if (_g1h->is_obj_ill(obj, hr)) { | |
2254 if (verbose_high()) | |
2255 gclog_or_tty->print_cr("[global] "PTR_FORMAT" is not considered " | |
2256 "marked", (void*) obj); | |
2257 | |
2258 // we need to mark it first | |
2259 if (_nextMarkBitMap->parMark(objAddr)) { | |
2260 // No OrderAccess:store_load() is needed. It is implicit in the | |
2261 // CAS done in parMark(objAddr) above | |
2262 HeapWord* finger = _finger; | |
2263 if (objAddr < finger) { | |
2264 if (verbose_high()) | |
2265 gclog_or_tty->print_cr("[global] below the global finger " | |
2266 "("PTR_FORMAT"), pushing it", finger); | |
2267 if (!mark_stack_push(obj)) { | |
2268 if (verbose_low()) | |
2269 gclog_or_tty->print_cr("[global] global stack overflow during " | |
2270 "deal_with_reference"); | |
2271 } | |
2272 } | |
2273 } | |
2274 } | |
2275 } | |
2276 } | |
2277 | |
2278 void ConcurrentMark::drainAllSATBBuffers() { | |
2279 CMGlobalObjectClosure oc(this); | |
2280 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); | |
2281 satb_mq_set.set_closure(&oc); | |
2282 | |
2283 while (satb_mq_set.apply_closure_to_completed_buffer()) { | |
2284 if (verbose_medium()) | |
2285 gclog_or_tty->print_cr("[global] processed an SATB buffer"); | |
2286 } | |
2287 | |
2288 // no need to check whether we should do this, as this is only | |
2289 // called during an evacuation pause | |
2290 satb_mq_set.iterate_closure_all_threads(); | |
2291 | |
2292 satb_mq_set.set_closure(NULL); | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
2293 assert(satb_mq_set.completed_buffers_num() == 0, "invariant"); |
342 | 2294 } |
2295 | |
2296 void ConcurrentMark::markPrev(oop p) { | |
2297 // Note we are overriding the read-only view of the prev map here, via | |
2298 // the cast. | |
2299 ((CMBitMap*)_prevMarkBitMap)->mark((HeapWord*)p); | |
2300 } | |
2301 | |
2302 void ConcurrentMark::clear(oop p) { | |
2303 assert(p != NULL && p->is_oop(), "expected an oop"); | |
2304 HeapWord* addr = (HeapWord*)p; | |
2305 assert(addr >= _nextMarkBitMap->startWord() || | |
2306 addr < _nextMarkBitMap->endWord(), "in a region"); | |
2307 | |
2308 _nextMarkBitMap->clear(addr); | |
2309 } | |
2310 | |
2311 void ConcurrentMark::clearRangeBothMaps(MemRegion mr) { | |
2312 // Note we are overriding the read-only view of the prev map here, via | |
2313 // the cast. | |
2314 ((CMBitMap*)_prevMarkBitMap)->clearRange(mr); | |
2315 _nextMarkBitMap->clearRange(mr); | |
2316 } | |
2317 | |
2318 HeapRegion* | |
2319 ConcurrentMark::claim_region(int task_num) { | |
2320 // "checkpoint" the finger | |
2321 HeapWord* finger = _finger; | |
2322 | |
2323 // _heap_end will not change underneath our feet; it only changes at | |
2324 // yield points. | |
2325 while (finger < _heap_end) { | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
2326 assert(_g1h->is_in_g1_reserved(finger), "invariant"); |
342 | 2327 |
2328 // is the gap between reading the finger and doing the CAS too long? | |
2329 | |
2330 HeapRegion* curr_region = _g1h->heap_region_containing(finger); | |
2331 HeapWord* bottom = curr_region->bottom(); | |
2332 HeapWord* end = curr_region->end(); | |
2333 HeapWord* limit = curr_region->next_top_at_mark_start(); | |
2334 | |
2335 if (verbose_low()) | |
2336 gclog_or_tty->print_cr("[%d] curr_region = "PTR_FORMAT" " | |
2337 "["PTR_FORMAT", "PTR_FORMAT"), " | |
2338 "limit = "PTR_FORMAT, | |
2339 task_num, curr_region, bottom, end, limit); | |
2340 | |
2341 HeapWord* res = | |
2342 (HeapWord*) Atomic::cmpxchg_ptr(end, &_finger, finger); | |
2343 if (res == finger) { | |
2344 // we succeeded | |
2345 | |
2346 // notice that _finger == end cannot be guaranteed here since, | |
2347 // someone else might have moved the finger even further | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
2348 assert(_finger >= end, "the finger should have moved forward"); |
342 | 2349 |
2350 if (verbose_low()) | |
2351 gclog_or_tty->print_cr("[%d] we were successful with region = " | |
2352 PTR_FORMAT, task_num, curr_region); | |
2353 | |
2354 if (limit > bottom) { | |
2355 if (verbose_low()) | |
2356 gclog_or_tty->print_cr("[%d] region "PTR_FORMAT" is not empty, " | |
2357 "returning it ", task_num, curr_region); | |
2358 return curr_region; | |
2359 } else { | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
2360 assert(limit == bottom, |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
2361 "the region limit should be at bottom"); |
342 | 2362 if (verbose_low()) |
2363 gclog_or_tty->print_cr("[%d] region "PTR_FORMAT" is empty, " | |
2364 "returning NULL", task_num, curr_region); | |
2365 // we return NULL and the caller should try calling | |
2366 // claim_region() again. | |
2367 return NULL; | |
2368 } | |
2369 } else { | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
2370 assert(_finger > finger, "the finger should have moved forward"); |
342 | 2371 if (verbose_low()) |
2372 gclog_or_tty->print_cr("[%d] somebody else moved the finger, " | |
2373 "global finger = "PTR_FORMAT", " | |
2374 "our finger = "PTR_FORMAT, | |
2375 task_num, _finger, finger); | |
2376 | |
2377 // read it again | |
2378 finger = _finger; | |
2379 } | |
2380 } | |
2381 | |
2382 return NULL; | |
2383 } | |
2384 | |
1835
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2385 bool ConcurrentMark::invalidate_aborted_regions_in_cset() { |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2386 bool result = false; |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2387 for (int i = 0; i < (int)_max_task_num; ++i) { |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2388 CMTask* the_task = _tasks[i]; |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2389 MemRegion mr = the_task->aborted_region(); |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2390 if (mr.start() != NULL) { |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2391 assert(mr.end() != NULL, "invariant"); |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2392 assert(mr.word_size() > 0, "invariant"); |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2393 HeapRegion* hr = _g1h->heap_region_containing(mr.start()); |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2394 assert(hr != NULL, "invariant"); |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2395 if (hr->in_collection_set()) { |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2396 // The region points into the collection set |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2397 the_task->set_aborted_region(MemRegion()); |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2398 result = true; |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2399 } |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2400 } |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2401 } |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2402 return result; |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2403 } |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2404 |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2405 bool ConcurrentMark::has_aborted_regions() { |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2406 for (int i = 0; i < (int)_max_task_num; ++i) { |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2407 CMTask* the_task = _tasks[i]; |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2408 MemRegion mr = the_task->aborted_region(); |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2409 if (mr.start() != NULL) { |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2410 assert(mr.end() != NULL, "invariant"); |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2411 assert(mr.word_size() > 0, "invariant"); |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2412 return true; |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2413 } |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2414 } |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2415 return false; |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2416 } |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2417 |
342 | 2418 void ConcurrentMark::oops_do(OopClosure* cl) { |
2419 if (_markStack.size() > 0 && verbose_low()) | |
2420 gclog_or_tty->print_cr("[global] scanning the global marking stack, " | |
2421 "size = %d", _markStack.size()); | |
2422 // we first iterate over the contents of the mark stack... | |
2423 _markStack.oops_do(cl); | |
2424 | |
2425 for (int i = 0; i < (int)_max_task_num; ++i) { | |
2426 OopTaskQueue* queue = _task_queues->queue((int)i); | |
2427 | |
2428 if (queue->size() > 0 && verbose_low()) | |
2429 gclog_or_tty->print_cr("[global] scanning task queue of task %d, " | |
2430 "size = %d", i, queue->size()); | |
2431 | |
2432 // ...then over the contents of the all the task queues. | |
2433 queue->oops_do(cl); | |
2434 } | |
2435 | |
1835
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2436 // Invalidate any entries, that are in the region stack, that |
342 | 2437 // point into the collection set |
2438 if (_regionStack.invalidate_entries_into_cset()) { | |
2439 // otherwise, any gray objects copied during the evacuation pause | |
2440 // might not be visited. | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
2441 assert(_should_gray_objects, "invariant"); |
342 | 2442 } |
1835
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2443 |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2444 // Invalidate any aborted regions, recorded in the individual CM |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2445 // tasks, that point into the collection set. |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2446 if (invalidate_aborted_regions_in_cset()) { |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2447 // otherwise, any gray objects copied during the evacuation pause |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2448 // might not be visited. |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2449 assert(_should_gray_objects, "invariant"); |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2450 } |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2451 |
342 | 2452 } |
2453 | |
2454 void ConcurrentMark::clear_marking_state() { | |
2455 _markStack.setEmpty(); | |
2456 _markStack.clear_overflow(); | |
2457 _regionStack.setEmpty(); | |
2458 _regionStack.clear_overflow(); | |
2459 clear_has_overflown(); | |
2460 _finger = _heap_start; | |
2461 | |
2462 for (int i = 0; i < (int)_max_task_num; ++i) { | |
2463 OopTaskQueue* queue = _task_queues->queue(i); | |
2464 queue->set_empty(); | |
1885
a5c514e74487
6988458: G1: assert(mr.end() <= _cm->finger()) failed: otherwise the region shouldn't be on the stack
johnc
parents:
1835
diff
changeset
|
2465 // Clear any partial regions from the CMTasks |
a5c514e74487
6988458: G1: assert(mr.end() <= _cm->finger()) failed: otherwise the region shouldn't be on the stack
johnc
parents:
1835
diff
changeset
|
2466 _tasks[i]->clear_aborted_region(); |
342 | 2467 } |
2468 } | |
2469 | |
2470 void ConcurrentMark::print_stats() { | |
2471 if (verbose_stats()) { | |
2472 gclog_or_tty->print_cr("---------------------------------------------------------------------"); | |
2473 for (size_t i = 0; i < _active_tasks; ++i) { | |
2474 _tasks[i]->print_stats(); | |
2475 gclog_or_tty->print_cr("---------------------------------------------------------------------"); | |
2476 } | |
2477 } | |
2478 } | |
2479 | |
2480 class CSMarkOopClosure: public OopClosure { | |
2481 friend class CSMarkBitMapClosure; | |
2482 | |
2483 G1CollectedHeap* _g1h; | |
2484 CMBitMap* _bm; | |
2485 ConcurrentMark* _cm; | |
2486 oop* _ms; | |
2487 jint* _array_ind_stack; | |
2488 int _ms_size; | |
2489 int _ms_ind; | |
2490 int _array_increment; | |
2491 | |
2492 bool push(oop obj, int arr_ind = 0) { | |
2493 if (_ms_ind == _ms_size) { | |
2494 gclog_or_tty->print_cr("Mark stack is full."); | |
2495 return false; | |
2496 } | |
2497 _ms[_ms_ind] = obj; | |
2498 if (obj->is_objArray()) _array_ind_stack[_ms_ind] = arr_ind; | |
2499 _ms_ind++; | |
2500 return true; | |
2501 } | |
2502 | |
2503 oop pop() { | |
2504 if (_ms_ind == 0) return NULL; | |
2505 else { | |
2506 _ms_ind--; | |
2507 return _ms[_ms_ind]; | |
2508 } | |
2509 } | |
2510 | |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
2511 template <class T> bool drain() { |
342 | 2512 while (_ms_ind > 0) { |
2513 oop obj = pop(); | |
2514 assert(obj != NULL, "Since index was non-zero."); | |
2515 if (obj->is_objArray()) { | |
2516 jint arr_ind = _array_ind_stack[_ms_ind]; | |
2517 objArrayOop aobj = objArrayOop(obj); | |
2518 jint len = aobj->length(); | |
2519 jint next_arr_ind = arr_ind + _array_increment; | |
2520 if (next_arr_ind < len) { | |
2521 push(obj, next_arr_ind); | |
2522 } | |
2523 // Now process this portion of this one. | |
2524 int lim = MIN2(next_arr_ind, len); | |
2525 for (int j = arr_ind; j < lim; j++) { | |
912
308762b2bf14
6872000: G1: compilation fails on linux/older gcc
apetrusenko
parents:
866
diff
changeset
|
2526 do_oop(aobj->objArrayOopDesc::obj_at_addr<T>(j)); |
342 | 2527 } |
2528 | |
2529 } else { | |
2530 obj->oop_iterate(this); | |
2531 } | |
2532 if (abort()) return false; | |
2533 } | |
2534 return true; | |
2535 } | |
2536 | |
2537 public: | |
2538 CSMarkOopClosure(ConcurrentMark* cm, int ms_size) : | |
2539 _g1h(G1CollectedHeap::heap()), | |
2540 _cm(cm), | |
2541 _bm(cm->nextMarkBitMap()), | |
2542 _ms_size(ms_size), _ms_ind(0), | |
2543 _ms(NEW_C_HEAP_ARRAY(oop, ms_size)), | |
2544 _array_ind_stack(NEW_C_HEAP_ARRAY(jint, ms_size)), | |
2545 _array_increment(MAX2(ms_size/8, 16)) | |
2546 {} | |
2547 | |
2548 ~CSMarkOopClosure() { | |
2549 FREE_C_HEAP_ARRAY(oop, _ms); | |
2550 FREE_C_HEAP_ARRAY(jint, _array_ind_stack); | |
2551 } | |
2552 | |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
2553 virtual void do_oop(narrowOop* p) { do_oop_work(p); } |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
2554 virtual void do_oop( oop* p) { do_oop_work(p); } |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
2555 |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
2556 template <class T> void do_oop_work(T* p) { |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
2557 T heap_oop = oopDesc::load_heap_oop(p); |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
2558 if (oopDesc::is_null(heap_oop)) return; |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
2559 oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); |
342 | 2560 if (obj->is_forwarded()) { |
2561 // If the object has already been forwarded, we have to make sure | |
2562 // that it's marked. So follow the forwarding pointer. Note that | |
2563 // this does the right thing for self-forwarding pointers in the | |
2564 // evacuation failure case. | |
2565 obj = obj->forwardee(); | |
2566 } | |
2567 HeapRegion* hr = _g1h->heap_region_containing(obj); | |
2568 if (hr != NULL) { | |
2569 if (hr->in_collection_set()) { | |
2570 if (_g1h->is_obj_ill(obj)) { | |
2571 _bm->mark((HeapWord*)obj); | |
2572 if (!push(obj)) { | |
2573 gclog_or_tty->print_cr("Setting abort in CSMarkOopClosure because push failed."); | |
2574 set_abort(); | |
2575 } | |
2576 } | |
2577 } else { | |
2578 // Outside the collection set; we need to gray it | |
2579 _cm->deal_with_reference(obj); | |
2580 } | |
2581 } | |
2582 } | |
2583 }; | |
2584 | |
2585 class CSMarkBitMapClosure: public BitMapClosure { | |
2586 G1CollectedHeap* _g1h; | |
2587 CMBitMap* _bitMap; | |
2588 ConcurrentMark* _cm; | |
2589 CSMarkOopClosure _oop_cl; | |
2590 public: | |
2591 CSMarkBitMapClosure(ConcurrentMark* cm, int ms_size) : | |
2592 _g1h(G1CollectedHeap::heap()), | |
2593 _bitMap(cm->nextMarkBitMap()), | |
2594 _oop_cl(cm, ms_size) | |
2595 {} | |
2596 | |
2597 ~CSMarkBitMapClosure() {} | |
2598 | |
2599 bool do_bit(size_t offset) { | |
2600 // convert offset into a HeapWord* | |
2601 HeapWord* addr = _bitMap->offsetToHeapWord(offset); | |
2602 assert(_bitMap->endWord() && addr < _bitMap->endWord(), | |
2603 "address out of range"); | |
2604 assert(_bitMap->isMarked(addr), "tautology"); | |
2605 oop obj = oop(addr); | |
2606 if (!obj->is_forwarded()) { | |
2607 if (!_oop_cl.push(obj)) return false; | |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
2608 if (UseCompressedOops) { |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
2609 if (!_oop_cl.drain<narrowOop>()) return false; |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
2610 } else { |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
2611 if (!_oop_cl.drain<oop>()) return false; |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
2612 } |
342 | 2613 } |
2614 // Otherwise... | |
2615 return true; | |
2616 } | |
2617 }; | |
2618 | |
2619 | |
2620 class CompleteMarkingInCSHRClosure: public HeapRegionClosure { | |
2621 CMBitMap* _bm; | |
2622 CSMarkBitMapClosure _bit_cl; | |
2623 enum SomePrivateConstants { | |
2624 MSSize = 1000 | |
2625 }; | |
2626 bool _completed; | |
2627 public: | |
2628 CompleteMarkingInCSHRClosure(ConcurrentMark* cm) : | |
2629 _bm(cm->nextMarkBitMap()), | |
2630 _bit_cl(cm, MSSize), | |
2631 _completed(true) | |
2632 {} | |
2633 | |
2634 ~CompleteMarkingInCSHRClosure() {} | |
2635 | |
2636 bool doHeapRegion(HeapRegion* r) { | |
2637 if (!r->evacuation_failed()) { | |
2638 MemRegion mr = MemRegion(r->bottom(), r->next_top_at_mark_start()); | |
2639 if (!mr.is_empty()) { | |
2640 if (!_bm->iterate(&_bit_cl, mr)) { | |
2641 _completed = false; | |
2642 return true; | |
2643 } | |
2644 } | |
2645 } | |
2646 return false; | |
2647 } | |
2648 | |
2649 bool completed() { return _completed; } | |
2650 }; | |
2651 | |
2652 class ClearMarksInHRClosure: public HeapRegionClosure { | |
2653 CMBitMap* _bm; | |
2654 public: | |
2655 ClearMarksInHRClosure(CMBitMap* bm): _bm(bm) { } | |
2656 | |
2657 bool doHeapRegion(HeapRegion* r) { | |
2658 if (!r->used_region().is_empty() && !r->evacuation_failed()) { | |
2659 MemRegion usedMR = r->used_region(); | |
2660 _bm->clearRange(r->used_region()); | |
2661 } | |
2662 return false; | |
2663 } | |
2664 }; | |
2665 | |
2666 void ConcurrentMark::complete_marking_in_collection_set() { | |
2667 G1CollectedHeap* g1h = G1CollectedHeap::heap(); | |
2668 | |
2669 if (!g1h->mark_in_progress()) { | |
2670 g1h->g1_policy()->record_mark_closure_time(0.0); | |
2671 return; | |
2672 } | |
2673 | |
2674 int i = 1; | |
2675 double start = os::elapsedTime(); | |
2676 while (true) { | |
2677 i++; | |
2678 CompleteMarkingInCSHRClosure cmplt(this); | |
2679 g1h->collection_set_iterate(&cmplt); | |
2680 if (cmplt.completed()) break; | |
2681 } | |
2682 double end_time = os::elapsedTime(); | |
2683 double elapsed_time_ms = (end_time - start) * 1000.0; | |
2684 g1h->g1_policy()->record_mark_closure_time(elapsed_time_ms); | |
2685 | |
2686 ClearMarksInHRClosure clr(nextMarkBitMap()); | |
2687 g1h->collection_set_iterate(&clr); | |
2688 } | |
2689 | |
2690 // The next two methods deal with the following optimisation. Some | |
2691 // objects are gray by being marked and located above the finger. If | |
2692 // they are copied, during an evacuation pause, below the finger then | |
2693 // the need to be pushed on the stack. The observation is that, if | |
2694 // there are no regions in the collection set located above the | |
2695 // finger, then the above cannot happen, hence we do not need to | |
2696 // explicitly gray any objects when copying them to below the | |
2697 // finger. The global stack will be scanned to ensure that, if it | |
2698 // points to objects being copied, it will update their | |
2699 // location. There is a tricky situation with the gray objects in | |
2700 // region stack that are being coped, however. See the comment in | |
2701 // newCSet(). | |
2702 | |
2703 void ConcurrentMark::newCSet() { | |
2704 if (!concurrent_marking_in_progress()) | |
2705 // nothing to do if marking is not in progress | |
2706 return; | |
2707 | |
2708 // find what the lowest finger is among the global and local fingers | |
2709 _min_finger = _finger; | |
2710 for (int i = 0; i < (int)_max_task_num; ++i) { | |
2711 CMTask* task = _tasks[i]; | |
2712 HeapWord* task_finger = task->finger(); | |
2713 if (task_finger != NULL && task_finger < _min_finger) | |
2714 _min_finger = task_finger; | |
2715 } | |
2716 | |
2717 _should_gray_objects = false; | |
2718 | |
2719 // This fixes a very subtle and fustrating bug. It might be the case | |
2720 // that, during en evacuation pause, heap regions that contain | |
2721 // objects that are gray (by being in regions contained in the | |
2722 // region stack) are included in the collection set. Since such gray | |
2723 // objects will be moved, and because it's not easy to redirect | |
2724 // region stack entries to point to a new location (because objects | |
2725 // in one region might be scattered to multiple regions after they | |
2726 // are copied), one option is to ensure that all marked objects | |
2727 // copied during a pause are pushed on the stack. Notice, however, | |
2728 // that this problem can only happen when the region stack is not | |
2729 // empty during an evacuation pause. So, we make the fix a bit less | |
2730 // conservative and ensure that regions are pushed on the stack, | |
2731 // irrespective whether all collection set regions are below the | |
2732 // finger, if the region stack is not empty. This is expected to be | |
2733 // a rare case, so I don't think it's necessary to be smarted about it. | |
1835
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2734 if (!region_stack_empty() || has_aborted_regions()) |
342 | 2735 _should_gray_objects = true; |
2736 } | |
2737 | |
2738 void ConcurrentMark::registerCSetRegion(HeapRegion* hr) { | |
2739 if (!concurrent_marking_in_progress()) | |
2740 return; | |
2741 | |
2742 HeapWord* region_end = hr->end(); | |
2743 if (region_end > _min_finger) | |
2744 _should_gray_objects = true; | |
2745 } | |
2746 | |
2747 // abandon current marking iteration due to a Full GC | |
2748 void ConcurrentMark::abort() { | |
2749 // Clear all marks to force marking thread to do nothing | |
2750 _nextMarkBitMap->clearAll(); | |
2751 // Empty mark stack | |
2752 clear_marking_state(); | |
1835
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2753 for (int i = 0; i < (int)_max_task_num; ++i) { |
342 | 2754 _tasks[i]->clear_region_fields(); |
1835
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
2755 } |
342 | 2756 _has_aborted = true; |
2757 | |
2758 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); | |
2759 satb_mq_set.abandon_partial_marking(); | |
1317
d4197f8d516a
6935821: G1: threads created during marking do not active their SATB queues
tonyp
parents:
1314
diff
changeset
|
2760 // This can be called either during or outside marking, we'll read |
d4197f8d516a
6935821: G1: threads created during marking do not active their SATB queues
tonyp
parents:
1314
diff
changeset
|
2761 // the expected_active value from the SATB queue set. |
d4197f8d516a
6935821: G1: threads created during marking do not active their SATB queues
tonyp
parents:
1314
diff
changeset
|
2762 satb_mq_set.set_active_all_threads( |
d4197f8d516a
6935821: G1: threads created during marking do not active their SATB queues
tonyp
parents:
1314
diff
changeset
|
2763 false, /* new active value */ |
d4197f8d516a
6935821: G1: threads created during marking do not active their SATB queues
tonyp
parents:
1314
diff
changeset
|
2764 satb_mq_set.is_active() /* expected_active */); |
342 | 2765 } |
2766 | |
2767 static void print_ms_time_info(const char* prefix, const char* name, | |
2768 NumberSeq& ns) { | |
2769 gclog_or_tty->print_cr("%s%5d %12s: total time = %8.2f s (avg = %8.2f ms).", | |
2770 prefix, ns.num(), name, ns.sum()/1000.0, ns.avg()); | |
2771 if (ns.num() > 0) { | |
2772 gclog_or_tty->print_cr("%s [std. dev = %8.2f ms, max = %8.2f ms]", | |
2773 prefix, ns.sd(), ns.maximum()); | |
2774 } | |
2775 } | |
2776 | |
2777 void ConcurrentMark::print_summary_info() { | |
2778 gclog_or_tty->print_cr(" Concurrent marking:"); | |
2779 print_ms_time_info(" ", "init marks", _init_times); | |
2780 print_ms_time_info(" ", "remarks", _remark_times); | |
2781 { | |
2782 print_ms_time_info(" ", "final marks", _remark_mark_times); | |
2783 print_ms_time_info(" ", "weak refs", _remark_weak_ref_times); | |
2784 | |
2785 } | |
2786 print_ms_time_info(" ", "cleanups", _cleanup_times); | |
2787 gclog_or_tty->print_cr(" Final counting total time = %8.2f s (avg = %8.2f ms).", | |
2788 _total_counting_time, | |
2789 (_cleanup_times.num() > 0 ? _total_counting_time * 1000.0 / | |
2790 (double)_cleanup_times.num() | |
2791 : 0.0)); | |
2792 if (G1ScrubRemSets) { | |
2793 gclog_or_tty->print_cr(" RS scrub total time = %8.2f s (avg = %8.2f ms).", | |
2794 _total_rs_scrub_time, | |
2795 (_cleanup_times.num() > 0 ? _total_rs_scrub_time * 1000.0 / | |
2796 (double)_cleanup_times.num() | |
2797 : 0.0)); | |
2798 } | |
2799 gclog_or_tty->print_cr(" Total stop_world time = %8.2f s.", | |
2800 (_init_times.sum() + _remark_times.sum() + | |
2801 _cleanup_times.sum())/1000.0); | |
2802 gclog_or_tty->print_cr(" Total concurrent time = %8.2f s " | |
2803 "(%8.2f s marking, %8.2f s counting).", | |
2804 cmThread()->vtime_accum(), | |
2805 cmThread()->vtime_mark_accum(), | |
2806 cmThread()->vtime_count_accum()); | |
2807 } | |
2808 | |
1019 | 2809 void ConcurrentMark::print_worker_threads_on(outputStream* st) const { |
2810 _parallel_workers->print_worker_threads_on(st); | |
2811 } | |
2812 | |
342 | 2813 // Closures |
2814 // XXX: there seems to be a lot of code duplication here; | |
2815 // should refactor and consolidate the shared code. | |
2816 | |
2817 // This closure is used to mark refs into the CMS generation in | |
2818 // the CMS bit map. Called at the first checkpoint. | |
2819 | |
2820 // We take a break if someone is trying to stop the world. | |
2821 bool ConcurrentMark::do_yield_check(int worker_i) { | |
2822 if (should_yield()) { | |
2823 if (worker_i == 0) | |
2824 _g1h->g1_policy()->record_concurrent_pause(); | |
2825 cmThread()->yield(); | |
2826 if (worker_i == 0) | |
2827 _g1h->g1_policy()->record_concurrent_pause_end(); | |
2828 return true; | |
2829 } else { | |
2830 return false; | |
2831 } | |
2832 } | |
2833 | |
2834 bool ConcurrentMark::should_yield() { | |
2835 return cmThread()->should_yield(); | |
2836 } | |
2837 | |
2838 bool ConcurrentMark::containing_card_is_marked(void* p) { | |
2839 size_t offset = pointer_delta(p, _g1h->reserved_region().start(), 1); | |
2840 return _card_bm.at(offset >> CardTableModRefBS::card_shift); | |
2841 } | |
2842 | |
2843 bool ConcurrentMark::containing_cards_are_marked(void* start, | |
2844 void* last) { | |
2845 return | |
2846 containing_card_is_marked(start) && | |
2847 containing_card_is_marked(last); | |
2848 } | |
2849 | |
2850 #ifndef PRODUCT | |
2851 // for debugging purposes | |
2852 void ConcurrentMark::print_finger() { | |
2853 gclog_or_tty->print_cr("heap ["PTR_FORMAT", "PTR_FORMAT"), global finger = "PTR_FORMAT, | |
2854 _heap_start, _heap_end, _finger); | |
2855 for (int i = 0; i < (int) _max_task_num; ++i) { | |
2856 gclog_or_tty->print(" %d: "PTR_FORMAT, i, _tasks[i]->finger()); | |
2857 } | |
2858 gclog_or_tty->print_cr(""); | |
2859 } | |
2860 #endif | |
2861 | |
2862 // Closure for iteration over bitmaps | |
2863 class CMBitMapClosure : public BitMapClosure { | |
2864 private: | |
2865 // the bitmap that is being iterated over | |
2866 CMBitMap* _nextMarkBitMap; | |
2867 ConcurrentMark* _cm; | |
2868 CMTask* _task; | |
2869 // true if we're scanning a heap region claimed by the task (so that | |
2870 // we move the finger along), false if we're not, i.e. currently when | |
2871 // scanning a heap region popped from the region stack (so that we | |
2872 // do not move the task finger along; it'd be a mistake if we did so). | |
2873 bool _scanning_heap_region; | |
2874 | |
2875 public: | |
2876 CMBitMapClosure(CMTask *task, | |
2877 ConcurrentMark* cm, | |
2878 CMBitMap* nextMarkBitMap) | |
2879 : _task(task), _cm(cm), _nextMarkBitMap(nextMarkBitMap) { } | |
2880 | |
2881 void set_scanning_heap_region(bool scanning_heap_region) { | |
2882 _scanning_heap_region = scanning_heap_region; | |
2883 } | |
2884 | |
2885 bool do_bit(size_t offset) { | |
2886 HeapWord* addr = _nextMarkBitMap->offsetToHeapWord(offset); | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
2887 assert(_nextMarkBitMap->isMarked(addr), "invariant"); |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
2888 assert( addr < _cm->finger(), "invariant"); |
342 | 2889 |
2890 if (_scanning_heap_region) { | |
2891 statsOnly( _task->increase_objs_found_on_bitmap() ); | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
2892 assert(addr >= _task->finger(), "invariant"); |
342 | 2893 // We move that task's local finger along. |
2894 _task->move_finger_to(addr); | |
2895 } else { | |
2896 // We move the task's region finger along. | |
2897 _task->move_region_finger_to(addr); | |
2898 } | |
2899 | |
2900 _task->scan_object(oop(addr)); | |
2901 // we only partially drain the local queue and global stack | |
2902 _task->drain_local_queue(true); | |
2903 _task->drain_global_stack(true); | |
2904 | |
2905 // if the has_aborted flag has been raised, we need to bail out of | |
2906 // the iteration | |
2907 return !_task->has_aborted(); | |
2908 } | |
2909 }; | |
2910 | |
2911 // Closure for iterating over objects, currently only used for | |
2912 // processing SATB buffers. | |
2913 class CMObjectClosure : public ObjectClosure { | |
2914 private: | |
2915 CMTask* _task; | |
2916 | |
2917 public: | |
2918 void do_object(oop obj) { | |
2919 _task->deal_with_reference(obj); | |
2920 } | |
2921 | |
2922 CMObjectClosure(CMTask* task) : _task(task) { } | |
2923 }; | |
2924 | |
2925 // Closure for iterating over object fields | |
2926 class CMOopClosure : public OopClosure { | |
2927 private: | |
2928 G1CollectedHeap* _g1h; | |
2929 ConcurrentMark* _cm; | |
2930 CMTask* _task; | |
2931 | |
2932 public: | |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
2933 virtual void do_oop(narrowOop* p) { do_oop_work(p); } |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
2934 virtual void do_oop( oop* p) { do_oop_work(p); } |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
2935 |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
2936 template <class T> void do_oop_work(T* p) { |
2152 | 2937 assert( _g1h->is_in_g1_reserved((HeapWord*) p), "invariant"); |
2938 assert(!_g1h->is_on_free_list( | |
2939 _g1h->heap_region_containing((HeapWord*) p)), "invariant"); | |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
2940 |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
2941 oop obj = oopDesc::load_decode_heap_oop(p); |
342 | 2942 if (_cm->verbose_high()) |
2943 gclog_or_tty->print_cr("[%d] we're looking at location " | |
2944 "*"PTR_FORMAT" = "PTR_FORMAT, | |
2945 _task->task_id(), p, (void*) obj); | |
2946 _task->deal_with_reference(obj); | |
2947 } | |
2948 | |
2949 CMOopClosure(G1CollectedHeap* g1h, | |
2950 ConcurrentMark* cm, | |
2951 CMTask* task) | |
1974
fd1d227ef1b9
6983204: G1: Nightly test nsk/regression/b4958615 failing with +ExplicitGCInvokesConcurrent
johnc
parents:
1972
diff
changeset
|
2952 : _g1h(g1h), _cm(cm), _task(task) |
fd1d227ef1b9
6983204: G1: Nightly test nsk/regression/b4958615 failing with +ExplicitGCInvokesConcurrent
johnc
parents:
1972
diff
changeset
|
2953 { |
fd1d227ef1b9
6983204: G1: Nightly test nsk/regression/b4958615 failing with +ExplicitGCInvokesConcurrent
johnc
parents:
1972
diff
changeset
|
2954 _ref_processor = g1h->ref_processor(); |
fd1d227ef1b9
6983204: G1: Nightly test nsk/regression/b4958615 failing with +ExplicitGCInvokesConcurrent
johnc
parents:
1972
diff
changeset
|
2955 assert(_ref_processor != NULL, "should not be NULL"); |
fd1d227ef1b9
6983204: G1: Nightly test nsk/regression/b4958615 failing with +ExplicitGCInvokesConcurrent
johnc
parents:
1972
diff
changeset
|
2956 } |
342 | 2957 }; |
2958 | |
2959 void CMTask::setup_for_region(HeapRegion* hr) { | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
2960 // Separated the asserts so that we know which one fires. |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
2961 assert(hr != NULL, |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
2962 "claim_region() should have filtered out continues humongous regions"); |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
2963 assert(!hr->continuesHumongous(), |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
2964 "claim_region() should have filtered out continues humongous regions"); |
342 | 2965 |
2966 if (_cm->verbose_low()) | |
2967 gclog_or_tty->print_cr("[%d] setting up for region "PTR_FORMAT, | |
2968 _task_id, hr); | |
2969 | |
2970 _curr_region = hr; | |
2971 _finger = hr->bottom(); | |
2972 update_region_limit(); | |
2973 } | |
2974 | |
2975 void CMTask::update_region_limit() { | |
2976 HeapRegion* hr = _curr_region; | |
2977 HeapWord* bottom = hr->bottom(); | |
2978 HeapWord* limit = hr->next_top_at_mark_start(); | |
2979 | |
2980 if (limit == bottom) { | |
2981 if (_cm->verbose_low()) | |
2982 gclog_or_tty->print_cr("[%d] found an empty region " | |
2983 "["PTR_FORMAT", "PTR_FORMAT")", | |
2984 _task_id, bottom, limit); | |
2985 // The region was collected underneath our feet. | |
2986 // We set the finger to bottom to ensure that the bitmap | |
2987 // iteration that will follow this will not do anything. | |
2988 // (this is not a condition that holds when we set the region up, | |
2989 // as the region is not supposed to be empty in the first place) | |
2990 _finger = bottom; | |
2991 } else if (limit >= _region_limit) { | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
2992 assert(limit >= _finger, "peace of mind"); |
342 | 2993 } else { |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
2994 assert(limit < _region_limit, "only way to get here"); |
342 | 2995 // This can happen under some pretty unusual circumstances. An |
2996 // evacuation pause empties the region underneath our feet (NTAMS | |
2997 // at bottom). We then do some allocation in the region (NTAMS | |
2998 // stays at bottom), followed by the region being used as a GC | |
2999 // alloc region (NTAMS will move to top() and the objects | |
3000 // originally below it will be grayed). All objects now marked in | |
3001 // the region are explicitly grayed, if below the global finger, | |
3002 // and we do not need in fact to scan anything else. So, we simply | |
3003 // set _finger to be limit to ensure that the bitmap iteration | |
3004 // doesn't do anything. | |
3005 _finger = limit; | |
3006 } | |
3007 | |
3008 _region_limit = limit; | |
3009 } | |
3010 | |
3011 void CMTask::giveup_current_region() { | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3012 assert(_curr_region != NULL, "invariant"); |
342 | 3013 if (_cm->verbose_low()) |
3014 gclog_or_tty->print_cr("[%d] giving up region "PTR_FORMAT, | |
3015 _task_id, _curr_region); | |
3016 clear_region_fields(); | |
3017 } | |
3018 | |
3019 void CMTask::clear_region_fields() { | |
3020 // Values for these three fields that indicate that we're not | |
3021 // holding on to a region. | |
3022 _curr_region = NULL; | |
3023 _finger = NULL; | |
3024 _region_limit = NULL; | |
3025 | |
3026 _region_finger = NULL; | |
3027 } | |
3028 | |
3029 void CMTask::reset(CMBitMap* nextMarkBitMap) { | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3030 guarantee(nextMarkBitMap != NULL, "invariant"); |
342 | 3031 |
3032 if (_cm->verbose_low()) | |
3033 gclog_or_tty->print_cr("[%d] resetting", _task_id); | |
3034 | |
3035 _nextMarkBitMap = nextMarkBitMap; | |
3036 clear_region_fields(); | |
1885
a5c514e74487
6988458: G1: assert(mr.end() <= _cm->finger()) failed: otherwise the region shouldn't be on the stack
johnc
parents:
1835
diff
changeset
|
3037 assert(_aborted_region.is_empty(), "should have been cleared"); |
342 | 3038 |
3039 _calls = 0; | |
3040 _elapsed_time_ms = 0.0; | |
3041 _termination_time_ms = 0.0; | |
3042 _termination_start_time_ms = 0.0; | |
3043 | |
3044 #if _MARKING_STATS_ | |
3045 _local_pushes = 0; | |
3046 _local_pops = 0; | |
3047 _local_max_size = 0; | |
3048 _objs_scanned = 0; | |
3049 _global_pushes = 0; | |
3050 _global_pops = 0; | |
3051 _global_max_size = 0; | |
3052 _global_transfers_to = 0; | |
3053 _global_transfers_from = 0; | |
3054 _region_stack_pops = 0; | |
3055 _regions_claimed = 0; | |
3056 _objs_found_on_bitmap = 0; | |
3057 _satb_buffers_processed = 0; | |
3058 _steal_attempts = 0; | |
3059 _steals = 0; | |
3060 _aborted = 0; | |
3061 _aborted_overflow = 0; | |
3062 _aborted_cm_aborted = 0; | |
3063 _aborted_yield = 0; | |
3064 _aborted_timed_out = 0; | |
3065 _aborted_satb = 0; | |
3066 _aborted_termination = 0; | |
3067 #endif // _MARKING_STATS_ | |
3068 } | |
3069 | |
3070 bool CMTask::should_exit_termination() { | |
3071 regular_clock_call(); | |
3072 // This is called when we are in the termination protocol. We should | |
3073 // quit if, for some reason, this task wants to abort or the global | |
3074 // stack is not empty (this means that we can get work from it). | |
3075 return !_cm->mark_stack_empty() || has_aborted(); | |
3076 } | |
3077 | |
3078 // This determines whether the method below will check both the local | |
3079 // and global fingers when determining whether to push on the stack a | |
3080 // gray object (value 1) or whether it will only check the global one | |
3081 // (value 0). The tradeoffs are that the former will be a bit more | |
3082 // accurate and possibly push less on the stack, but it might also be | |
3083 // a little bit slower. | |
3084 | |
3085 #define _CHECK_BOTH_FINGERS_ 1 | |
3086 | |
3087 void CMTask::deal_with_reference(oop obj) { | |
3088 if (_cm->verbose_high()) | |
3089 gclog_or_tty->print_cr("[%d] we're dealing with reference = "PTR_FORMAT, | |
3090 _task_id, (void*) obj); | |
3091 | |
3092 ++_refs_reached; | |
3093 | |
3094 HeapWord* objAddr = (HeapWord*) obj; | |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
831
diff
changeset
|
3095 assert(obj->is_oop_or_null(true /* ignore mark word */), "Error"); |
342 | 3096 if (_g1h->is_in_g1_reserved(objAddr)) { |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3097 assert(obj != NULL, "is_in_g1_reserved should ensure this"); |
342 | 3098 HeapRegion* hr = _g1h->heap_region_containing(obj); |
3099 if (_g1h->is_obj_ill(obj, hr)) { | |
3100 if (_cm->verbose_high()) | |
3101 gclog_or_tty->print_cr("[%d] "PTR_FORMAT" is not considered marked", | |
3102 _task_id, (void*) obj); | |
3103 | |
3104 // we need to mark it first | |
3105 if (_nextMarkBitMap->parMark(objAddr)) { | |
3106 // No OrderAccess:store_load() is needed. It is implicit in the | |
3107 // CAS done in parMark(objAddr) above | |
3108 HeapWord* global_finger = _cm->finger(); | |
3109 | |
3110 #if _CHECK_BOTH_FINGERS_ | |
3111 // we will check both the local and global fingers | |
3112 | |
3113 if (_finger != NULL && objAddr < _finger) { | |
3114 if (_cm->verbose_high()) | |
3115 gclog_or_tty->print_cr("[%d] below the local finger ("PTR_FORMAT"), " | |
3116 "pushing it", _task_id, _finger); | |
3117 push(obj); | |
3118 } else if (_curr_region != NULL && objAddr < _region_limit) { | |
3119 // do nothing | |
3120 } else if (objAddr < global_finger) { | |
3121 // Notice that the global finger might be moving forward | |
3122 // concurrently. This is not a problem. In the worst case, we | |
3123 // mark the object while it is above the global finger and, by | |
3124 // the time we read the global finger, it has moved forward | |
3125 // passed this object. In this case, the object will probably | |
3126 // be visited when a task is scanning the region and will also | |
3127 // be pushed on the stack. So, some duplicate work, but no | |
3128 // correctness problems. | |
3129 | |
3130 if (_cm->verbose_high()) | |
3131 gclog_or_tty->print_cr("[%d] below the global finger " | |
3132 "("PTR_FORMAT"), pushing it", | |
3133 _task_id, global_finger); | |
3134 push(obj); | |
3135 } else { | |
3136 // do nothing | |
3137 } | |
3138 #else // _CHECK_BOTH_FINGERS_ | |
3139 // we will only check the global finger | |
3140 | |
3141 if (objAddr < global_finger) { | |
3142 // see long comment above | |
3143 | |
3144 if (_cm->verbose_high()) | |
3145 gclog_or_tty->print_cr("[%d] below the global finger " | |
3146 "("PTR_FORMAT"), pushing it", | |
3147 _task_id, global_finger); | |
3148 push(obj); | |
3149 } | |
3150 #endif // _CHECK_BOTH_FINGERS_ | |
3151 } | |
3152 } | |
3153 } | |
3154 } | |
3155 | |
3156 void CMTask::push(oop obj) { | |
3157 HeapWord* objAddr = (HeapWord*) obj; | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3158 assert(_g1h->is_in_g1_reserved(objAddr), "invariant"); |
2152 | 3159 assert(!_g1h->is_on_free_list( |
3160 _g1h->heap_region_containing((HeapWord*) objAddr)), "invariant"); | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3161 assert(!_g1h->is_obj_ill(obj), "invariant"); |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3162 assert(_nextMarkBitMap->isMarked(objAddr), "invariant"); |
342 | 3163 |
3164 if (_cm->verbose_high()) | |
3165 gclog_or_tty->print_cr("[%d] pushing "PTR_FORMAT, _task_id, (void*) obj); | |
3166 | |
3167 if (!_task_queue->push(obj)) { | |
3168 // The local task queue looks full. We need to push some entries | |
3169 // to the global stack. | |
3170 | |
3171 if (_cm->verbose_medium()) | |
3172 gclog_or_tty->print_cr("[%d] task queue overflow, " | |
3173 "moving entries to the global stack", | |
3174 _task_id); | |
3175 move_entries_to_global_stack(); | |
3176 | |
3177 // this should succeed since, even if we overflow the global | |
3178 // stack, we should have definitely removed some entries from the | |
3179 // local queue. So, there must be space on it. | |
3180 bool success = _task_queue->push(obj); | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3181 assert(success, "invariant"); |
342 | 3182 } |
3183 | |
3184 statsOnly( int tmp_size = _task_queue->size(); | |
3185 if (tmp_size > _local_max_size) | |
3186 _local_max_size = tmp_size; | |
3187 ++_local_pushes ); | |
3188 } | |
3189 | |
3190 void CMTask::reached_limit() { | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3191 assert(_words_scanned >= _words_scanned_limit || |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3192 _refs_reached >= _refs_reached_limit , |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3193 "shouldn't have been called otherwise"); |
342 | 3194 regular_clock_call(); |
3195 } | |
3196 | |
3197 void CMTask::regular_clock_call() { | |
3198 if (has_aborted()) | |
3199 return; | |
3200 | |
3201 // First, we need to recalculate the words scanned and refs reached | |
3202 // limits for the next clock call. | |
3203 recalculate_limits(); | |
3204 | |
3205 // During the regular clock call we do the following | |
3206 | |
3207 // (1) If an overflow has been flagged, then we abort. | |
3208 if (_cm->has_overflown()) { | |
3209 set_has_aborted(); | |
3210 return; | |
3211 } | |
3212 | |
3213 // If we are not concurrent (i.e. we're doing remark) we don't need | |
3214 // to check anything else. The other steps are only needed during | |
3215 // the concurrent marking phase. | |
3216 if (!concurrent()) | |
3217 return; | |
3218 | |
3219 // (2) If marking has been aborted for Full GC, then we also abort. | |
3220 if (_cm->has_aborted()) { | |
3221 set_has_aborted(); | |
3222 statsOnly( ++_aborted_cm_aborted ); | |
3223 return; | |
3224 } | |
3225 | |
3226 double curr_time_ms = os::elapsedVTime() * 1000.0; | |
3227 | |
3228 // (3) If marking stats are enabled, then we update the step history. | |
3229 #if _MARKING_STATS_ | |
3230 if (_words_scanned >= _words_scanned_limit) | |
3231 ++_clock_due_to_scanning; | |
3232 if (_refs_reached >= _refs_reached_limit) | |
3233 ++_clock_due_to_marking; | |
3234 | |
3235 double last_interval_ms = curr_time_ms - _interval_start_time_ms; | |
3236 _interval_start_time_ms = curr_time_ms; | |
3237 _all_clock_intervals_ms.add(last_interval_ms); | |
3238 | |
3239 if (_cm->verbose_medium()) { | |
3240 gclog_or_tty->print_cr("[%d] regular clock, interval = %1.2lfms, " | |
3241 "scanned = %d%s, refs reached = %d%s", | |
3242 _task_id, last_interval_ms, | |
3243 _words_scanned, | |
3244 (_words_scanned >= _words_scanned_limit) ? " (*)" : "", | |
3245 _refs_reached, | |
3246 (_refs_reached >= _refs_reached_limit) ? " (*)" : ""); | |
3247 } | |
3248 #endif // _MARKING_STATS_ | |
3249 | |
3250 // (4) We check whether we should yield. If we have to, then we abort. | |
3251 if (_cm->should_yield()) { | |
3252 // We should yield. To do this we abort the task. The caller is | |
3253 // responsible for yielding. | |
3254 set_has_aborted(); | |
3255 statsOnly( ++_aborted_yield ); | |
3256 return; | |
3257 } | |
3258 | |
3259 // (5) We check whether we've reached our time quota. If we have, | |
3260 // then we abort. | |
3261 double elapsed_time_ms = curr_time_ms - _start_time_ms; | |
3262 if (elapsed_time_ms > _time_target_ms) { | |
3263 set_has_aborted(); | |
3264 _has_aborted_timed_out = true; | |
3265 statsOnly( ++_aborted_timed_out ); | |
3266 return; | |
3267 } | |
3268 | |
3269 // (6) Finally, we check whether there are enough completed STAB | |
3270 // buffers available for processing. If there are, we abort. | |
3271 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); | |
3272 if (!_draining_satb_buffers && satb_mq_set.process_completed_buffers()) { | |
3273 if (_cm->verbose_low()) | |
3274 gclog_or_tty->print_cr("[%d] aborting to deal with pending SATB buffers", | |
3275 _task_id); | |
3276 // we do need to process SATB buffers, we'll abort and restart | |
3277 // the marking task to do so | |
3278 set_has_aborted(); | |
3279 statsOnly( ++_aborted_satb ); | |
3280 return; | |
3281 } | |
3282 } | |
3283 | |
3284 void CMTask::recalculate_limits() { | |
3285 _real_words_scanned_limit = _words_scanned + words_scanned_period; | |
3286 _words_scanned_limit = _real_words_scanned_limit; | |
3287 | |
3288 _real_refs_reached_limit = _refs_reached + refs_reached_period; | |
3289 _refs_reached_limit = _real_refs_reached_limit; | |
3290 } | |
3291 | |
3292 void CMTask::decrease_limits() { | |
3293 // This is called when we believe that we're going to do an infrequent | |
3294 // operation which will increase the per byte scanned cost (i.e. move | |
3295 // entries to/from the global stack). It basically tries to decrease the | |
3296 // scanning limit so that the clock is called earlier. | |
3297 | |
3298 if (_cm->verbose_medium()) | |
3299 gclog_or_tty->print_cr("[%d] decreasing limits", _task_id); | |
3300 | |
3301 _words_scanned_limit = _real_words_scanned_limit - | |
3302 3 * words_scanned_period / 4; | |
3303 _refs_reached_limit = _real_refs_reached_limit - | |
3304 3 * refs_reached_period / 4; | |
3305 } | |
3306 | |
3307 void CMTask::move_entries_to_global_stack() { | |
3308 // local array where we'll store the entries that will be popped | |
3309 // from the local queue | |
3310 oop buffer[global_stack_transfer_size]; | |
3311 | |
3312 int n = 0; | |
3313 oop obj; | |
3314 while (n < global_stack_transfer_size && _task_queue->pop_local(obj)) { | |
3315 buffer[n] = obj; | |
3316 ++n; | |
3317 } | |
3318 | |
3319 if (n > 0) { | |
3320 // we popped at least one entry from the local queue | |
3321 | |
3322 statsOnly( ++_global_transfers_to; _local_pops += n ); | |
3323 | |
3324 if (!_cm->mark_stack_push(buffer, n)) { | |
3325 if (_cm->verbose_low()) | |
3326 gclog_or_tty->print_cr("[%d] aborting due to global stack overflow", _task_id); | |
3327 set_has_aborted(); | |
3328 } else { | |
3329 // the transfer was successful | |
3330 | |
3331 if (_cm->verbose_medium()) | |
3332 gclog_or_tty->print_cr("[%d] pushed %d entries to the global stack", | |
3333 _task_id, n); | |
3334 statsOnly( int tmp_size = _cm->mark_stack_size(); | |
3335 if (tmp_size > _global_max_size) | |
3336 _global_max_size = tmp_size; | |
3337 _global_pushes += n ); | |
3338 } | |
3339 } | |
3340 | |
3341 // this operation was quite expensive, so decrease the limits | |
3342 decrease_limits(); | |
3343 } | |
3344 | |
3345 void CMTask::get_entries_from_global_stack() { | |
3346 // local array where we'll store the entries that will be popped | |
3347 // from the global stack. | |
3348 oop buffer[global_stack_transfer_size]; | |
3349 int n; | |
3350 _cm->mark_stack_pop(buffer, global_stack_transfer_size, &n); | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3351 assert(n <= global_stack_transfer_size, |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3352 "we should not pop more than the given limit"); |
342 | 3353 if (n > 0) { |
3354 // yes, we did actually pop at least one entry | |
3355 | |
3356 statsOnly( ++_global_transfers_from; _global_pops += n ); | |
3357 if (_cm->verbose_medium()) | |
3358 gclog_or_tty->print_cr("[%d] popped %d entries from the global stack", | |
3359 _task_id, n); | |
3360 for (int i = 0; i < n; ++i) { | |
3361 bool success = _task_queue->push(buffer[i]); | |
3362 // We only call this when the local queue is empty or under a | |
3363 // given target limit. So, we do not expect this push to fail. | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3364 assert(success, "invariant"); |
342 | 3365 } |
3366 | |
3367 statsOnly( int tmp_size = _task_queue->size(); | |
3368 if (tmp_size > _local_max_size) | |
3369 _local_max_size = tmp_size; | |
3370 _local_pushes += n ); | |
3371 } | |
3372 | |
3373 // this operation was quite expensive, so decrease the limits | |
3374 decrease_limits(); | |
3375 } | |
3376 | |
3377 void CMTask::drain_local_queue(bool partially) { | |
3378 if (has_aborted()) | |
3379 return; | |
3380 | |
3381 // Decide what the target size is, depending whether we're going to | |
3382 // drain it partially (so that other tasks can steal if they run out | |
3383 // of things to do) or totally (at the very end). | |
3384 size_t target_size; | |
3385 if (partially) | |
3386 target_size = MIN2((size_t)_task_queue->max_elems()/3, GCDrainStackTargetSize); | |
3387 else | |
3388 target_size = 0; | |
3389 | |
3390 if (_task_queue->size() > target_size) { | |
3391 if (_cm->verbose_high()) | |
3392 gclog_or_tty->print_cr("[%d] draining local queue, target size = %d", | |
3393 _task_id, target_size); | |
3394 | |
3395 oop obj; | |
3396 bool ret = _task_queue->pop_local(obj); | |
3397 while (ret) { | |
3398 statsOnly( ++_local_pops ); | |
3399 | |
3400 if (_cm->verbose_high()) | |
3401 gclog_or_tty->print_cr("[%d] popped "PTR_FORMAT, _task_id, | |
3402 (void*) obj); | |
3403 | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3404 assert(_g1h->is_in_g1_reserved((HeapWord*) obj), "invariant" ); |
2152 | 3405 assert(!_g1h->is_on_free_list( |
3406 _g1h->heap_region_containing((HeapWord*) obj)), "invariant"); | |
342 | 3407 |
3408 scan_object(obj); | |
3409 | |
3410 if (_task_queue->size() <= target_size || has_aborted()) | |
3411 ret = false; | |
3412 else | |
3413 ret = _task_queue->pop_local(obj); | |
3414 } | |
3415 | |
3416 if (_cm->verbose_high()) | |
3417 gclog_or_tty->print_cr("[%d] drained local queue, size = %d", | |
3418 _task_id, _task_queue->size()); | |
3419 } | |
3420 } | |
3421 | |
3422 void CMTask::drain_global_stack(bool partially) { | |
3423 if (has_aborted()) | |
3424 return; | |
3425 | |
3426 // We have a policy to drain the local queue before we attempt to | |
3427 // drain the global stack. | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3428 assert(partially || _task_queue->size() == 0, "invariant"); |
342 | 3429 |
3430 // Decide what the target size is, depending whether we're going to | |
3431 // drain it partially (so that other tasks can steal if they run out | |
3432 // of things to do) or totally (at the very end). Notice that, | |
3433 // because we move entries from the global stack in chunks or | |
3434 // because another task might be doing the same, we might in fact | |
3435 // drop below the target. But, this is not a problem. | |
3436 size_t target_size; | |
3437 if (partially) | |
3438 target_size = _cm->partial_mark_stack_size_target(); | |
3439 else | |
3440 target_size = 0; | |
3441 | |
3442 if (_cm->mark_stack_size() > target_size) { | |
3443 if (_cm->verbose_low()) | |
3444 gclog_or_tty->print_cr("[%d] draining global_stack, target size %d", | |
3445 _task_id, target_size); | |
3446 | |
3447 while (!has_aborted() && _cm->mark_stack_size() > target_size) { | |
3448 get_entries_from_global_stack(); | |
3449 drain_local_queue(partially); | |
3450 } | |
3451 | |
3452 if (_cm->verbose_low()) | |
3453 gclog_or_tty->print_cr("[%d] drained global stack, size = %d", | |
3454 _task_id, _cm->mark_stack_size()); | |
3455 } | |
3456 } | |
3457 | |
3458 // SATB Queue has several assumptions on whether to call the par or | |
3459 // non-par versions of the methods. this is why some of the code is | |
3460 // replicated. We should really get rid of the single-threaded version | |
3461 // of the code to simplify things. | |
3462 void CMTask::drain_satb_buffers() { | |
3463 if (has_aborted()) | |
3464 return; | |
3465 | |
3466 // We set this so that the regular clock knows that we're in the | |
3467 // middle of draining buffers and doesn't set the abort flag when it | |
3468 // notices that SATB buffers are available for draining. It'd be | |
3469 // very counter productive if it did that. :-) | |
3470 _draining_satb_buffers = true; | |
3471 | |
3472 CMObjectClosure oc(this); | |
3473 SATBMarkQueueSet& satb_mq_set = JavaThread::satb_mark_queue_set(); | |
1833
8b10f48633dc
6984287: Regularize how GC parallel workers are specified.
jmasa
parents:
1719
diff
changeset
|
3474 if (G1CollectedHeap::use_parallel_gc_threads()) |
342 | 3475 satb_mq_set.set_par_closure(_task_id, &oc); |
3476 else | |
3477 satb_mq_set.set_closure(&oc); | |
3478 | |
3479 // This keeps claiming and applying the closure to completed buffers | |
3480 // until we run out of buffers or we need to abort. | |
1833
8b10f48633dc
6984287: Regularize how GC parallel workers are specified.
jmasa
parents:
1719
diff
changeset
|
3481 if (G1CollectedHeap::use_parallel_gc_threads()) { |
342 | 3482 while (!has_aborted() && |
3483 satb_mq_set.par_apply_closure_to_completed_buffer(_task_id)) { | |
3484 if (_cm->verbose_medium()) | |
3485 gclog_or_tty->print_cr("[%d] processed an SATB buffer", _task_id); | |
3486 statsOnly( ++_satb_buffers_processed ); | |
3487 regular_clock_call(); | |
3488 } | |
3489 } else { | |
3490 while (!has_aborted() && | |
3491 satb_mq_set.apply_closure_to_completed_buffer()) { | |
3492 if (_cm->verbose_medium()) | |
3493 gclog_or_tty->print_cr("[%d] processed an SATB buffer", _task_id); | |
3494 statsOnly( ++_satb_buffers_processed ); | |
3495 regular_clock_call(); | |
3496 } | |
3497 } | |
3498 | |
3499 if (!concurrent() && !has_aborted()) { | |
3500 // We should only do this during remark. | |
1833
8b10f48633dc
6984287: Regularize how GC parallel workers are specified.
jmasa
parents:
1719
diff
changeset
|
3501 if (G1CollectedHeap::use_parallel_gc_threads()) |
342 | 3502 satb_mq_set.par_iterate_closure_all_threads(_task_id); |
3503 else | |
3504 satb_mq_set.iterate_closure_all_threads(); | |
3505 } | |
3506 | |
3507 _draining_satb_buffers = false; | |
3508 | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3509 assert(has_aborted() || |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3510 concurrent() || |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3511 satb_mq_set.completed_buffers_num() == 0, "invariant"); |
342 | 3512 |
1833
8b10f48633dc
6984287: Regularize how GC parallel workers are specified.
jmasa
parents:
1719
diff
changeset
|
3513 if (G1CollectedHeap::use_parallel_gc_threads()) |
342 | 3514 satb_mq_set.set_par_closure(_task_id, NULL); |
3515 else | |
3516 satb_mq_set.set_closure(NULL); | |
3517 | |
3518 // again, this was a potentially expensive operation, decrease the | |
3519 // limits to get the regular clock call early | |
3520 decrease_limits(); | |
3521 } | |
3522 | |
3523 void CMTask::drain_region_stack(BitMapClosure* bc) { | |
3524 if (has_aborted()) | |
3525 return; | |
3526 | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3527 assert(_region_finger == NULL, |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3528 "it should be NULL when we're not scanning a region"); |
342 | 3529 |
1835
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
3530 if (!_cm->region_stack_empty() || !_aborted_region.is_empty()) { |
342 | 3531 if (_cm->verbose_low()) |
3532 gclog_or_tty->print_cr("[%d] draining region stack, size = %d", | |
3533 _task_id, _cm->region_stack_size()); | |
3534 | |
1835
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
3535 MemRegion mr; |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
3536 |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
3537 if (!_aborted_region.is_empty()) { |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
3538 mr = _aborted_region; |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
3539 _aborted_region = MemRegion(); |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
3540 |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
3541 if (_cm->verbose_low()) |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
3542 gclog_or_tty->print_cr("[%d] scanning aborted region [ " PTR_FORMAT ", " PTR_FORMAT " )", |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
3543 _task_id, mr.start(), mr.end()); |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
3544 } else { |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
3545 mr = _cm->region_stack_pop_lock_free(); |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
3546 // it returns MemRegion() if the pop fails |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
3547 statsOnly(if (mr.start() != NULL) ++_region_stack_pops ); |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
3548 } |
342 | 3549 |
3550 while (mr.start() != NULL) { | |
3551 if (_cm->verbose_medium()) | |
3552 gclog_or_tty->print_cr("[%d] we are scanning region " | |
3553 "["PTR_FORMAT", "PTR_FORMAT")", | |
3554 _task_id, mr.start(), mr.end()); | |
1835
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
3555 |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3556 assert(mr.end() <= _cm->finger(), |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3557 "otherwise the region shouldn't be on the stack"); |
342 | 3558 assert(!mr.is_empty(), "Only non-empty regions live on the region stack"); |
3559 if (_nextMarkBitMap->iterate(bc, mr)) { | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3560 assert(!has_aborted(), |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3561 "cannot abort the task without aborting the bitmap iteration"); |
342 | 3562 |
3563 // We finished iterating over the region without aborting. | |
3564 regular_clock_call(); | |
3565 if (has_aborted()) | |
3566 mr = MemRegion(); | |
3567 else { | |
1835
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
3568 mr = _cm->region_stack_pop_lock_free(); |
342 | 3569 // it returns MemRegion() if the pop fails |
3570 statsOnly(if (mr.start() != NULL) ++_region_stack_pops ); | |
3571 } | |
3572 } else { | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3573 assert(has_aborted(), "currently the only way to do so"); |
342 | 3574 |
3575 // The only way to abort the bitmap iteration is to return | |
3576 // false from the do_bit() method. However, inside the | |
3577 // do_bit() method we move the _region_finger to point to the | |
3578 // object currently being looked at. So, if we bail out, we | |
3579 // have definitely set _region_finger to something non-null. | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3580 assert(_region_finger != NULL, "invariant"); |
342 | 3581 |
1835
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
3582 // Make sure that any previously aborted region has been |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
3583 // cleared. |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
3584 assert(_aborted_region.is_empty(), "aborted region not cleared"); |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
3585 |
342 | 3586 // The iteration was actually aborted. So now _region_finger |
3587 // points to the address of the object we last scanned. If we | |
3588 // leave it there, when we restart this task, we will rescan | |
3589 // the object. It is easy to avoid this. We move the finger by | |
3590 // enough to point to the next possible object header (the | |
3591 // bitmap knows by how much we need to move it as it knows its | |
3592 // granularity). | |
3593 MemRegion newRegion = | |
3594 MemRegion(_nextMarkBitMap->nextWord(_region_finger), mr.end()); | |
3595 | |
3596 if (!newRegion.is_empty()) { | |
3597 if (_cm->verbose_low()) { | |
1835
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
3598 gclog_or_tty->print_cr("[%d] recording unscanned region" |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
3599 "[" PTR_FORMAT "," PTR_FORMAT ") in CMTask", |
342 | 3600 _task_id, |
3601 newRegion.start(), newRegion.end()); | |
3602 } | |
1835
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
3603 // Now record the part of the region we didn't scan to |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
3604 // make sure this task scans it later. |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
3605 _aborted_region = newRegion; |
342 | 3606 } |
3607 // break from while | |
3608 mr = MemRegion(); | |
3609 } | |
3610 _region_finger = NULL; | |
3611 } | |
3612 | |
3613 if (_cm->verbose_low()) | |
3614 gclog_or_tty->print_cr("[%d] drained region stack, size = %d", | |
3615 _task_id, _cm->region_stack_size()); | |
3616 } | |
3617 } | |
3618 | |
3619 void CMTask::print_stats() { | |
3620 gclog_or_tty->print_cr("Marking Stats, task = %d, calls = %d", | |
3621 _task_id, _calls); | |
3622 gclog_or_tty->print_cr(" Elapsed time = %1.2lfms, Termination time = %1.2lfms", | |
3623 _elapsed_time_ms, _termination_time_ms); | |
3624 gclog_or_tty->print_cr(" Step Times (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms", | |
3625 _step_times_ms.num(), _step_times_ms.avg(), | |
3626 _step_times_ms.sd()); | |
3627 gclog_or_tty->print_cr(" max = %1.2lfms, total = %1.2lfms", | |
3628 _step_times_ms.maximum(), _step_times_ms.sum()); | |
3629 | |
3630 #if _MARKING_STATS_ | |
3631 gclog_or_tty->print_cr(" Clock Intervals (cum): num = %d, avg = %1.2lfms, sd = %1.2lfms", | |
3632 _all_clock_intervals_ms.num(), _all_clock_intervals_ms.avg(), | |
3633 _all_clock_intervals_ms.sd()); | |
3634 gclog_or_tty->print_cr(" max = %1.2lfms, total = %1.2lfms", | |
3635 _all_clock_intervals_ms.maximum(), | |
3636 _all_clock_intervals_ms.sum()); | |
3637 gclog_or_tty->print_cr(" Clock Causes (cum): scanning = %d, marking = %d", | |
3638 _clock_due_to_scanning, _clock_due_to_marking); | |
3639 gclog_or_tty->print_cr(" Objects: scanned = %d, found on the bitmap = %d", | |
3640 _objs_scanned, _objs_found_on_bitmap); | |
3641 gclog_or_tty->print_cr(" Local Queue: pushes = %d, pops = %d, max size = %d", | |
3642 _local_pushes, _local_pops, _local_max_size); | |
3643 gclog_or_tty->print_cr(" Global Stack: pushes = %d, pops = %d, max size = %d", | |
3644 _global_pushes, _global_pops, _global_max_size); | |
3645 gclog_or_tty->print_cr(" transfers to = %d, transfers from = %d", | |
3646 _global_transfers_to,_global_transfers_from); | |
3647 gclog_or_tty->print_cr(" Regions: claimed = %d, Region Stack: pops = %d", | |
3648 _regions_claimed, _region_stack_pops); | |
3649 gclog_or_tty->print_cr(" SATB buffers: processed = %d", _satb_buffers_processed); | |
3650 gclog_or_tty->print_cr(" Steals: attempts = %d, successes = %d", | |
3651 _steal_attempts, _steals); | |
3652 gclog_or_tty->print_cr(" Aborted: %d, due to", _aborted); | |
3653 gclog_or_tty->print_cr(" overflow: %d, global abort: %d, yield: %d", | |
3654 _aborted_overflow, _aborted_cm_aborted, _aborted_yield); | |
3655 gclog_or_tty->print_cr(" time out: %d, SATB: %d, termination: %d", | |
3656 _aborted_timed_out, _aborted_satb, _aborted_termination); | |
3657 #endif // _MARKING_STATS_ | |
3658 } | |
3659 | |
3660 /***************************************************************************** | |
3661 | |
3662 The do_marking_step(time_target_ms) method is the building block | |
3663 of the parallel marking framework. It can be called in parallel | |
3664 with other invocations of do_marking_step() on different tasks | |
3665 (but only one per task, obviously) and concurrently with the | |
3666 mutator threads, or during remark, hence it eliminates the need | |
3667 for two versions of the code. When called during remark, it will | |
3668 pick up from where the task left off during the concurrent marking | |
3669 phase. Interestingly, tasks are also claimable during evacuation | |
3670 pauses too, since do_marking_step() ensures that it aborts before | |
3671 it needs to yield. | |
3672 | |
3673 The data structures that is uses to do marking work are the | |
3674 following: | |
3675 | |
3676 (1) Marking Bitmap. If there are gray objects that appear only | |
3677 on the bitmap (this happens either when dealing with an overflow | |
3678 or when the initial marking phase has simply marked the roots | |
3679 and didn't push them on the stack), then tasks claim heap | |
3680 regions whose bitmap they then scan to find gray objects. A | |
3681 global finger indicates where the end of the last claimed region | |
3682 is. A local finger indicates how far into the region a task has | |
3683 scanned. The two fingers are used to determine how to gray an | |
3684 object (i.e. whether simply marking it is OK, as it will be | |
3685 visited by a task in the future, or whether it needs to be also | |
3686 pushed on a stack). | |
3687 | |
3688 (2) Local Queue. The local queue of the task which is accessed | |
3689 reasonably efficiently by the task. Other tasks can steal from | |
3690 it when they run out of work. Throughout the marking phase, a | |
3691 task attempts to keep its local queue short but not totally | |
3692 empty, so that entries are available for stealing by other | |
3693 tasks. Only when there is no more work, a task will totally | |
3694 drain its local queue. | |
3695 | |
3696 (3) Global Mark Stack. This handles local queue overflow. During | |
3697 marking only sets of entries are moved between it and the local | |
3698 queues, as access to it requires a mutex and more fine-grain | |
3699 interaction with it which might cause contention. If it | |
3700 overflows, then the marking phase should restart and iterate | |
3701 over the bitmap to identify gray objects. Throughout the marking | |
3702 phase, tasks attempt to keep the global mark stack at a small | |
3703 length but not totally empty, so that entries are available for | |
3704 popping by other tasks. Only when there is no more work, tasks | |
3705 will totally drain the global mark stack. | |
3706 | |
3707 (4) Global Region Stack. Entries on it correspond to areas of | |
3708 the bitmap that need to be scanned since they contain gray | |
3709 objects. Pushes on the region stack only happen during | |
3710 evacuation pauses and typically correspond to areas covered by | |
3711 GC LABS. If it overflows, then the marking phase should restart | |
3712 and iterate over the bitmap to identify gray objects. Tasks will | |
3713 try to totally drain the region stack as soon as possible. | |
3714 | |
3715 (5) SATB Buffer Queue. This is where completed SATB buffers are | |
3716 made available. Buffers are regularly removed from this queue | |
3717 and scanned for roots, so that the queue doesn't get too | |
3718 long. During remark, all completed buffers are processed, as | |
3719 well as the filled in parts of any uncompleted buffers. | |
3720 | |
3721 The do_marking_step() method tries to abort when the time target | |
3722 has been reached. There are a few other cases when the | |
3723 do_marking_step() method also aborts: | |
3724 | |
3725 (1) When the marking phase has been aborted (after a Full GC). | |
3726 | |
3727 (2) When a global overflow (either on the global stack or the | |
3728 region stack) has been triggered. Before the task aborts, it | |
3729 will actually sync up with the other tasks to ensure that all | |
3730 the marking data structures (local queues, stacks, fingers etc.) | |
3731 are re-initialised so that when do_marking_step() completes, | |
3732 the marking phase can immediately restart. | |
3733 | |
3734 (3) When enough completed SATB buffers are available. The | |
3735 do_marking_step() method only tries to drain SATB buffers right | |
3736 at the beginning. So, if enough buffers are available, the | |
3737 marking step aborts and the SATB buffers are processed at | |
3738 the beginning of the next invocation. | |
3739 | |
3740 (4) To yield. when we have to yield then we abort and yield | |
3741 right at the end of do_marking_step(). This saves us from a lot | |
3742 of hassle as, by yielding we might allow a Full GC. If this | |
3743 happens then objects will be compacted underneath our feet, the | |
3744 heap might shrink, etc. We save checking for this by just | |
3745 aborting and doing the yield right at the end. | |
3746 | |
3747 From the above it follows that the do_marking_step() method should | |
3748 be called in a loop (or, otherwise, regularly) until it completes. | |
3749 | |
3750 If a marking step completes without its has_aborted() flag being | |
3751 true, it means it has completed the current marking phase (and | |
3752 also all other marking tasks have done so and have all synced up). | |
3753 | |
3754 A method called regular_clock_call() is invoked "regularly" (in | |
3755 sub ms intervals) throughout marking. It is this clock method that | |
3756 checks all the abort conditions which were mentioned above and | |
3757 decides when the task should abort. A work-based scheme is used to | |
3758 trigger this clock method: when the number of object words the | |
3759 marking phase has scanned or the number of references the marking | |
3760 phase has visited reach a given limit. Additional invocations to | |
3761 the method clock have been planted in a few other strategic places | |
3762 too. The initial reason for the clock method was to avoid calling | |
3763 vtime too regularly, as it is quite expensive. So, once it was in | |
3764 place, it was natural to piggy-back all the other conditions on it | |
3765 too and not constantly check them throughout the code. | |
3766 | |
3767 *****************************************************************************/ | |
3768 | |
3769 void CMTask::do_marking_step(double time_target_ms) { | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3770 assert(time_target_ms >= 1.0, "minimum granularity is 1ms"); |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3771 assert(concurrent() == _cm->concurrent(), "they should be the same"); |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3772 |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3773 assert(concurrent() || _cm->region_stack_empty(), |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3774 "the region stack should have been cleared before remark"); |
1835
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
3775 assert(concurrent() || !_cm->has_aborted_regions(), |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
3776 "aborted regions should have been cleared before remark"); |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3777 assert(_region_finger == NULL, |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3778 "this should be non-null only when a region is being scanned"); |
342 | 3779 |
3780 G1CollectorPolicy* g1_policy = _g1h->g1_policy(); | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3781 assert(_task_queues != NULL, "invariant"); |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3782 assert(_task_queue != NULL, "invariant"); |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3783 assert(_task_queues->queue(_task_id) == _task_queue, "invariant"); |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3784 |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3785 assert(!_claimed, |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3786 "only one thread should claim this task at any one time"); |
342 | 3787 |
3788 // OK, this doesn't safeguard again all possible scenarios, as it is | |
3789 // possible for two threads to set the _claimed flag at the same | |
3790 // time. But it is only for debugging purposes anyway and it will | |
3791 // catch most problems. | |
3792 _claimed = true; | |
3793 | |
3794 _start_time_ms = os::elapsedVTime() * 1000.0; | |
3795 statsOnly( _interval_start_time_ms = _start_time_ms ); | |
3796 | |
3797 double diff_prediction_ms = | |
3798 g1_policy->get_new_prediction(&_marking_step_diffs_ms); | |
3799 _time_target_ms = time_target_ms - diff_prediction_ms; | |
3800 | |
3801 // set up the variables that are used in the work-based scheme to | |
3802 // call the regular clock method | |
3803 _words_scanned = 0; | |
3804 _refs_reached = 0; | |
3805 recalculate_limits(); | |
3806 | |
3807 // clear all flags | |
3808 clear_has_aborted(); | |
3809 _has_aborted_timed_out = false; | |
3810 _draining_satb_buffers = false; | |
3811 | |
3812 ++_calls; | |
3813 | |
3814 if (_cm->verbose_low()) | |
3815 gclog_or_tty->print_cr("[%d] >>>>>>>>>> START, call = %d, " | |
3816 "target = %1.2lfms >>>>>>>>>>", | |
3817 _task_id, _calls, _time_target_ms); | |
3818 | |
3819 // Set up the bitmap and oop closures. Anything that uses them is | |
3820 // eventually called from this method, so it is OK to allocate these | |
3821 // statically. | |
3822 CMBitMapClosure bitmap_closure(this, _cm, _nextMarkBitMap); | |
3823 CMOopClosure oop_closure(_g1h, _cm, this); | |
3824 set_oop_closure(&oop_closure); | |
3825 | |
3826 if (_cm->has_overflown()) { | |
3827 // This can happen if the region stack or the mark stack overflows | |
3828 // during a GC pause and this task, after a yield point, | |
3829 // restarts. We have to abort as we need to get into the overflow | |
3830 // protocol which happens right at the end of this task. | |
3831 set_has_aborted(); | |
3832 } | |
3833 | |
3834 // First drain any available SATB buffers. After this, we will not | |
3835 // look at SATB buffers before the next invocation of this method. | |
3836 // If enough completed SATB buffers are queued up, the regular clock | |
3837 // will abort this task so that it restarts. | |
3838 drain_satb_buffers(); | |
3839 // ...then partially drain the local queue and the global stack | |
3840 drain_local_queue(true); | |
3841 drain_global_stack(true); | |
3842 | |
3843 // Then totally drain the region stack. We will not look at | |
3844 // it again before the next invocation of this method. Entries on | |
3845 // the region stack are only added during evacuation pauses, for | |
3846 // which we have to yield. When we do, we abort the task anyway so | |
3847 // it will look at the region stack again when it restarts. | |
3848 bitmap_closure.set_scanning_heap_region(false); | |
3849 drain_region_stack(&bitmap_closure); | |
3850 // ...then partially drain the local queue and the global stack | |
3851 drain_local_queue(true); | |
3852 drain_global_stack(true); | |
3853 | |
3854 do { | |
3855 if (!has_aborted() && _curr_region != NULL) { | |
3856 // This means that we're already holding on to a region. | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3857 assert(_finger != NULL, "if region is not NULL, then the finger " |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3858 "should not be NULL either"); |
342 | 3859 |
3860 // We might have restarted this task after an evacuation pause | |
3861 // which might have evacuated the region we're holding on to | |
3862 // underneath our feet. Let's read its limit again to make sure | |
3863 // that we do not iterate over a region of the heap that | |
3864 // contains garbage (update_region_limit() will also move | |
3865 // _finger to the start of the region if it is found empty). | |
3866 update_region_limit(); | |
3867 // We will start from _finger not from the start of the region, | |
3868 // as we might be restarting this task after aborting half-way | |
3869 // through scanning this region. In this case, _finger points to | |
3870 // the address where we last found a marked object. If this is a | |
3871 // fresh region, _finger points to start(). | |
3872 MemRegion mr = MemRegion(_finger, _region_limit); | |
3873 | |
3874 if (_cm->verbose_low()) | |
3875 gclog_or_tty->print_cr("[%d] we're scanning part " | |
3876 "["PTR_FORMAT", "PTR_FORMAT") " | |
3877 "of region "PTR_FORMAT, | |
3878 _task_id, _finger, _region_limit, _curr_region); | |
3879 | |
3880 // Let's iterate over the bitmap of the part of the | |
3881 // region that is left. | |
3882 bitmap_closure.set_scanning_heap_region(true); | |
3883 if (mr.is_empty() || | |
3884 _nextMarkBitMap->iterate(&bitmap_closure, mr)) { | |
3885 // We successfully completed iterating over the region. Now, | |
3886 // let's give up the region. | |
3887 giveup_current_region(); | |
3888 regular_clock_call(); | |
3889 } else { | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3890 assert(has_aborted(), "currently the only way to do so"); |
342 | 3891 // The only way to abort the bitmap iteration is to return |
3892 // false from the do_bit() method. However, inside the | |
3893 // do_bit() method we move the _finger to point to the | |
3894 // object currently being looked at. So, if we bail out, we | |
3895 // have definitely set _finger to something non-null. | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3896 assert(_finger != NULL, "invariant"); |
342 | 3897 |
3898 // Region iteration was actually aborted. So now _finger | |
3899 // points to the address of the object we last scanned. If we | |
3900 // leave it there, when we restart this task, we will rescan | |
3901 // the object. It is easy to avoid this. We move the finger by | |
3902 // enough to point to the next possible object header (the | |
3903 // bitmap knows by how much we need to move it as it knows its | |
3904 // granularity). | |
1314
3f0549ed0c98
6921710: G1: assert(new_finger >= _finger && new_finger < _region_limit,"invariant")
apetrusenko
parents:
1284
diff
changeset
|
3905 assert(_finger < _region_limit, "invariant"); |
3f0549ed0c98
6921710: G1: assert(new_finger >= _finger && new_finger < _region_limit,"invariant")
apetrusenko
parents:
1284
diff
changeset
|
3906 HeapWord* new_finger = _nextMarkBitMap->nextWord(_finger); |
3f0549ed0c98
6921710: G1: assert(new_finger >= _finger && new_finger < _region_limit,"invariant")
apetrusenko
parents:
1284
diff
changeset
|
3907 // Check if bitmap iteration was aborted while scanning the last object |
3f0549ed0c98
6921710: G1: assert(new_finger >= _finger && new_finger < _region_limit,"invariant")
apetrusenko
parents:
1284
diff
changeset
|
3908 if (new_finger >= _region_limit) { |
3f0549ed0c98
6921710: G1: assert(new_finger >= _finger && new_finger < _region_limit,"invariant")
apetrusenko
parents:
1284
diff
changeset
|
3909 giveup_current_region(); |
3f0549ed0c98
6921710: G1: assert(new_finger >= _finger && new_finger < _region_limit,"invariant")
apetrusenko
parents:
1284
diff
changeset
|
3910 } else { |
3f0549ed0c98
6921710: G1: assert(new_finger >= _finger && new_finger < _region_limit,"invariant")
apetrusenko
parents:
1284
diff
changeset
|
3911 move_finger_to(new_finger); |
3f0549ed0c98
6921710: G1: assert(new_finger >= _finger && new_finger < _region_limit,"invariant")
apetrusenko
parents:
1284
diff
changeset
|
3912 } |
342 | 3913 } |
3914 } | |
3915 // At this point we have either completed iterating over the | |
3916 // region we were holding on to, or we have aborted. | |
3917 | |
3918 // We then partially drain the local queue and the global stack. | |
3919 // (Do we really need this?) | |
3920 drain_local_queue(true); | |
3921 drain_global_stack(true); | |
3922 | |
3923 // Read the note on the claim_region() method on why it might | |
3924 // return NULL with potentially more regions available for | |
3925 // claiming and why we have to check out_of_regions() to determine | |
3926 // whether we're done or not. | |
3927 while (!has_aborted() && _curr_region == NULL && !_cm->out_of_regions()) { | |
3928 // We are going to try to claim a new region. We should have | |
3929 // given up on the previous one. | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3930 // Separated the asserts so that we know which one fires. |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3931 assert(_curr_region == NULL, "invariant"); |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3932 assert(_finger == NULL, "invariant"); |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3933 assert(_region_limit == NULL, "invariant"); |
342 | 3934 if (_cm->verbose_low()) |
3935 gclog_or_tty->print_cr("[%d] trying to claim a new region", _task_id); | |
3936 HeapRegion* claimed_region = _cm->claim_region(_task_id); | |
3937 if (claimed_region != NULL) { | |
3938 // Yes, we managed to claim one | |
3939 statsOnly( ++_regions_claimed ); | |
3940 | |
3941 if (_cm->verbose_low()) | |
3942 gclog_or_tty->print_cr("[%d] we successfully claimed " | |
3943 "region "PTR_FORMAT, | |
3944 _task_id, claimed_region); | |
3945 | |
3946 setup_for_region(claimed_region); | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3947 assert(_curr_region == claimed_region, "invariant"); |
342 | 3948 } |
3949 // It is important to call the regular clock here. It might take | |
3950 // a while to claim a region if, for example, we hit a large | |
3951 // block of empty regions. So we need to call the regular clock | |
3952 // method once round the loop to make sure it's called | |
3953 // frequently enough. | |
3954 regular_clock_call(); | |
3955 } | |
3956 | |
3957 if (!has_aborted() && _curr_region == NULL) { | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3958 assert(_cm->out_of_regions(), |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3959 "at this point we should be out of regions"); |
342 | 3960 } |
3961 } while ( _curr_region != NULL && !has_aborted()); | |
3962 | |
3963 if (!has_aborted()) { | |
3964 // We cannot check whether the global stack is empty, since other | |
343
afc1ce1efe66
6710665: G1: guarantee(_cm->out_of_regions() && _cm->region_stack_empty() && _task_queue->size() == 0, ...)
iveresov
parents:
342
diff
changeset
|
3965 // tasks might be pushing objects to it concurrently. We also cannot |
afc1ce1efe66
6710665: G1: guarantee(_cm->out_of_regions() && _cm->region_stack_empty() && _task_queue->size() == 0, ...)
iveresov
parents:
342
diff
changeset
|
3966 // check if the region stack is empty because if a thread is aborting |
afc1ce1efe66
6710665: G1: guarantee(_cm->out_of_regions() && _cm->region_stack_empty() && _task_queue->size() == 0, ...)
iveresov
parents:
342
diff
changeset
|
3967 // it can push a partially done region back. |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3968 assert(_cm->out_of_regions(), |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3969 "at this point we should be out of regions"); |
342 | 3970 |
3971 if (_cm->verbose_low()) | |
3972 gclog_or_tty->print_cr("[%d] all regions claimed", _task_id); | |
3973 | |
3974 // Try to reduce the number of available SATB buffers so that | |
3975 // remark has less work to do. | |
3976 drain_satb_buffers(); | |
3977 } | |
3978 | |
3979 // Since we've done everything else, we can now totally drain the | |
3980 // local queue and global stack. | |
3981 drain_local_queue(false); | |
3982 drain_global_stack(false); | |
3983 | |
3984 // Attempt at work stealing from other task's queues. | |
3985 if (!has_aborted()) { | |
3986 // We have not aborted. This means that we have finished all that | |
3987 // we could. Let's try to do some stealing... | |
3988 | |
3989 // We cannot check whether the global stack is empty, since other | |
343
afc1ce1efe66
6710665: G1: guarantee(_cm->out_of_regions() && _cm->region_stack_empty() && _task_queue->size() == 0, ...)
iveresov
parents:
342
diff
changeset
|
3990 // tasks might be pushing objects to it concurrently. We also cannot |
afc1ce1efe66
6710665: G1: guarantee(_cm->out_of_regions() && _cm->region_stack_empty() && _task_queue->size() == 0, ...)
iveresov
parents:
342
diff
changeset
|
3991 // check if the region stack is empty because if a thread is aborting |
afc1ce1efe66
6710665: G1: guarantee(_cm->out_of_regions() && _cm->region_stack_empty() && _task_queue->size() == 0, ...)
iveresov
parents:
342
diff
changeset
|
3992 // it can push a partially done region back. |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3993 assert(_cm->out_of_regions() && _task_queue->size() == 0, |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
3994 "only way to reach here"); |
342 | 3995 |
3996 if (_cm->verbose_low()) | |
3997 gclog_or_tty->print_cr("[%d] starting to steal", _task_id); | |
3998 | |
3999 while (!has_aborted()) { | |
4000 oop obj; | |
4001 statsOnly( ++_steal_attempts ); | |
4002 | |
4003 if (_cm->try_stealing(_task_id, &_hash_seed, obj)) { | |
4004 if (_cm->verbose_medium()) | |
4005 gclog_or_tty->print_cr("[%d] stolen "PTR_FORMAT" successfully", | |
4006 _task_id, (void*) obj); | |
4007 | |
4008 statsOnly( ++_steals ); | |
4009 | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
4010 assert(_nextMarkBitMap->isMarked((HeapWord*) obj), |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
4011 "any stolen object should be marked"); |
342 | 4012 scan_object(obj); |
4013 | |
4014 // And since we're towards the end, let's totally drain the | |
4015 // local queue and global stack. | |
4016 drain_local_queue(false); | |
4017 drain_global_stack(false); | |
4018 } else { | |
4019 break; | |
4020 } | |
4021 } | |
4022 } | |
4023 | |
4024 // We still haven't aborted. Now, let's try to get into the | |
4025 // termination protocol. | |
4026 if (!has_aborted()) { | |
4027 // We cannot check whether the global stack is empty, since other | |
343
afc1ce1efe66
6710665: G1: guarantee(_cm->out_of_regions() && _cm->region_stack_empty() && _task_queue->size() == 0, ...)
iveresov
parents:
342
diff
changeset
|
4028 // tasks might be concurrently pushing objects on it. We also cannot |
afc1ce1efe66
6710665: G1: guarantee(_cm->out_of_regions() && _cm->region_stack_empty() && _task_queue->size() == 0, ...)
iveresov
parents:
342
diff
changeset
|
4029 // check if the region stack is empty because if a thread is aborting |
afc1ce1efe66
6710665: G1: guarantee(_cm->out_of_regions() && _cm->region_stack_empty() && _task_queue->size() == 0, ...)
iveresov
parents:
342
diff
changeset
|
4030 // it can push a partially done region back. |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
4031 // Separated the asserts so that we know which one fires. |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
4032 assert(_cm->out_of_regions(), "only way to reach here"); |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
4033 assert(_task_queue->size() == 0, "only way to reach here"); |
342 | 4034 |
4035 if (_cm->verbose_low()) | |
4036 gclog_or_tty->print_cr("[%d] starting termination protocol", _task_id); | |
4037 | |
4038 _termination_start_time_ms = os::elapsedVTime() * 1000.0; | |
4039 // The CMTask class also extends the TerminatorTerminator class, | |
4040 // hence its should_exit_termination() method will also decide | |
4041 // whether to exit the termination protocol or not. | |
4042 bool finished = _cm->terminator()->offer_termination(this); | |
4043 double termination_end_time_ms = os::elapsedVTime() * 1000.0; | |
4044 _termination_time_ms += | |
4045 termination_end_time_ms - _termination_start_time_ms; | |
4046 | |
4047 if (finished) { | |
4048 // We're all done. | |
4049 | |
4050 if (_task_id == 0) { | |
4051 // let's allow task 0 to do this | |
4052 if (concurrent()) { | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
4053 assert(_cm->concurrent_marking_in_progress(), "invariant"); |
342 | 4054 // we need to set this to false before the next |
4055 // safepoint. This way we ensure that the marking phase | |
4056 // doesn't observe any more heap expansions. | |
4057 _cm->clear_concurrent_marking_in_progress(); | |
4058 } | |
4059 } | |
4060 | |
4061 // We can now guarantee that the global stack is empty, since | |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
4062 // all other tasks have finished. We separated the guarantees so |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
4063 // that, if a condition is false, we can immediately find out |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
4064 // which one. |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
4065 guarantee(_cm->out_of_regions(), "only way to reach here"); |
1835
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
4066 guarantee(_aborted_region.is_empty(), "only way to reach here"); |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
4067 guarantee(_cm->region_stack_empty(), "only way to reach here"); |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
4068 guarantee(_cm->mark_stack_empty(), "only way to reach here"); |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
4069 guarantee(_task_queue->size() == 0, "only way to reach here"); |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
4070 guarantee(!_cm->has_overflown(), "only way to reach here"); |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
4071 guarantee(!_cm->mark_stack_overflow(), "only way to reach here"); |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
4072 guarantee(!_cm->region_stack_overflow(), "only way to reach here"); |
342 | 4073 |
4074 if (_cm->verbose_low()) | |
4075 gclog_or_tty->print_cr("[%d] all tasks terminated", _task_id); | |
4076 } else { | |
4077 // Apparently there's more work to do. Let's abort this task. It | |
4078 // will restart it and we can hopefully find more things to do. | |
4079 | |
4080 if (_cm->verbose_low()) | |
4081 gclog_or_tty->print_cr("[%d] apparently there is more work to do", _task_id); | |
4082 | |
4083 set_has_aborted(); | |
4084 statsOnly( ++_aborted_termination ); | |
4085 } | |
4086 } | |
4087 | |
4088 // Mainly for debugging purposes to make sure that a pointer to the | |
4089 // closure which was statically allocated in this frame doesn't | |
4090 // escape it by accident. | |
4091 set_oop_closure(NULL); | |
4092 double end_time_ms = os::elapsedVTime() * 1000.0; | |
4093 double elapsed_time_ms = end_time_ms - _start_time_ms; | |
4094 // Update the step history. | |
4095 _step_times_ms.add(elapsed_time_ms); | |
4096 | |
4097 if (has_aborted()) { | |
4098 // The task was aborted for some reason. | |
4099 | |
4100 statsOnly( ++_aborted ); | |
4101 | |
4102 if (_has_aborted_timed_out) { | |
4103 double diff_ms = elapsed_time_ms - _time_target_ms; | |
4104 // Keep statistics of how well we did with respect to hitting | |
4105 // our target only if we actually timed out (if we aborted for | |
4106 // other reasons, then the results might get skewed). | |
4107 _marking_step_diffs_ms.add(diff_ms); | |
4108 } | |
4109 | |
4110 if (_cm->has_overflown()) { | |
4111 // This is the interesting one. We aborted because a global | |
4112 // overflow was raised. This means we have to restart the | |
4113 // marking phase and start iterating over regions. However, in | |
4114 // order to do this we have to make sure that all tasks stop | |
4115 // what they are doing and re-initialise in a safe manner. We | |
4116 // will achieve this with the use of two barrier sync points. | |
4117 | |
4118 if (_cm->verbose_low()) | |
4119 gclog_or_tty->print_cr("[%d] detected overflow", _task_id); | |
4120 | |
4121 _cm->enter_first_sync_barrier(_task_id); | |
4122 // When we exit this sync barrier we know that all tasks have | |
4123 // stopped doing marking work. So, it's now safe to | |
4124 // re-initialise our data structures. At the end of this method, | |
4125 // task 0 will clear the global data structures. | |
4126 | |
4127 statsOnly( ++_aborted_overflow ); | |
4128 | |
4129 // We clear the local state of this task... | |
4130 clear_region_fields(); | |
4131 | |
4132 // ...and enter the second barrier. | |
4133 _cm->enter_second_sync_barrier(_task_id); | |
4134 // At this point everything has bee re-initialised and we're | |
4135 // ready to restart. | |
4136 } | |
4137 | |
4138 if (_cm->verbose_low()) { | |
4139 gclog_or_tty->print_cr("[%d] <<<<<<<<<< ABORTING, target = %1.2lfms, " | |
4140 "elapsed = %1.2lfms <<<<<<<<<<", | |
4141 _task_id, _time_target_ms, elapsed_time_ms); | |
4142 if (_cm->has_aborted()) | |
4143 gclog_or_tty->print_cr("[%d] ========== MARKING ABORTED ==========", | |
4144 _task_id); | |
4145 } | |
4146 } else { | |
4147 if (_cm->verbose_low()) | |
4148 gclog_or_tty->print_cr("[%d] <<<<<<<<<< FINISHED, target = %1.2lfms, " | |
4149 "elapsed = %1.2lfms <<<<<<<<<<", | |
4150 _task_id, _time_target_ms, elapsed_time_ms); | |
4151 } | |
4152 | |
4153 _claimed = false; | |
4154 } | |
4155 | |
4156 CMTask::CMTask(int task_id, | |
4157 ConcurrentMark* cm, | |
4158 CMTaskQueue* task_queue, | |
4159 CMTaskQueueSet* task_queues) | |
4160 : _g1h(G1CollectedHeap::heap()), | |
4161 _task_id(task_id), _cm(cm), | |
4162 _claimed(false), | |
4163 _nextMarkBitMap(NULL), _hash_seed(17), | |
4164 _task_queue(task_queue), | |
4165 _task_queues(task_queues), | |
1835
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
4166 _oop_closure(NULL), |
4805b9f4779e
6941395: G1: Use only lock-free versions of region stack push() and pop()
johnc
parents:
1833
diff
changeset
|
4167 _aborted_region(MemRegion()) { |
1023
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
4168 guarantee(task_queue != NULL, "invariant"); |
11d4857fe5e1
6888619: G1: too many guarantees in concurrent marking
tonyp
parents:
1022
diff
changeset
|
4169 guarantee(task_queues != NULL, "invariant"); |
342 | 4170 |
4171 statsOnly( _clock_due_to_scanning = 0; | |
4172 _clock_due_to_marking = 0 ); | |
4173 | |
4174 _marking_step_diffs_ms.add(0.5); | |
4175 } |