Mercurial > hg > graal-compiler
annotate src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp @ 4710:41406797186b
7113012: G1: rename not-fully-young GCs as "mixed"
Summary: Renamed partially-young GCs as mixed and fully-young GCs as young. Change all external output that includes those terms (GC log and GC ergo log) as well as any comments, fields, methods, etc. The changeset also includes very minor code tidying up (added some curly brackets).
Reviewed-by: johnc, brutisso
author | tonyp |
---|---|
date | Fri, 16 Dec 2011 02:14:27 -0500 |
parents | 5e5d4821bf07 |
children | 720b6a76dd9d |
rev | line source |
---|---|
342 | 1 /* |
2173 | 2 * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. |
342 | 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
4 * | |
5 * This code is free software; you can redistribute it and/or modify it | |
6 * under the terms of the GNU General Public License version 2 only, as | |
7 * published by the Free Software Foundation. | |
8 * | |
9 * This code is distributed in the hope that it will be useful, but WITHOUT | |
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
12 * version 2 for more details (a copy is included in the LICENSE file that | |
13 * accompanied this code). | |
14 * | |
15 * You should have received a copy of the GNU General Public License version | |
16 * 2 along with this work; if not, write to the Free Software Foundation, | |
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |
18 * | |
1552
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1390
diff
changeset
|
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1390
diff
changeset
|
20 * or visit www.oracle.com if you need additional information or have any |
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1390
diff
changeset
|
21 * questions. |
342 | 22 * |
23 */ | |
24 | |
1972 | 25 #include "precompiled.hpp" |
26 #include "gc_implementation/g1/concurrentG1Refine.hpp" | |
27 #include "gc_implementation/g1/g1BlockOffsetTable.inline.hpp" | |
28 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp" | |
29 #include "gc_implementation/g1/heapRegionRemSet.hpp" | |
30 #include "gc_implementation/g1/heapRegionSeq.inline.hpp" | |
31 #include "memory/allocation.hpp" | |
32 #include "memory/space.inline.hpp" | |
33 #include "utilities/bitMap.inline.hpp" | |
34 #include "utilities/globalDefinitions.hpp" | |
342 | 35 |
36 #define HRRS_VERBOSE 0 | |
37 | |
38 #define PRT_COUNT_OCCUPIED 1 | |
39 | |
40 // OtherRegionsTable | |
41 | |
42 class PerRegionTable: public CHeapObj { | |
43 friend class OtherRegionsTable; | |
44 friend class HeapRegionRemSetIterator; | |
45 | |
46 HeapRegion* _hr; | |
47 BitMap _bm; | |
48 #if PRT_COUNT_OCCUPIED | |
49 jint _occupied; | |
50 #endif | |
51 PerRegionTable* _next_free; | |
52 | |
53 PerRegionTable* next_free() { return _next_free; } | |
54 void set_next_free(PerRegionTable* prt) { _next_free = prt; } | |
55 | |
56 | |
57 static PerRegionTable* _free_list; | |
58 | |
59 #ifdef _MSC_VER | |
60 // For some reason even though the classes are marked as friend they are unable | |
61 // to access CardsPerRegion when private/protected. Only the windows c++ compiler | |
62 // says this Sun CC and linux gcc don't have a problem with access when private | |
63 | |
64 public: | |
65 | |
66 #endif // _MSC_VER | |
67 | |
68 protected: | |
69 // We need access in order to union things into the base table. | |
70 BitMap* bm() { return &_bm; } | |
71 | |
545 | 72 #if PRT_COUNT_OCCUPIED |
342 | 73 void recount_occupied() { |
74 _occupied = (jint) bm()->count_one_bits(); | |
75 } | |
545 | 76 #endif |
342 | 77 |
78 PerRegionTable(HeapRegion* hr) : | |
79 _hr(hr), | |
80 #if PRT_COUNT_OCCUPIED | |
81 _occupied(0), | |
82 #endif | |
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
83 _bm(HeapRegion::CardsPerRegion, false /* in-resource-area */) |
342 | 84 {} |
85 | |
86 static void free(PerRegionTable* prt) { | |
87 while (true) { | |
88 PerRegionTable* fl = _free_list; | |
89 prt->set_next_free(fl); | |
90 PerRegionTable* res = | |
91 (PerRegionTable*) | |
92 Atomic::cmpxchg_ptr(prt, &_free_list, fl); | |
93 if (res == fl) return; | |
94 } | |
95 ShouldNotReachHere(); | |
96 } | |
97 | |
98 static PerRegionTable* alloc(HeapRegion* hr) { | |
99 PerRegionTable* fl = _free_list; | |
100 while (fl != NULL) { | |
101 PerRegionTable* nxt = fl->next_free(); | |
102 PerRegionTable* res = | |
103 (PerRegionTable*) | |
104 Atomic::cmpxchg_ptr(nxt, &_free_list, fl); | |
105 if (res == fl) { | |
106 fl->init(hr); | |
107 return fl; | |
108 } else { | |
109 fl = _free_list; | |
110 } | |
111 } | |
112 assert(fl == NULL, "Loop condition."); | |
113 return new PerRegionTable(hr); | |
114 } | |
115 | |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
116 void add_card_work(CardIdx_t from_card, bool par) { |
342 | 117 if (!_bm.at(from_card)) { |
118 if (par) { | |
119 if (_bm.par_at_put(from_card, 1)) { | |
120 #if PRT_COUNT_OCCUPIED | |
121 Atomic::inc(&_occupied); | |
122 #endif | |
123 } | |
124 } else { | |
125 _bm.at_put(from_card, 1); | |
126 #if PRT_COUNT_OCCUPIED | |
127 _occupied++; | |
128 #endif | |
129 } | |
130 } | |
131 } | |
132 | |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
133 void add_reference_work(OopOrNarrowOopStar from, bool par) { |
342 | 134 // Must make this robust in case "from" is not in "_hr", because of |
135 // concurrency. | |
136 | |
137 #if HRRS_VERBOSE | |
138 gclog_or_tty->print_cr(" PRT::Add_reference_work(" PTR_FORMAT "->" PTR_FORMAT").", | |
139 from, *from); | |
140 #endif | |
141 | |
142 HeapRegion* loc_hr = hr(); | |
143 // If the test below fails, then this table was reused concurrently | |
144 // with this operation. This is OK, since the old table was coarsened, | |
145 // and adding a bit to the new table is never incorrect. | |
4020
5e5d4821bf07
7097516: G1: assert(0<= from_card && from_card<HeapRegion::CardsPerRegion) failed: Must be in range.
brutisso
parents:
3986
diff
changeset
|
146 // If the table used to belong to a continues humongous region and is |
5e5d4821bf07
7097516: G1: assert(0<= from_card && from_card<HeapRegion::CardsPerRegion) failed: Must be in range.
brutisso
parents:
3986
diff
changeset
|
147 // now reused for the corresponding start humongous region, we need to |
5e5d4821bf07
7097516: G1: assert(0<= from_card && from_card<HeapRegion::CardsPerRegion) failed: Must be in range.
brutisso
parents:
3986
diff
changeset
|
148 // make sure that we detect this. Thus, we call is_in_reserved_raw() |
5e5d4821bf07
7097516: G1: assert(0<= from_card && from_card<HeapRegion::CardsPerRegion) failed: Must be in range.
brutisso
parents:
3986
diff
changeset
|
149 // instead of just is_in_reserved() here. |
5e5d4821bf07
7097516: G1: assert(0<= from_card && from_card<HeapRegion::CardsPerRegion) failed: Must be in range.
brutisso
parents:
3986
diff
changeset
|
150 if (loc_hr->is_in_reserved_raw(from)) { |
342 | 151 size_t hw_offset = pointer_delta((HeapWord*)from, loc_hr->bottom()); |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
152 CardIdx_t from_card = (CardIdx_t) |
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
153 hw_offset >> (CardTableModRefBS::card_shift - LogHeapWordSize); |
342 | 154 |
3986
65a8ff39a6da
7095194: G1: HeapRegion::GrainBytes, GrainWords, and CardsPerRegion should be size_t
johnc
parents:
3830
diff
changeset
|
155 assert(0 <= from_card && (size_t)from_card < HeapRegion::CardsPerRegion, |
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
156 "Must be in range."); |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
157 add_card_work(from_card, par); |
342 | 158 } |
159 } | |
160 | |
161 public: | |
162 | |
163 HeapRegion* hr() const { return _hr; } | |
164 | |
165 #if PRT_COUNT_OCCUPIED | |
166 jint occupied() const { | |
167 // Overkill, but if we ever need it... | |
168 // guarantee(_occupied == _bm.count_one_bits(), "Check"); | |
169 return _occupied; | |
170 } | |
171 #else | |
172 jint occupied() const { | |
173 return _bm.count_one_bits(); | |
174 } | |
175 #endif | |
176 | |
177 void init(HeapRegion* hr) { | |
178 _hr = hr; | |
179 #if PRT_COUNT_OCCUPIED | |
180 _occupied = 0; | |
181 #endif | |
182 _bm.clear(); | |
183 } | |
184 | |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
185 void add_reference(OopOrNarrowOopStar from) { |
342 | 186 add_reference_work(from, /*parallel*/ true); |
187 } | |
188 | |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
189 void seq_add_reference(OopOrNarrowOopStar from) { |
342 | 190 add_reference_work(from, /*parallel*/ false); |
191 } | |
192 | |
193 void scrub(CardTableModRefBS* ctbs, BitMap* card_bm) { | |
194 HeapWord* hr_bot = hr()->bottom(); | |
489
2494ab195856
6653214: MemoryPoolMXBean.setUsageThreshold() does not support large heap sizes.
swamyv
parents:
342
diff
changeset
|
195 size_t hr_first_card_index = ctbs->index_for(hr_bot); |
342 | 196 bm()->set_intersection_at_offset(*card_bm, hr_first_card_index); |
197 #if PRT_COUNT_OCCUPIED | |
198 recount_occupied(); | |
199 #endif | |
200 } | |
201 | |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
202 void add_card(CardIdx_t from_card_index) { |
342 | 203 add_card_work(from_card_index, /*parallel*/ true); |
204 } | |
205 | |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
206 void seq_add_card(CardIdx_t from_card_index) { |
342 | 207 add_card_work(from_card_index, /*parallel*/ false); |
208 } | |
209 | |
210 // (Destructively) union the bitmap of the current table into the given | |
211 // bitmap (which is assumed to be of the same size.) | |
212 void union_bitmap_into(BitMap* bm) { | |
213 bm->set_union(_bm); | |
214 } | |
215 | |
216 // Mem size in bytes. | |
217 size_t mem_size() const { | |
218 return sizeof(this) + _bm.size_in_words() * HeapWordSize; | |
219 } | |
220 | |
221 static size_t fl_mem_size() { | |
222 PerRegionTable* cur = _free_list; | |
223 size_t res = 0; | |
224 while (cur != NULL) { | |
225 res += sizeof(PerRegionTable); | |
226 cur = cur->next_free(); | |
227 } | |
228 return res; | |
229 } | |
230 | |
231 // Requires "from" to be in "hr()". | |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
232 bool contains_reference(OopOrNarrowOopStar from) const { |
342 | 233 assert(hr()->is_in_reserved(from), "Precondition."); |
234 size_t card_ind = pointer_delta(from, hr()->bottom(), | |
235 CardTableModRefBS::card_size); | |
236 return _bm.at(card_ind); | |
237 } | |
238 }; | |
239 | |
240 PerRegionTable* PerRegionTable::_free_list = NULL; | |
241 | |
242 | |
243 #define COUNT_PAR_EXPANDS 0 | |
244 | |
245 #if COUNT_PAR_EXPANDS | |
246 static jint n_par_expands = 0; | |
247 static jint n_par_contracts = 0; | |
248 static jint par_expand_list_len = 0; | |
249 static jint max_par_expand_list_len = 0; | |
250 | |
251 static void print_par_expand() { | |
252 Atomic::inc(&n_par_expands); | |
253 Atomic::inc(&par_expand_list_len); | |
254 if (par_expand_list_len > max_par_expand_list_len) { | |
255 max_par_expand_list_len = par_expand_list_len; | |
256 } | |
257 if ((n_par_expands % 10) == 0) { | |
258 gclog_or_tty->print_cr("\n\n%d par expands: %d contracts, " | |
259 "len = %d, max_len = %d\n.", | |
260 n_par_expands, n_par_contracts, par_expand_list_len, | |
261 max_par_expand_list_len); | |
262 } | |
263 } | |
264 #endif | |
265 | |
266 class PosParPRT: public PerRegionTable { | |
267 PerRegionTable** _par_tables; | |
268 | |
269 enum SomePrivateConstants { | |
270 ReserveParTableExpansion = 1 | |
271 }; | |
272 | |
273 void par_contract() { | |
274 assert(_par_tables != NULL, "Precondition."); | |
275 int n = HeapRegionRemSet::num_par_rem_sets()-1; | |
276 for (int i = 0; i < n; i++) { | |
277 _par_tables[i]->union_bitmap_into(bm()); | |
278 PerRegionTable::free(_par_tables[i]); | |
279 _par_tables[i] = NULL; | |
280 } | |
281 #if PRT_COUNT_OCCUPIED | |
282 // We must recount the "occupied." | |
283 recount_occupied(); | |
284 #endif | |
285 FREE_C_HEAP_ARRAY(PerRegionTable*, _par_tables); | |
286 _par_tables = NULL; | |
287 #if COUNT_PAR_EXPANDS | |
288 Atomic::inc(&n_par_contracts); | |
289 Atomic::dec(&par_expand_list_len); | |
290 #endif | |
291 } | |
292 | |
293 static PerRegionTable** _par_table_fl; | |
294 | |
295 PosParPRT* _next; | |
296 | |
297 static PosParPRT* _free_list; | |
298 | |
299 PerRegionTable** par_tables() const { | |
300 assert(uintptr_t(NULL) == 0, "Assumption."); | |
301 if (uintptr_t(_par_tables) <= ReserveParTableExpansion) | |
302 return NULL; | |
303 else | |
304 return _par_tables; | |
305 } | |
306 | |
307 PosParPRT* _next_par_expanded; | |
308 PosParPRT* next_par_expanded() { return _next_par_expanded; } | |
309 void set_next_par_expanded(PosParPRT* ppprt) { _next_par_expanded = ppprt; } | |
310 static PosParPRT* _par_expanded_list; | |
311 | |
312 public: | |
313 | |
314 PosParPRT(HeapRegion* hr) : PerRegionTable(hr), _par_tables(NULL) {} | |
315 | |
316 jint occupied() const { | |
317 jint res = PerRegionTable::occupied(); | |
318 if (par_tables() != NULL) { | |
319 for (int i = 0; i < HeapRegionRemSet::num_par_rem_sets()-1; i++) { | |
320 res += par_tables()[i]->occupied(); | |
321 } | |
322 } | |
323 return res; | |
324 } | |
325 | |
326 void init(HeapRegion* hr) { | |
327 PerRegionTable::init(hr); | |
328 _next = NULL; | |
329 if (par_tables() != NULL) { | |
330 for (int i = 0; i < HeapRegionRemSet::num_par_rem_sets()-1; i++) { | |
331 par_tables()[i]->init(hr); | |
332 } | |
333 } | |
334 } | |
335 | |
336 static void free(PosParPRT* prt) { | |
337 while (true) { | |
338 PosParPRT* fl = _free_list; | |
339 prt->set_next(fl); | |
340 PosParPRT* res = | |
341 (PosParPRT*) | |
342 Atomic::cmpxchg_ptr(prt, &_free_list, fl); | |
343 if (res == fl) return; | |
344 } | |
345 ShouldNotReachHere(); | |
346 } | |
347 | |
348 static PosParPRT* alloc(HeapRegion* hr) { | |
349 PosParPRT* fl = _free_list; | |
350 while (fl != NULL) { | |
351 PosParPRT* nxt = fl->next(); | |
352 PosParPRT* res = | |
353 (PosParPRT*) | |
354 Atomic::cmpxchg_ptr(nxt, &_free_list, fl); | |
355 if (res == fl) { | |
356 fl->init(hr); | |
357 return fl; | |
358 } else { | |
359 fl = _free_list; | |
360 } | |
361 } | |
362 assert(fl == NULL, "Loop condition."); | |
363 return new PosParPRT(hr); | |
364 } | |
365 | |
366 PosParPRT* next() const { return _next; } | |
367 void set_next(PosParPRT* nxt) { _next = nxt; } | |
368 PosParPRT** next_addr() { return &_next; } | |
369 | |
1259
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
370 bool should_expand(int tid) { |
3830
f44782f04dd4
7039627: G1: avoid BOT updates for survivor allocations and dirty survivor regions incrementally
tonyp
parents:
3777
diff
changeset
|
371 // Given that we now defer RSet updates for after a GC we don't |
f44782f04dd4
7039627: G1: avoid BOT updates for survivor allocations and dirty survivor regions incrementally
tonyp
parents:
3777
diff
changeset
|
372 // really need to expand the tables any more. This code should be |
f44782f04dd4
7039627: G1: avoid BOT updates for survivor allocations and dirty survivor regions incrementally
tonyp
parents:
3777
diff
changeset
|
373 // cleaned up in the future (see CR 6921087). |
f44782f04dd4
7039627: G1: avoid BOT updates for survivor allocations and dirty survivor regions incrementally
tonyp
parents:
3777
diff
changeset
|
374 return false; |
1259
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
375 } |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
376 |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
377 void par_expand() { |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
378 int n = HeapRegionRemSet::num_par_rem_sets()-1; |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
379 if (n <= 0) return; |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
380 if (_par_tables == NULL) { |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
381 PerRegionTable* res = |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
382 (PerRegionTable*) |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
383 Atomic::cmpxchg_ptr((PerRegionTable*)ReserveParTableExpansion, |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
384 &_par_tables, NULL); |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
385 if (res != NULL) return; |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
386 // Otherwise, we reserved the right to do the expansion. |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
387 |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
388 PerRegionTable** ptables = NEW_C_HEAP_ARRAY(PerRegionTable*, n); |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
389 for (int i = 0; i < n; i++) { |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
390 PerRegionTable* ptable = PerRegionTable::alloc(hr()); |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
391 ptables[i] = ptable; |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
392 } |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
393 // Here we do not need an atomic. |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
394 _par_tables = ptables; |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
395 #if COUNT_PAR_EXPANDS |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
396 print_par_expand(); |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
397 #endif |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
398 // We must put this table on the expanded list. |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
399 PosParPRT* exp_head = _par_expanded_list; |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
400 while (true) { |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
401 set_next_par_expanded(exp_head); |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
402 PosParPRT* res = |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
403 (PosParPRT*) |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
404 Atomic::cmpxchg_ptr(this, &_par_expanded_list, exp_head); |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
405 if (res == exp_head) return; |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
406 // Otherwise. |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
407 exp_head = res; |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
408 } |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
409 ShouldNotReachHere(); |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
410 } |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
411 } |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
412 |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
413 void add_reference(OopOrNarrowOopStar from, int tid) { |
342 | 414 // Expand if necessary. |
415 PerRegionTable** pt = par_tables(); | |
416 if (pt != NULL) { | |
417 // We always have to assume that mods to table 0 are in parallel, | |
418 // because of the claiming scheme in parallel expansion. A thread | |
419 // with tid != 0 that finds the table to be NULL, but doesn't succeed | |
420 // in claiming the right of expanding it, will end up in the else | |
421 // clause of the above if test. That thread could be delayed, and a | |
422 // thread 0 add reference could see the table expanded, and come | |
423 // here. Both threads would be adding in parallel. But we get to | |
424 // not use atomics for tids > 0. | |
425 if (tid == 0) { | |
426 PerRegionTable::add_reference(from); | |
427 } else { | |
428 pt[tid-1]->seq_add_reference(from); | |
429 } | |
430 } else { | |
431 // Not expanded -- add to the base table. | |
432 PerRegionTable::add_reference(from); | |
433 } | |
434 } | |
435 | |
436 void scrub(CardTableModRefBS* ctbs, BitMap* card_bm) { | |
437 assert(_par_tables == NULL, "Precondition"); | |
438 PerRegionTable::scrub(ctbs, card_bm); | |
439 } | |
440 | |
441 size_t mem_size() const { | |
442 size_t res = | |
443 PerRegionTable::mem_size() + sizeof(this) - sizeof(PerRegionTable); | |
444 if (_par_tables != NULL) { | |
445 for (int i = 0; i < HeapRegionRemSet::num_par_rem_sets()-1; i++) { | |
446 res += _par_tables[i]->mem_size(); | |
447 } | |
448 } | |
449 return res; | |
450 } | |
451 | |
452 static size_t fl_mem_size() { | |
453 PosParPRT* cur = _free_list; | |
454 size_t res = 0; | |
455 while (cur != NULL) { | |
456 res += sizeof(PosParPRT); | |
457 cur = cur->next(); | |
458 } | |
459 return res; | |
460 } | |
461 | |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
462 bool contains_reference(OopOrNarrowOopStar from) const { |
342 | 463 if (PerRegionTable::contains_reference(from)) return true; |
464 if (_par_tables != NULL) { | |
465 for (int i = 0; i < HeapRegionRemSet::num_par_rem_sets()-1; i++) { | |
466 if (_par_tables[i]->contains_reference(from)) return true; | |
467 } | |
468 } | |
469 return false; | |
470 } | |
471 | |
472 static void par_contract_all(); | |
473 }; | |
474 | |
475 void PosParPRT::par_contract_all() { | |
476 PosParPRT* hd = _par_expanded_list; | |
477 while (hd != NULL) { | |
478 PosParPRT* nxt = hd->next_par_expanded(); | |
479 PosParPRT* res = | |
480 (PosParPRT*) | |
481 Atomic::cmpxchg_ptr(nxt, &_par_expanded_list, hd); | |
482 if (res == hd) { | |
483 // We claimed the right to contract this table. | |
484 hd->set_next_par_expanded(NULL); | |
485 hd->par_contract(); | |
486 hd = _par_expanded_list; | |
487 } else { | |
488 hd = res; | |
489 } | |
490 } | |
491 } | |
492 | |
493 PosParPRT* PosParPRT::_free_list = NULL; | |
494 PosParPRT* PosParPRT::_par_expanded_list = NULL; | |
495 | |
496 jint OtherRegionsTable::_cache_probes = 0; | |
497 jint OtherRegionsTable::_cache_hits = 0; | |
498 | |
499 size_t OtherRegionsTable::_max_fine_entries = 0; | |
500 size_t OtherRegionsTable::_mod_max_fine_entries_mask = 0; | |
501 #if SAMPLE_FOR_EVICTION | |
502 size_t OtherRegionsTable::_fine_eviction_stride = 0; | |
503 size_t OtherRegionsTable::_fine_eviction_sample_size = 0; | |
504 #endif | |
505 | |
506 OtherRegionsTable::OtherRegionsTable(HeapRegion* hr) : | |
507 _g1h(G1CollectedHeap::heap()), | |
508 _m(Mutex::leaf, "An OtherRegionsTable lock", true), | |
509 _hr(hr), | |
510 _coarse_map(G1CollectedHeap::heap()->max_regions(), | |
511 false /* in-resource-area */), | |
512 _fine_grain_regions(NULL), | |
513 _n_fine_entries(0), _n_coarse_entries(0), | |
514 #if SAMPLE_FOR_EVICTION | |
515 _fine_eviction_start(0), | |
516 #endif | |
517 _sparse_table(hr) | |
518 { | |
519 typedef PosParPRT* PosParPRTPtr; | |
520 if (_max_fine_entries == 0) { | |
521 assert(_mod_max_fine_entries_mask == 0, "Both or none."); | |
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
522 size_t max_entries_log = (size_t)log2_long((jlong)G1RSetRegionEntries); |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
523 _max_fine_entries = (size_t)(1 << max_entries_log); |
342 | 524 _mod_max_fine_entries_mask = _max_fine_entries - 1; |
525 #if SAMPLE_FOR_EVICTION | |
526 assert(_fine_eviction_sample_size == 0 | |
527 && _fine_eviction_stride == 0, "All init at same time."); | |
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
528 _fine_eviction_sample_size = MAX2((size_t)4, max_entries_log); |
342 | 529 _fine_eviction_stride = _max_fine_entries / _fine_eviction_sample_size; |
530 #endif | |
531 } | |
532 _fine_grain_regions = new PosParPRTPtr[_max_fine_entries]; | |
533 if (_fine_grain_regions == NULL) | |
534 vm_exit_out_of_memory(sizeof(void*)*_max_fine_entries, | |
535 "Failed to allocate _fine_grain_entries."); | |
536 for (size_t i = 0; i < _max_fine_entries; i++) { | |
537 _fine_grain_regions[i] = NULL; | |
538 } | |
539 } | |
540 | |
541 int** OtherRegionsTable::_from_card_cache = NULL; | |
542 size_t OtherRegionsTable::_from_card_cache_max_regions = 0; | |
543 size_t OtherRegionsTable::_from_card_cache_mem_size = 0; | |
544 | |
545 void OtherRegionsTable::init_from_card_cache(size_t max_regions) { | |
546 _from_card_cache_max_regions = max_regions; | |
547 | |
548 int n_par_rs = HeapRegionRemSet::num_par_rem_sets(); | |
549 _from_card_cache = NEW_C_HEAP_ARRAY(int*, n_par_rs); | |
550 for (int i = 0; i < n_par_rs; i++) { | |
551 _from_card_cache[i] = NEW_C_HEAP_ARRAY(int, max_regions); | |
552 for (size_t j = 0; j < max_regions; j++) { | |
553 _from_card_cache[i][j] = -1; // An invalid value. | |
554 } | |
555 } | |
556 _from_card_cache_mem_size = n_par_rs * max_regions * sizeof(int); | |
557 } | |
558 | |
559 void OtherRegionsTable::shrink_from_card_cache(size_t new_n_regs) { | |
560 for (int i = 0; i < HeapRegionRemSet::num_par_rem_sets(); i++) { | |
561 assert(new_n_regs <= _from_card_cache_max_regions, "Must be within max."); | |
562 for (size_t j = new_n_regs; j < _from_card_cache_max_regions; j++) { | |
563 _from_card_cache[i][j] = -1; // An invalid value. | |
564 } | |
565 } | |
566 } | |
567 | |
568 #ifndef PRODUCT | |
569 void OtherRegionsTable::print_from_card_cache() { | |
570 for (int i = 0; i < HeapRegionRemSet::num_par_rem_sets(); i++) { | |
571 for (size_t j = 0; j < _from_card_cache_max_regions; j++) { | |
572 gclog_or_tty->print_cr("_from_card_cache[%d][%d] = %d.", | |
573 i, j, _from_card_cache[i][j]); | |
574 } | |
575 } | |
576 } | |
577 #endif | |
578 | |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
579 void OtherRegionsTable::add_reference(OopOrNarrowOopStar from, int tid) { |
342 | 580 size_t cur_hrs_ind = hr()->hrs_index(); |
581 | |
582 #if HRRS_VERBOSE | |
583 gclog_or_tty->print_cr("ORT::add_reference_work(" PTR_FORMAT "->" PTR_FORMAT ").", | |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
584 from, |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
585 UseCompressedOops |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
586 ? oopDesc::load_decode_heap_oop((narrowOop*)from) |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
587 : oopDesc::load_decode_heap_oop((oop*)from)); |
342 | 588 #endif |
589 | |
590 int from_card = (int)(uintptr_t(from) >> CardTableModRefBS::card_shift); | |
591 | |
592 #if HRRS_VERBOSE | |
593 gclog_or_tty->print_cr("Table for [" PTR_FORMAT "...): card %d (cache = %d)", | |
594 hr()->bottom(), from_card, | |
595 _from_card_cache[tid][cur_hrs_ind]); | |
596 #endif | |
597 | |
598 #define COUNT_CACHE 0 | |
599 #if COUNT_CACHE | |
600 jint p = Atomic::add(1, &_cache_probes); | |
601 if ((p % 10000) == 0) { | |
602 jint hits = _cache_hits; | |
603 gclog_or_tty->print_cr("%d/%d = %5.2f%% RS cache hits.", | |
604 _cache_hits, p, 100.0* (float)hits/(float)p); | |
605 } | |
606 #endif | |
607 if (from_card == _from_card_cache[tid][cur_hrs_ind]) { | |
608 #if HRRS_VERBOSE | |
609 gclog_or_tty->print_cr(" from-card cache hit."); | |
610 #endif | |
611 #if COUNT_CACHE | |
612 Atomic::inc(&_cache_hits); | |
613 #endif | |
614 assert(contains_reference(from), "We just added it!"); | |
615 return; | |
616 } else { | |
617 _from_card_cache[tid][cur_hrs_ind] = from_card; | |
618 } | |
619 | |
620 // Note that this may be a continued H region. | |
621 HeapRegion* from_hr = _g1h->heap_region_containing_raw(from); | |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
622 RegionIdx_t from_hrs_ind = (RegionIdx_t) from_hr->hrs_index(); |
342 | 623 |
624 // If the region is already coarsened, return. | |
625 if (_coarse_map.at(from_hrs_ind)) { | |
626 #if HRRS_VERBOSE | |
627 gclog_or_tty->print_cr(" coarse map hit."); | |
628 #endif | |
629 assert(contains_reference(from), "We just added it!"); | |
630 return; | |
631 } | |
632 | |
633 // Otherwise find a per-region table to add it to. | |
634 size_t ind = from_hrs_ind & _mod_max_fine_entries_mask; | |
635 PosParPRT* prt = find_region_table(ind, from_hr); | |
636 if (prt == NULL) { | |
637 MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag); | |
638 // Confirm that it's really not there... | |
639 prt = find_region_table(ind, from_hr); | |
640 if (prt == NULL) { | |
641 | |
642 uintptr_t from_hr_bot_card_index = | |
643 uintptr_t(from_hr->bottom()) | |
644 >> CardTableModRefBS::card_shift; | |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
645 CardIdx_t card_index = from_card - from_hr_bot_card_index; |
3986
65a8ff39a6da
7095194: G1: HeapRegion::GrainBytes, GrainWords, and CardsPerRegion should be size_t
johnc
parents:
3830
diff
changeset
|
646 assert(0 <= card_index && (size_t)card_index < HeapRegion::CardsPerRegion, |
342 | 647 "Must be in range."); |
648 if (G1HRRSUseSparseTable && | |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
649 _sparse_table.add_card(from_hrs_ind, card_index)) { |
342 | 650 if (G1RecordHRRSOops) { |
651 HeapRegionRemSet::record(hr(), from); | |
652 #if HRRS_VERBOSE | |
653 gclog_or_tty->print(" Added card " PTR_FORMAT " to region " | |
654 "[" PTR_FORMAT "...) for ref " PTR_FORMAT ".\n", | |
655 align_size_down(uintptr_t(from), | |
656 CardTableModRefBS::card_size), | |
657 hr()->bottom(), from); | |
658 #endif | |
659 } | |
660 #if HRRS_VERBOSE | |
661 gclog_or_tty->print_cr(" added card to sparse table."); | |
662 #endif | |
663 assert(contains_reference_locked(from), "We just added it!"); | |
664 return; | |
665 } else { | |
666 #if HRRS_VERBOSE | |
667 gclog_or_tty->print_cr(" [tid %d] sparse table entry " | |
668 "overflow(f: %d, t: %d)", | |
669 tid, from_hrs_ind, cur_hrs_ind); | |
670 #endif | |
671 } | |
672 | |
673 if (_n_fine_entries == _max_fine_entries) { | |
674 prt = delete_region_table(); | |
675 } else { | |
676 prt = PosParPRT::alloc(from_hr); | |
677 } | |
678 prt->init(from_hr); | |
679 | |
680 PosParPRT* first_prt = _fine_grain_regions[ind]; | |
681 prt->set_next(first_prt); // XXX Maybe move to init? | |
682 _fine_grain_regions[ind] = prt; | |
683 _n_fine_entries++; | |
684 | |
685 if (G1HRRSUseSparseTable) { | |
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
686 // Transfer from sparse to fine-grain. |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
687 SparsePRTEntry *sprt_entry = _sparse_table.get_entry(from_hrs_ind); |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
688 assert(sprt_entry != NULL, "There should have been an entry"); |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
689 for (int i = 0; i < SparsePRTEntry::cards_num(); i++) { |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
690 CardIdx_t c = sprt_entry->card(i); |
342 | 691 if (c != SparsePRTEntry::NullEntry) { |
692 prt->add_card(c); | |
693 } | |
694 } | |
695 // Now we can delete the sparse entry. | |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
696 bool res = _sparse_table.delete_entry(from_hrs_ind); |
342 | 697 assert(res, "It should have been there."); |
698 } | |
699 } | |
700 assert(prt != NULL && prt->hr() == from_hr, "consequence"); | |
701 } | |
702 // Note that we can't assert "prt->hr() == from_hr", because of the | |
703 // possibility of concurrent reuse. But see head comment of | |
704 // OtherRegionsTable for why this is OK. | |
705 assert(prt != NULL, "Inv"); | |
706 | |
1259
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
707 if (prt->should_expand(tid)) { |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
708 MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag); |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
709 HeapRegion* prt_hr = prt->hr(); |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
710 if (prt_hr == from_hr) { |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
711 // Make sure the table still corresponds to the same region |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
712 prt->par_expand(); |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
713 prt->add_reference(from, tid); |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
714 } |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
715 // else: The table has been concurrently coarsened, evicted, and |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
716 // the table data structure re-used for another table. So, we |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
717 // don't need to add the reference any more given that the table |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
718 // has been coarsened and the whole region will be scanned anyway. |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
719 } else { |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
720 prt->add_reference(from, tid); |
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
721 } |
342 | 722 if (G1RecordHRRSOops) { |
723 HeapRegionRemSet::record(hr(), from); | |
724 #if HRRS_VERBOSE | |
725 gclog_or_tty->print("Added card " PTR_FORMAT " to region " | |
726 "[" PTR_FORMAT "...) for ref " PTR_FORMAT ".\n", | |
727 align_size_down(uintptr_t(from), | |
728 CardTableModRefBS::card_size), | |
729 hr()->bottom(), from); | |
730 #endif | |
731 } | |
732 assert(contains_reference(from), "We just added it!"); | |
733 } | |
734 | |
735 PosParPRT* | |
736 OtherRegionsTable::find_region_table(size_t ind, HeapRegion* hr) const { | |
737 assert(0 <= ind && ind < _max_fine_entries, "Preconditions."); | |
738 PosParPRT* prt = _fine_grain_regions[ind]; | |
739 while (prt != NULL && prt->hr() != hr) { | |
740 prt = prt->next(); | |
741 } | |
742 // Loop postcondition is the method postcondition. | |
743 return prt; | |
744 } | |
745 | |
746 | |
747 #define DRT_CENSUS 0 | |
748 | |
749 #if DRT_CENSUS | |
750 static const int HistoSize = 6; | |
751 static int global_histo[HistoSize] = { 0, 0, 0, 0, 0, 0 }; | |
752 static int coarsenings = 0; | |
753 static int occ_sum = 0; | |
754 #endif | |
755 | |
756 jint OtherRegionsTable::_n_coarsenings = 0; | |
757 | |
758 PosParPRT* OtherRegionsTable::delete_region_table() { | |
759 #if DRT_CENSUS | |
760 int histo[HistoSize] = { 0, 0, 0, 0, 0, 0 }; | |
761 const int histo_limits[] = { 1, 4, 16, 64, 256, 2048 }; | |
762 #endif | |
763 | |
764 assert(_m.owned_by_self(), "Precondition"); | |
765 assert(_n_fine_entries == _max_fine_entries, "Precondition"); | |
766 PosParPRT* max = NULL; | |
767 jint max_occ = 0; | |
768 PosParPRT** max_prev; | |
769 size_t max_ind; | |
770 | |
771 #if SAMPLE_FOR_EVICTION | |
772 size_t i = _fine_eviction_start; | |
773 for (size_t k = 0; k < _fine_eviction_sample_size; k++) { | |
774 size_t ii = i; | |
775 // Make sure we get a non-NULL sample. | |
776 while (_fine_grain_regions[ii] == NULL) { | |
777 ii++; | |
778 if (ii == _max_fine_entries) ii = 0; | |
779 guarantee(ii != i, "We must find one."); | |
780 } | |
781 PosParPRT** prev = &_fine_grain_regions[ii]; | |
782 PosParPRT* cur = *prev; | |
783 while (cur != NULL) { | |
784 jint cur_occ = cur->occupied(); | |
785 if (max == NULL || cur_occ > max_occ) { | |
786 max = cur; | |
787 max_prev = prev; | |
788 max_ind = i; | |
789 max_occ = cur_occ; | |
790 } | |
791 prev = cur->next_addr(); | |
792 cur = cur->next(); | |
793 } | |
794 i = i + _fine_eviction_stride; | |
795 if (i >= _n_fine_entries) i = i - _n_fine_entries; | |
796 } | |
797 _fine_eviction_start++; | |
798 if (_fine_eviction_start >= _n_fine_entries) | |
799 _fine_eviction_start -= _n_fine_entries; | |
800 #else | |
801 for (int i = 0; i < _max_fine_entries; i++) { | |
802 PosParPRT** prev = &_fine_grain_regions[i]; | |
803 PosParPRT* cur = *prev; | |
804 while (cur != NULL) { | |
805 jint cur_occ = cur->occupied(); | |
806 #if DRT_CENSUS | |
807 for (int k = 0; k < HistoSize; k++) { | |
808 if (cur_occ <= histo_limits[k]) { | |
809 histo[k]++; global_histo[k]++; break; | |
810 } | |
811 } | |
812 #endif | |
813 if (max == NULL || cur_occ > max_occ) { | |
814 max = cur; | |
815 max_prev = prev; | |
816 max_ind = i; | |
817 max_occ = cur_occ; | |
818 } | |
819 prev = cur->next_addr(); | |
820 cur = cur->next(); | |
821 } | |
822 } | |
823 #endif | |
824 // XXX | |
825 guarantee(max != NULL, "Since _n_fine_entries > 0"); | |
826 #if DRT_CENSUS | |
827 gclog_or_tty->print_cr("In a coarsening: histo of occs:"); | |
828 for (int k = 0; k < HistoSize; k++) { | |
829 gclog_or_tty->print_cr(" <= %4d: %5d.", histo_limits[k], histo[k]); | |
830 } | |
831 coarsenings++; | |
832 occ_sum += max_occ; | |
833 if ((coarsenings % 100) == 0) { | |
834 gclog_or_tty->print_cr("\ncoarsenings = %d; global summary:", coarsenings); | |
835 for (int k = 0; k < HistoSize; k++) { | |
836 gclog_or_tty->print_cr(" <= %4d: %5d.", histo_limits[k], global_histo[k]); | |
837 } | |
838 gclog_or_tty->print_cr("Avg occ of deleted region = %6.2f.", | |
839 (float)occ_sum/(float)coarsenings); | |
840 } | |
841 #endif | |
842 | |
843 // Set the corresponding coarse bit. | |
3766 | 844 size_t max_hrs_index = max->hr()->hrs_index(); |
342 | 845 if (!_coarse_map.at(max_hrs_index)) { |
846 _coarse_map.at_put(max_hrs_index, true); | |
847 _n_coarse_entries++; | |
848 #if 0 | |
849 gclog_or_tty->print("Coarsened entry in region [" PTR_FORMAT "...] " | |
850 "for region [" PTR_FORMAT "...] (%d coarse entries).\n", | |
851 hr()->bottom(), | |
852 max->hr()->bottom(), | |
853 _n_coarse_entries); | |
854 #endif | |
855 } | |
856 | |
857 // Unsplice. | |
858 *max_prev = max->next(); | |
859 Atomic::inc(&_n_coarsenings); | |
860 _n_fine_entries--; | |
861 return max; | |
862 } | |
863 | |
864 | |
865 // At present, this must be called stop-world single-threaded. | |
866 void OtherRegionsTable::scrub(CardTableModRefBS* ctbs, | |
867 BitMap* region_bm, BitMap* card_bm) { | |
868 // First eliminated garbage regions from the coarse map. | |
869 if (G1RSScrubVerbose) | |
3766 | 870 gclog_or_tty->print_cr("Scrubbing region "SIZE_FORMAT":", |
871 hr()->hrs_index()); | |
342 | 872 |
873 assert(_coarse_map.size() == region_bm->size(), "Precondition"); | |
874 if (G1RSScrubVerbose) | |
875 gclog_or_tty->print(" Coarse map: before = %d...", _n_coarse_entries); | |
876 _coarse_map.set_intersection(*region_bm); | |
877 _n_coarse_entries = _coarse_map.count_one_bits(); | |
878 if (G1RSScrubVerbose) | |
879 gclog_or_tty->print_cr(" after = %d.", _n_coarse_entries); | |
880 | |
881 // Now do the fine-grained maps. | |
882 for (size_t i = 0; i < _max_fine_entries; i++) { | |
883 PosParPRT* cur = _fine_grain_regions[i]; | |
884 PosParPRT** prev = &_fine_grain_regions[i]; | |
885 while (cur != NULL) { | |
886 PosParPRT* nxt = cur->next(); | |
887 // If the entire region is dead, eliminate. | |
888 if (G1RSScrubVerbose) | |
3766 | 889 gclog_or_tty->print_cr(" For other region "SIZE_FORMAT":", |
890 cur->hr()->hrs_index()); | |
342 | 891 if (!region_bm->at(cur->hr()->hrs_index())) { |
892 *prev = nxt; | |
893 cur->set_next(NULL); | |
894 _n_fine_entries--; | |
895 if (G1RSScrubVerbose) | |
896 gclog_or_tty->print_cr(" deleted via region map."); | |
897 PosParPRT::free(cur); | |
898 } else { | |
899 // Do fine-grain elimination. | |
900 if (G1RSScrubVerbose) | |
901 gclog_or_tty->print(" occ: before = %4d.", cur->occupied()); | |
902 cur->scrub(ctbs, card_bm); | |
903 if (G1RSScrubVerbose) | |
904 gclog_or_tty->print_cr(" after = %4d.", cur->occupied()); | |
905 // Did that empty the table completely? | |
906 if (cur->occupied() == 0) { | |
907 *prev = nxt; | |
908 cur->set_next(NULL); | |
909 _n_fine_entries--; | |
910 PosParPRT::free(cur); | |
911 } else { | |
912 prev = cur->next_addr(); | |
913 } | |
914 } | |
915 cur = nxt; | |
916 } | |
917 } | |
918 // Since we may have deleted a from_card_cache entry from the RS, clear | |
919 // the FCC. | |
920 clear_fcc(); | |
921 } | |
922 | |
923 | |
924 size_t OtherRegionsTable::occupied() const { | |
925 // Cast away const in this case. | |
926 MutexLockerEx x((Mutex*)&_m, Mutex::_no_safepoint_check_flag); | |
927 size_t sum = occ_fine(); | |
928 sum += occ_sparse(); | |
929 sum += occ_coarse(); | |
930 return sum; | |
931 } | |
932 | |
933 size_t OtherRegionsTable::occ_fine() const { | |
934 size_t sum = 0; | |
935 for (size_t i = 0; i < _max_fine_entries; i++) { | |
936 PosParPRT* cur = _fine_grain_regions[i]; | |
937 while (cur != NULL) { | |
938 sum += cur->occupied(); | |
939 cur = cur->next(); | |
940 } | |
941 } | |
942 return sum; | |
943 } | |
944 | |
945 size_t OtherRegionsTable::occ_coarse() const { | |
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
946 return (_n_coarse_entries * HeapRegion::CardsPerRegion); |
342 | 947 } |
948 | |
949 size_t OtherRegionsTable::occ_sparse() const { | |
950 return _sparse_table.occupied(); | |
951 } | |
952 | |
953 size_t OtherRegionsTable::mem_size() const { | |
954 // Cast away const in this case. | |
955 MutexLockerEx x((Mutex*)&_m, Mutex::_no_safepoint_check_flag); | |
956 size_t sum = 0; | |
957 for (size_t i = 0; i < _max_fine_entries; i++) { | |
958 PosParPRT* cur = _fine_grain_regions[i]; | |
959 while (cur != NULL) { | |
960 sum += cur->mem_size(); | |
961 cur = cur->next(); | |
962 } | |
963 } | |
964 sum += (sizeof(PosParPRT*) * _max_fine_entries); | |
965 sum += (_coarse_map.size_in_words() * HeapWordSize); | |
966 sum += (_sparse_table.mem_size()); | |
967 sum += sizeof(*this) - sizeof(_sparse_table); // Avoid double counting above. | |
968 return sum; | |
969 } | |
970 | |
971 size_t OtherRegionsTable::static_mem_size() { | |
972 return _from_card_cache_mem_size; | |
973 } | |
974 | |
975 size_t OtherRegionsTable::fl_mem_size() { | |
976 return PerRegionTable::fl_mem_size() + PosParPRT::fl_mem_size(); | |
977 } | |
978 | |
979 void OtherRegionsTable::clear_fcc() { | |
980 for (int i = 0; i < HeapRegionRemSet::num_par_rem_sets(); i++) { | |
981 _from_card_cache[i][hr()->hrs_index()] = -1; | |
982 } | |
983 } | |
984 | |
985 void OtherRegionsTable::clear() { | |
986 MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag); | |
987 for (size_t i = 0; i < _max_fine_entries; i++) { | |
988 PosParPRT* cur = _fine_grain_regions[i]; | |
989 while (cur != NULL) { | |
990 PosParPRT* nxt = cur->next(); | |
991 PosParPRT::free(cur); | |
992 cur = nxt; | |
993 } | |
994 _fine_grain_regions[i] = NULL; | |
995 } | |
996 _sparse_table.clear(); | |
997 _coarse_map.clear(); | |
998 _n_fine_entries = 0; | |
999 _n_coarse_entries = 0; | |
1000 | |
1001 clear_fcc(); | |
1002 } | |
1003 | |
1004 void OtherRegionsTable::clear_incoming_entry(HeapRegion* from_hr) { | |
1005 MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag); | |
3766 | 1006 size_t hrs_ind = from_hr->hrs_index(); |
342 | 1007 size_t ind = hrs_ind & _mod_max_fine_entries_mask; |
1008 if (del_single_region_table(ind, from_hr)) { | |
1009 assert(!_coarse_map.at(hrs_ind), "Inv"); | |
1010 } else { | |
1011 _coarse_map.par_at_put(hrs_ind, 0); | |
1012 } | |
1013 // Check to see if any of the fcc entries come from here. | |
3766 | 1014 size_t hr_ind = hr()->hrs_index(); |
342 | 1015 for (int tid = 0; tid < HeapRegionRemSet::num_par_rem_sets(); tid++) { |
1016 int fcc_ent = _from_card_cache[tid][hr_ind]; | |
1017 if (fcc_ent != -1) { | |
1018 HeapWord* card_addr = (HeapWord*) | |
1019 (uintptr_t(fcc_ent) << CardTableModRefBS::card_shift); | |
1020 if (hr()->is_in_reserved(card_addr)) { | |
1021 // Clear the from card cache. | |
1022 _from_card_cache[tid][hr_ind] = -1; | |
1023 } | |
1024 } | |
1025 } | |
1026 } | |
1027 | |
1028 bool OtherRegionsTable::del_single_region_table(size_t ind, | |
1029 HeapRegion* hr) { | |
1030 assert(0 <= ind && ind < _max_fine_entries, "Preconditions."); | |
1031 PosParPRT** prev_addr = &_fine_grain_regions[ind]; | |
1032 PosParPRT* prt = *prev_addr; | |
1033 while (prt != NULL && prt->hr() != hr) { | |
1034 prev_addr = prt->next_addr(); | |
1035 prt = prt->next(); | |
1036 } | |
1037 if (prt != NULL) { | |
1038 assert(prt->hr() == hr, "Loop postcondition."); | |
1039 *prev_addr = prt->next(); | |
1040 PosParPRT::free(prt); | |
1041 _n_fine_entries--; | |
1042 return true; | |
1043 } else { | |
1044 return false; | |
1045 } | |
1046 } | |
1047 | |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1048 bool OtherRegionsTable::contains_reference(OopOrNarrowOopStar from) const { |
342 | 1049 // Cast away const in this case. |
1050 MutexLockerEx x((Mutex*)&_m, Mutex::_no_safepoint_check_flag); | |
1051 return contains_reference_locked(from); | |
1052 } | |
1053 | |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1054 bool OtherRegionsTable::contains_reference_locked(OopOrNarrowOopStar from) const { |
342 | 1055 HeapRegion* hr = _g1h->heap_region_containing_raw(from); |
1056 if (hr == NULL) return false; | |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
1057 RegionIdx_t hr_ind = (RegionIdx_t) hr->hrs_index(); |
342 | 1058 // Is this region in the coarse map? |
1059 if (_coarse_map.at(hr_ind)) return true; | |
1060 | |
1061 PosParPRT* prt = find_region_table(hr_ind & _mod_max_fine_entries_mask, | |
1062 hr); | |
1063 if (prt != NULL) { | |
1064 return prt->contains_reference(from); | |
1065 | |
1066 } else { | |
1067 uintptr_t from_card = | |
1068 (uintptr_t(from) >> CardTableModRefBS::card_shift); | |
1069 uintptr_t hr_bot_card_index = | |
1070 uintptr_t(hr->bottom()) >> CardTableModRefBS::card_shift; | |
1071 assert(from_card >= hr_bot_card_index, "Inv"); | |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
1072 CardIdx_t card_index = from_card - hr_bot_card_index; |
3986
65a8ff39a6da
7095194: G1: HeapRegion::GrainBytes, GrainWords, and CardsPerRegion should be size_t
johnc
parents:
3830
diff
changeset
|
1073 assert(0 <= card_index && (size_t)card_index < HeapRegion::CardsPerRegion, |
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
1074 "Must be in range."); |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
1075 return _sparse_table.contains_card(hr_ind, card_index); |
342 | 1076 } |
1077 | |
1078 | |
1079 } | |
1080 | |
2173 | 1081 void |
1082 OtherRegionsTable::do_cleanup_work(HRRSCleanupTask* hrrs_cleanup_task) { | |
1083 _sparse_table.do_cleanup_work(hrrs_cleanup_task); | |
1084 } | |
1085 | |
795
215f81b4d9b3
6841831: G1: assert(contains_reference(from),"We just added it!") fires
iveresov
parents:
794
diff
changeset
|
1086 // Determines how many threads can add records to an rset in parallel. |
215f81b4d9b3
6841831: G1: assert(contains_reference(from),"We just added it!") fires
iveresov
parents:
794
diff
changeset
|
1087 // This can be done by either mutator threads together with the |
215f81b4d9b3
6841831: G1: assert(contains_reference(from),"We just added it!") fires
iveresov
parents:
794
diff
changeset
|
1088 // concurrent refinement threads or GC threads. |
342 | 1089 int HeapRegionRemSet::num_par_rem_sets() { |
795
215f81b4d9b3
6841831: G1: assert(contains_reference(from),"We just added it!") fires
iveresov
parents:
794
diff
changeset
|
1090 return (int)MAX2(DirtyCardQueueSet::num_par_ids() + ConcurrentG1Refine::thread_num(), ParallelGCThreads); |
342 | 1091 } |
1092 | |
1093 HeapRegionRemSet::HeapRegionRemSet(G1BlockOffsetSharedArray* bosa, | |
1094 HeapRegion* hr) | |
3777
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
1095 : _bosa(bosa), _other_regions(hr) { |
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
1096 reset_for_par_iteration(); |
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
1097 } |
342 | 1098 |
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1099 void HeapRegionRemSet::setup_remset_size() { |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1100 // Setup sparse and fine-grain tables sizes. |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1101 // table_size = base * (log(region_size / 1M) + 1) |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1102 int region_size_log_mb = MAX2((int)HeapRegion::LogOfHRGrainBytes - (int)LOG_M, 0); |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1103 if (FLAG_IS_DEFAULT(G1RSetSparseRegionEntries)) { |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1104 G1RSetSparseRegionEntries = G1RSetSparseRegionEntriesBase * (region_size_log_mb + 1); |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1105 } |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1106 if (FLAG_IS_DEFAULT(G1RSetRegionEntries)) { |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1107 G1RSetRegionEntries = G1RSetRegionEntriesBase * (region_size_log_mb + 1); |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1108 } |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1109 guarantee(G1RSetSparseRegionEntries > 0 && G1RSetRegionEntries > 0 , "Sanity"); |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1110 } |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1111 |
342 | 1112 bool HeapRegionRemSet::claim_iter() { |
1113 if (_iter_state != Unclaimed) return false; | |
1114 jint res = Atomic::cmpxchg(Claimed, (jint*)(&_iter_state), Unclaimed); | |
1115 return (res == Unclaimed); | |
1116 } | |
1117 | |
1118 void HeapRegionRemSet::set_iter_complete() { | |
1119 _iter_state = Complete; | |
1120 } | |
1121 | |
1122 bool HeapRegionRemSet::iter_is_complete() { | |
1123 return _iter_state == Complete; | |
1124 } | |
1125 | |
1126 void HeapRegionRemSet::init_iterator(HeapRegionRemSetIterator* iter) const { | |
1127 iter->initialize(this); | |
1128 } | |
1129 | |
1130 #ifndef PRODUCT | |
1131 void HeapRegionRemSet::print() const { | |
1132 HeapRegionRemSetIterator iter; | |
1133 init_iterator(&iter); | |
1134 size_t card_index; | |
1135 while (iter.has_next(card_index)) { | |
1136 HeapWord* card_start = | |
1137 G1CollectedHeap::heap()->bot_shared()->address_for_index(card_index); | |
3777
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
1138 gclog_or_tty->print_cr(" Card " PTR_FORMAT, card_start); |
342 | 1139 } |
1140 // XXX | |
1141 if (iter.n_yielded() != occupied()) { | |
1142 gclog_or_tty->print_cr("Yielded disagrees with occupied:"); | |
1143 gclog_or_tty->print_cr(" %6d yielded (%6d coarse, %6d fine).", | |
1144 iter.n_yielded(), | |
1145 iter.n_yielded_coarse(), iter.n_yielded_fine()); | |
1146 gclog_or_tty->print_cr(" %6d occ (%6d coarse, %6d fine).", | |
1147 occupied(), occ_coarse(), occ_fine()); | |
1148 } | |
1149 guarantee(iter.n_yielded() == occupied(), | |
1150 "We should have yielded all the represented cards."); | |
1151 } | |
1152 #endif | |
1153 | |
1154 void HeapRegionRemSet::cleanup() { | |
1155 SparsePRT::cleanup_all(); | |
1156 } | |
1157 | |
1158 void HeapRegionRemSet::par_cleanup() { | |
1159 PosParPRT::par_contract_all(); | |
1160 } | |
1161 | |
1162 void HeapRegionRemSet::clear() { | |
1163 _other_regions.clear(); | |
1164 assert(occupied() == 0, "Should be clear."); | |
3777
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
1165 reset_for_par_iteration(); |
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
1166 } |
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
1167 |
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
1168 void HeapRegionRemSet::reset_for_par_iteration() { |
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
1169 _iter_state = Unclaimed; |
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
1170 _iter_claimed = 0; |
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
1171 // It's good to check this to make sure that the two methods are in sync. |
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
1172 assert(verify_ready_for_par_iteration(), "post-condition"); |
342 | 1173 } |
1174 | |
1175 void HeapRegionRemSet::scrub(CardTableModRefBS* ctbs, | |
1176 BitMap* region_bm, BitMap* card_bm) { | |
1177 _other_regions.scrub(ctbs, region_bm, card_bm); | |
1178 } | |
1179 | |
1180 //-------------------- Iteration -------------------- | |
1181 | |
1182 HeapRegionRemSetIterator:: | |
1183 HeapRegionRemSetIterator() : | |
1184 _hrrs(NULL), | |
1185 _g1h(G1CollectedHeap::heap()), | |
1186 _bosa(NULL), | |
1884
9f4848ebbabd
6992189: G1: inconsistent base used in sparse rem set iterator
tonyp
parents:
1552
diff
changeset
|
1187 _sparse_iter() { } |
342 | 1188 |
1189 void HeapRegionRemSetIterator::initialize(const HeapRegionRemSet* hrrs) { | |
1190 _hrrs = hrrs; | |
1191 _coarse_map = &_hrrs->_other_regions._coarse_map; | |
1192 _fine_grain_regions = _hrrs->_other_regions._fine_grain_regions; | |
1193 _bosa = _hrrs->bosa(); | |
1194 | |
1195 _is = Sparse; | |
1196 // Set these values so that we increment to the first region. | |
1197 _coarse_cur_region_index = -1; | |
3986
65a8ff39a6da
7095194: G1: HeapRegion::GrainBytes, GrainWords, and CardsPerRegion should be size_t
johnc
parents:
3830
diff
changeset
|
1198 _coarse_cur_region_cur_card = (HeapRegion::CardsPerRegion-1); |
342 | 1199 |
1200 _cur_region_cur_card = 0; | |
1201 | |
1202 _fine_array_index = -1; | |
1203 _fine_cur_prt = NULL; | |
1204 | |
1205 _n_yielded_coarse = 0; | |
1206 _n_yielded_fine = 0; | |
1207 _n_yielded_sparse = 0; | |
1208 | |
1209 _sparse_iter.init(&hrrs->_other_regions._sparse_table); | |
1210 } | |
1211 | |
1212 bool HeapRegionRemSetIterator::coarse_has_next(size_t& card_index) { | |
1213 if (_hrrs->_other_regions._n_coarse_entries == 0) return false; | |
1214 // Go to the next card. | |
1215 _coarse_cur_region_cur_card++; | |
1216 // Was the last the last card in the current region? | |
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
1217 if (_coarse_cur_region_cur_card == HeapRegion::CardsPerRegion) { |
342 | 1218 // Yes: find the next region. This may leave _coarse_cur_region_index |
1219 // Set to the last index, in which case there are no more coarse | |
1220 // regions. | |
1221 _coarse_cur_region_index = | |
1222 (int) _coarse_map->get_next_one_offset(_coarse_cur_region_index + 1); | |
1223 if ((size_t)_coarse_cur_region_index < _coarse_map->size()) { | |
1224 _coarse_cur_region_cur_card = 0; | |
1225 HeapWord* r_bot = | |
1226 _g1h->region_at(_coarse_cur_region_index)->bottom(); | |
1227 _cur_region_card_offset = _bosa->index_for(r_bot); | |
1228 } else { | |
1229 return false; | |
1230 } | |
1231 } | |
1232 // If we didn't return false above, then we can yield a card. | |
1233 card_index = _cur_region_card_offset + _coarse_cur_region_cur_card; | |
1234 return true; | |
1235 } | |
1236 | |
1237 void HeapRegionRemSetIterator::fine_find_next_non_null_prt() { | |
1238 // Otherwise, find the next bucket list in the array. | |
1239 _fine_array_index++; | |
1240 while (_fine_array_index < (int) OtherRegionsTable::_max_fine_entries) { | |
1241 _fine_cur_prt = _fine_grain_regions[_fine_array_index]; | |
1242 if (_fine_cur_prt != NULL) return; | |
1243 else _fine_array_index++; | |
1244 } | |
1245 assert(_fine_cur_prt == NULL, "Loop post"); | |
1246 } | |
1247 | |
1248 bool HeapRegionRemSetIterator::fine_has_next(size_t& card_index) { | |
1249 if (fine_has_next()) { | |
1250 _cur_region_cur_card = | |
1251 _fine_cur_prt->_bm.get_next_one_offset(_cur_region_cur_card + 1); | |
1252 } | |
1253 while (!fine_has_next()) { | |
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
1254 if (_cur_region_cur_card == (size_t) HeapRegion::CardsPerRegion) { |
342 | 1255 _cur_region_cur_card = 0; |
1256 _fine_cur_prt = _fine_cur_prt->next(); | |
1257 } | |
1258 if (_fine_cur_prt == NULL) { | |
1259 fine_find_next_non_null_prt(); | |
1260 if (_fine_cur_prt == NULL) return false; | |
1261 } | |
1262 assert(_fine_cur_prt != NULL && _cur_region_cur_card == 0, | |
1263 "inv."); | |
1264 HeapWord* r_bot = | |
1265 _fine_cur_prt->hr()->bottom(); | |
1266 _cur_region_card_offset = _bosa->index_for(r_bot); | |
1267 _cur_region_cur_card = _fine_cur_prt->_bm.get_next_one_offset(0); | |
1268 } | |
1269 assert(fine_has_next(), "Or else we exited the loop via the return."); | |
1270 card_index = _cur_region_card_offset + _cur_region_cur_card; | |
1271 return true; | |
1272 } | |
1273 | |
1274 bool HeapRegionRemSetIterator::fine_has_next() { | |
1275 return | |
1276 _fine_cur_prt != NULL && | |
3986
65a8ff39a6da
7095194: G1: HeapRegion::GrainBytes, GrainWords, and CardsPerRegion should be size_t
johnc
parents:
3830
diff
changeset
|
1277 _cur_region_cur_card < HeapRegion::CardsPerRegion; |
342 | 1278 } |
1279 | |
1280 bool HeapRegionRemSetIterator::has_next(size_t& card_index) { | |
1281 switch (_is) { | |
1282 case Sparse: | |
1283 if (_sparse_iter.has_next(card_index)) { | |
1284 _n_yielded_sparse++; | |
1285 return true; | |
1286 } | |
1287 // Otherwise, deliberate fall-through | |
1288 _is = Fine; | |
1289 case Fine: | |
1290 if (fine_has_next(card_index)) { | |
1291 _n_yielded_fine++; | |
1292 return true; | |
1293 } | |
1294 // Otherwise, deliberate fall-through | |
1295 _is = Coarse; | |
1296 case Coarse: | |
1297 if (coarse_has_next(card_index)) { | |
1298 _n_yielded_coarse++; | |
1299 return true; | |
1300 } | |
1301 // Otherwise... | |
1302 break; | |
1303 } | |
1304 assert(ParallelGCThreads > 1 || | |
1305 n_yielded() == _hrrs->occupied(), | |
1306 "Should have yielded all the cards in the rem set " | |
1307 "(in the non-par case)."); | |
1308 return false; | |
1309 } | |
1310 | |
1311 | |
1312 | |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1313 OopOrNarrowOopStar* HeapRegionRemSet::_recorded_oops = NULL; |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1314 HeapWord** HeapRegionRemSet::_recorded_cards = NULL; |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1315 HeapRegion** HeapRegionRemSet::_recorded_regions = NULL; |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1316 int HeapRegionRemSet::_n_recorded = 0; |
342 | 1317 |
1318 HeapRegionRemSet::Event* HeapRegionRemSet::_recorded_events = NULL; | |
1319 int* HeapRegionRemSet::_recorded_event_index = NULL; | |
1320 int HeapRegionRemSet::_n_recorded_events = 0; | |
1321 | |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1322 void HeapRegionRemSet::record(HeapRegion* hr, OopOrNarrowOopStar f) { |
342 | 1323 if (_recorded_oops == NULL) { |
1324 assert(_n_recorded == 0 | |
1325 && _recorded_cards == NULL | |
1326 && _recorded_regions == NULL, | |
1327 "Inv"); | |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1328 _recorded_oops = NEW_C_HEAP_ARRAY(OopOrNarrowOopStar, MaxRecorded); |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1329 _recorded_cards = NEW_C_HEAP_ARRAY(HeapWord*, MaxRecorded); |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1330 _recorded_regions = NEW_C_HEAP_ARRAY(HeapRegion*, MaxRecorded); |
342 | 1331 } |
1332 if (_n_recorded == MaxRecorded) { | |
1333 gclog_or_tty->print_cr("Filled up 'recorded' (%d).", MaxRecorded); | |
1334 } else { | |
1335 _recorded_cards[_n_recorded] = | |
1336 (HeapWord*)align_size_down(uintptr_t(f), | |
1337 CardTableModRefBS::card_size); | |
1338 _recorded_oops[_n_recorded] = f; | |
1339 _recorded_regions[_n_recorded] = hr; | |
1340 _n_recorded++; | |
1341 } | |
1342 } | |
1343 | |
1344 void HeapRegionRemSet::record_event(Event evnt) { | |
1345 if (!G1RecordHRRSEvents) return; | |
1346 | |
1347 if (_recorded_events == NULL) { | |
1348 assert(_n_recorded_events == 0 | |
1349 && _recorded_event_index == NULL, | |
1350 "Inv"); | |
1351 _recorded_events = NEW_C_HEAP_ARRAY(Event, MaxRecordedEvents); | |
1352 _recorded_event_index = NEW_C_HEAP_ARRAY(int, MaxRecordedEvents); | |
1353 } | |
1354 if (_n_recorded_events == MaxRecordedEvents) { | |
1355 gclog_or_tty->print_cr("Filled up 'recorded_events' (%d).", MaxRecordedEvents); | |
1356 } else { | |
1357 _recorded_events[_n_recorded_events] = evnt; | |
1358 _recorded_event_index[_n_recorded_events] = _n_recorded; | |
1359 _n_recorded_events++; | |
1360 } | |
1361 } | |
1362 | |
1363 void HeapRegionRemSet::print_event(outputStream* str, Event evnt) { | |
1364 switch (evnt) { | |
1365 case Event_EvacStart: | |
1366 str->print("Evac Start"); | |
1367 break; | |
1368 case Event_EvacEnd: | |
1369 str->print("Evac End"); | |
1370 break; | |
1371 case Event_RSUpdateEnd: | |
1372 str->print("RS Update End"); | |
1373 break; | |
1374 } | |
1375 } | |
1376 | |
1377 void HeapRegionRemSet::print_recorded() { | |
1378 int cur_evnt = 0; | |
1379 Event cur_evnt_kind; | |
1380 int cur_evnt_ind = 0; | |
1381 if (_n_recorded_events > 0) { | |
1382 cur_evnt_kind = _recorded_events[cur_evnt]; | |
1383 cur_evnt_ind = _recorded_event_index[cur_evnt]; | |
1384 } | |
1385 | |
1386 for (int i = 0; i < _n_recorded; i++) { | |
1387 while (cur_evnt < _n_recorded_events && i == cur_evnt_ind) { | |
1388 gclog_or_tty->print("Event: "); | |
1389 print_event(gclog_or_tty, cur_evnt_kind); | |
1390 gclog_or_tty->print_cr(""); | |
1391 cur_evnt++; | |
1392 if (cur_evnt < MaxRecordedEvents) { | |
1393 cur_evnt_kind = _recorded_events[cur_evnt]; | |
1394 cur_evnt_ind = _recorded_event_index[cur_evnt]; | |
1395 } | |
1396 } | |
1397 gclog_or_tty->print("Added card " PTR_FORMAT " to region [" PTR_FORMAT "...]" | |
1398 " for ref " PTR_FORMAT ".\n", | |
1399 _recorded_cards[i], _recorded_regions[i]->bottom(), | |
1400 _recorded_oops[i]); | |
1401 } | |
1402 } | |
1403 | |
2173 | 1404 void HeapRegionRemSet::reset_for_cleanup_tasks() { |
1405 SparsePRT::reset_for_cleanup_tasks(); | |
1406 } | |
1407 | |
1408 void HeapRegionRemSet::do_cleanup_work(HRRSCleanupTask* hrrs_cleanup_task) { | |
1409 _other_regions.do_cleanup_work(hrrs_cleanup_task); | |
1410 } | |
1411 | |
1412 void | |
1413 HeapRegionRemSet::finish_cleanup_task(HRRSCleanupTask* hrrs_cleanup_task) { | |
1414 SparsePRT::finish_cleanup_task(hrrs_cleanup_task); | |
1415 } | |
1416 | |
342 | 1417 #ifndef PRODUCT |
1418 void HeapRegionRemSet::test() { | |
1419 os::sleep(Thread::current(), (jlong)5000, false); | |
1420 G1CollectedHeap* g1h = G1CollectedHeap::heap(); | |
1421 | |
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1422 // Run with "-XX:G1LogRSetRegionEntries=2", so that 1 and 5 end up in same |
342 | 1423 // hash bucket. |
1424 HeapRegion* hr0 = g1h->region_at(0); | |
1425 HeapRegion* hr1 = g1h->region_at(1); | |
1426 HeapRegion* hr2 = g1h->region_at(5); | |
1427 HeapRegion* hr3 = g1h->region_at(6); | |
1428 HeapRegion* hr4 = g1h->region_at(7); | |
1429 HeapRegion* hr5 = g1h->region_at(8); | |
1430 | |
1431 HeapWord* hr1_start = hr1->bottom(); | |
1432 HeapWord* hr1_mid = hr1_start + HeapRegion::GrainWords/2; | |
1433 HeapWord* hr1_last = hr1->end() - 1; | |
1434 | |
1435 HeapWord* hr2_start = hr2->bottom(); | |
1436 HeapWord* hr2_mid = hr2_start + HeapRegion::GrainWords/2; | |
1437 HeapWord* hr2_last = hr2->end() - 1; | |
1438 | |
1439 HeapWord* hr3_start = hr3->bottom(); | |
1440 HeapWord* hr3_mid = hr3_start + HeapRegion::GrainWords/2; | |
1441 HeapWord* hr3_last = hr3->end() - 1; | |
1442 | |
1443 HeapRegionRemSet* hrrs = hr0->rem_set(); | |
1444 | |
1445 // Make three references from region 0x101... | |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1446 hrrs->add_reference((OopOrNarrowOopStar)hr1_start); |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1447 hrrs->add_reference((OopOrNarrowOopStar)hr1_mid); |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1448 hrrs->add_reference((OopOrNarrowOopStar)hr1_last); |
342 | 1449 |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1450 hrrs->add_reference((OopOrNarrowOopStar)hr2_start); |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1451 hrrs->add_reference((OopOrNarrowOopStar)hr2_mid); |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1452 hrrs->add_reference((OopOrNarrowOopStar)hr2_last); |
342 | 1453 |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1454 hrrs->add_reference((OopOrNarrowOopStar)hr3_start); |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1455 hrrs->add_reference((OopOrNarrowOopStar)hr3_mid); |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1456 hrrs->add_reference((OopOrNarrowOopStar)hr3_last); |
342 | 1457 |
1458 // Now cause a coarsening. | |
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1459 hrrs->add_reference((OopOrNarrowOopStar)hr4->bottom()); |
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1460 hrrs->add_reference((OopOrNarrowOopStar)hr5->bottom()); |
342 | 1461 |
1462 // Now, does iteration yield these three? | |
1463 HeapRegionRemSetIterator iter; | |
1464 hrrs->init_iterator(&iter); | |
1465 size_t sum = 0; | |
1466 size_t card_index; | |
1467 while (iter.has_next(card_index)) { | |
1468 HeapWord* card_start = | |
1469 G1CollectedHeap::heap()->bot_shared()->address_for_index(card_index); | |
1470 gclog_or_tty->print_cr(" Card " PTR_FORMAT ".", card_start); | |
1471 sum++; | |
1472 } | |
1473 guarantee(sum == 11 - 3 + 2048, "Failure"); | |
1474 guarantee(sum == hrrs->occupied(), "Failure"); | |
1475 } | |
1476 #endif |