Mercurial > hg > graal-jvmci-8
annotate src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp @ 3777:e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
Summary: Remove two unnecessary iterations over the collection set which are supposed to prepare the RSet's of the CSet regions for parallel iterations (we'll make sure this is done incrementally). I'll piggyback on this CR the removal of the G1_REM_SET_LOGGING code.
Reviewed-by: brutisso, johnc
| author | tonyp |
|---|---|
| date | Tue, 21 Jun 2011 15:23:07 -0400 |
| parents | c3f1170908be |
| children | f44782f04dd4 |
| rev | line source |
|---|---|
| 342 | 1 /* |
| 2173 | 2 * Copyright (c) 2001, 2011, Oracle and/or its affiliates. All rights reserved. |
| 342 | 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| 4 * | |
| 5 * This code is free software; you can redistribute it and/or modify it | |
| 6 * under the terms of the GNU General Public License version 2 only, as | |
| 7 * published by the Free Software Foundation. | |
| 8 * | |
| 9 * This code is distributed in the hope that it will be useful, but WITHOUT | |
| 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
| 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
| 12 * version 2 for more details (a copy is included in the LICENSE file that | |
| 13 * accompanied this code). | |
| 14 * | |
| 15 * You should have received a copy of the GNU General Public License version | |
| 16 * 2 along with this work; if not, write to the Free Software Foundation, | |
| 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |
| 18 * | |
|
1552
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1390
diff
changeset
|
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
|
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1390
diff
changeset
|
20 * or visit www.oracle.com if you need additional information or have any |
|
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1390
diff
changeset
|
21 * questions. |
| 342 | 22 * |
| 23 */ | |
| 24 | |
| 1972 | 25 #include "precompiled.hpp" |
| 26 #include "gc_implementation/g1/concurrentG1Refine.hpp" | |
| 27 #include "gc_implementation/g1/g1BlockOffsetTable.inline.hpp" | |
| 28 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp" | |
| 29 #include "gc_implementation/g1/heapRegionRemSet.hpp" | |
| 30 #include "gc_implementation/g1/heapRegionSeq.inline.hpp" | |
| 31 #include "memory/allocation.hpp" | |
| 32 #include "memory/space.inline.hpp" | |
| 33 #include "utilities/bitMap.inline.hpp" | |
| 34 #include "utilities/globalDefinitions.hpp" | |
| 342 | 35 |
| 36 #define HRRS_VERBOSE 0 | |
| 37 | |
| 38 #define PRT_COUNT_OCCUPIED 1 | |
| 39 | |
| 40 // OtherRegionsTable | |
| 41 | |
| 42 class PerRegionTable: public CHeapObj { | |
| 43 friend class OtherRegionsTable; | |
| 44 friend class HeapRegionRemSetIterator; | |
| 45 | |
| 46 HeapRegion* _hr; | |
| 47 BitMap _bm; | |
| 48 #if PRT_COUNT_OCCUPIED | |
| 49 jint _occupied; | |
| 50 #endif | |
| 51 PerRegionTable* _next_free; | |
| 52 | |
| 53 PerRegionTable* next_free() { return _next_free; } | |
| 54 void set_next_free(PerRegionTable* prt) { _next_free = prt; } | |
| 55 | |
| 56 | |
| 57 static PerRegionTable* _free_list; | |
| 58 | |
| 59 #ifdef _MSC_VER | |
| 60 // For some reason even though the classes are marked as friend they are unable | |
| 61 // to access CardsPerRegion when private/protected. Only the windows c++ compiler | |
| 62 // says this Sun CC and linux gcc don't have a problem with access when private | |
| 63 | |
| 64 public: | |
| 65 | |
| 66 #endif // _MSC_VER | |
| 67 | |
| 68 protected: | |
| 69 // We need access in order to union things into the base table. | |
| 70 BitMap* bm() { return &_bm; } | |
| 71 | |
| 545 | 72 #if PRT_COUNT_OCCUPIED |
| 342 | 73 void recount_occupied() { |
| 74 _occupied = (jint) bm()->count_one_bits(); | |
| 75 } | |
| 545 | 76 #endif |
| 342 | 77 |
| 78 PerRegionTable(HeapRegion* hr) : | |
| 79 _hr(hr), | |
| 80 #if PRT_COUNT_OCCUPIED | |
| 81 _occupied(0), | |
| 82 #endif | |
|
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
83 _bm(HeapRegion::CardsPerRegion, false /* in-resource-area */) |
| 342 | 84 {} |
| 85 | |
| 86 static void free(PerRegionTable* prt) { | |
| 87 while (true) { | |
| 88 PerRegionTable* fl = _free_list; | |
| 89 prt->set_next_free(fl); | |
| 90 PerRegionTable* res = | |
| 91 (PerRegionTable*) | |
| 92 Atomic::cmpxchg_ptr(prt, &_free_list, fl); | |
| 93 if (res == fl) return; | |
| 94 } | |
| 95 ShouldNotReachHere(); | |
| 96 } | |
| 97 | |
| 98 static PerRegionTable* alloc(HeapRegion* hr) { | |
| 99 PerRegionTable* fl = _free_list; | |
| 100 while (fl != NULL) { | |
| 101 PerRegionTable* nxt = fl->next_free(); | |
| 102 PerRegionTable* res = | |
| 103 (PerRegionTable*) | |
| 104 Atomic::cmpxchg_ptr(nxt, &_free_list, fl); | |
| 105 if (res == fl) { | |
| 106 fl->init(hr); | |
| 107 return fl; | |
| 108 } else { | |
| 109 fl = _free_list; | |
| 110 } | |
| 111 } | |
| 112 assert(fl == NULL, "Loop condition."); | |
| 113 return new PerRegionTable(hr); | |
| 114 } | |
| 115 | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
116 void add_card_work(CardIdx_t from_card, bool par) { |
| 342 | 117 if (!_bm.at(from_card)) { |
| 118 if (par) { | |
| 119 if (_bm.par_at_put(from_card, 1)) { | |
| 120 #if PRT_COUNT_OCCUPIED | |
| 121 Atomic::inc(&_occupied); | |
| 122 #endif | |
| 123 } | |
| 124 } else { | |
| 125 _bm.at_put(from_card, 1); | |
| 126 #if PRT_COUNT_OCCUPIED | |
| 127 _occupied++; | |
| 128 #endif | |
| 129 } | |
| 130 } | |
| 131 } | |
| 132 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
133 void add_reference_work(OopOrNarrowOopStar from, bool par) { |
| 342 | 134 // Must make this robust in case "from" is not in "_hr", because of |
| 135 // concurrency. | |
| 136 | |
| 137 #if HRRS_VERBOSE | |
| 138 gclog_or_tty->print_cr(" PRT::Add_reference_work(" PTR_FORMAT "->" PTR_FORMAT").", | |
| 139 from, *from); | |
| 140 #endif | |
| 141 | |
| 142 HeapRegion* loc_hr = hr(); | |
| 143 // If the test below fails, then this table was reused concurrently | |
| 144 // with this operation. This is OK, since the old table was coarsened, | |
| 145 // and adding a bit to the new table is never incorrect. | |
| 146 if (loc_hr->is_in_reserved(from)) { | |
| 147 size_t hw_offset = pointer_delta((HeapWord*)from, loc_hr->bottom()); | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
148 CardIdx_t from_card = (CardIdx_t) |
|
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
149 hw_offset >> (CardTableModRefBS::card_shift - LogHeapWordSize); |
| 342 | 150 |
|
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
151 assert(0 <= from_card && from_card < HeapRegion::CardsPerRegion, |
|
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
152 "Must be in range."); |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
153 add_card_work(from_card, par); |
| 342 | 154 } |
| 155 } | |
| 156 | |
| 157 public: | |
| 158 | |
| 159 HeapRegion* hr() const { return _hr; } | |
| 160 | |
| 161 #if PRT_COUNT_OCCUPIED | |
| 162 jint occupied() const { | |
| 163 // Overkill, but if we ever need it... | |
| 164 // guarantee(_occupied == _bm.count_one_bits(), "Check"); | |
| 165 return _occupied; | |
| 166 } | |
| 167 #else | |
| 168 jint occupied() const { | |
| 169 return _bm.count_one_bits(); | |
| 170 } | |
| 171 #endif | |
| 172 | |
| 173 void init(HeapRegion* hr) { | |
| 174 _hr = hr; | |
| 175 #if PRT_COUNT_OCCUPIED | |
| 176 _occupied = 0; | |
| 177 #endif | |
| 178 _bm.clear(); | |
| 179 } | |
| 180 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
181 void add_reference(OopOrNarrowOopStar from) { |
| 342 | 182 add_reference_work(from, /*parallel*/ true); |
| 183 } | |
| 184 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
185 void seq_add_reference(OopOrNarrowOopStar from) { |
| 342 | 186 add_reference_work(from, /*parallel*/ false); |
| 187 } | |
| 188 | |
| 189 void scrub(CardTableModRefBS* ctbs, BitMap* card_bm) { | |
| 190 HeapWord* hr_bot = hr()->bottom(); | |
|
489
2494ab195856
6653214: MemoryPoolMXBean.setUsageThreshold() does not support large heap sizes.
swamyv
parents:
342
diff
changeset
|
191 size_t hr_first_card_index = ctbs->index_for(hr_bot); |
| 342 | 192 bm()->set_intersection_at_offset(*card_bm, hr_first_card_index); |
| 193 #if PRT_COUNT_OCCUPIED | |
| 194 recount_occupied(); | |
| 195 #endif | |
| 196 } | |
| 197 | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
198 void add_card(CardIdx_t from_card_index) { |
| 342 | 199 add_card_work(from_card_index, /*parallel*/ true); |
| 200 } | |
| 201 | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
202 void seq_add_card(CardIdx_t from_card_index) { |
| 342 | 203 add_card_work(from_card_index, /*parallel*/ false); |
| 204 } | |
| 205 | |
| 206 // (Destructively) union the bitmap of the current table into the given | |
| 207 // bitmap (which is assumed to be of the same size.) | |
| 208 void union_bitmap_into(BitMap* bm) { | |
| 209 bm->set_union(_bm); | |
| 210 } | |
| 211 | |
| 212 // Mem size in bytes. | |
| 213 size_t mem_size() const { | |
| 214 return sizeof(this) + _bm.size_in_words() * HeapWordSize; | |
| 215 } | |
| 216 | |
| 217 static size_t fl_mem_size() { | |
| 218 PerRegionTable* cur = _free_list; | |
| 219 size_t res = 0; | |
| 220 while (cur != NULL) { | |
| 221 res += sizeof(PerRegionTable); | |
| 222 cur = cur->next_free(); | |
| 223 } | |
| 224 return res; | |
| 225 } | |
| 226 | |
| 227 // Requires "from" to be in "hr()". | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
228 bool contains_reference(OopOrNarrowOopStar from) const { |
| 342 | 229 assert(hr()->is_in_reserved(from), "Precondition."); |
| 230 size_t card_ind = pointer_delta(from, hr()->bottom(), | |
| 231 CardTableModRefBS::card_size); | |
| 232 return _bm.at(card_ind); | |
| 233 } | |
| 234 }; | |
| 235 | |
| 236 PerRegionTable* PerRegionTable::_free_list = NULL; | |
| 237 | |
| 238 | |
| 239 #define COUNT_PAR_EXPANDS 0 | |
| 240 | |
| 241 #if COUNT_PAR_EXPANDS | |
| 242 static jint n_par_expands = 0; | |
| 243 static jint n_par_contracts = 0; | |
| 244 static jint par_expand_list_len = 0; | |
| 245 static jint max_par_expand_list_len = 0; | |
| 246 | |
| 247 static void print_par_expand() { | |
| 248 Atomic::inc(&n_par_expands); | |
| 249 Atomic::inc(&par_expand_list_len); | |
| 250 if (par_expand_list_len > max_par_expand_list_len) { | |
| 251 max_par_expand_list_len = par_expand_list_len; | |
| 252 } | |
| 253 if ((n_par_expands % 10) == 0) { | |
| 254 gclog_or_tty->print_cr("\n\n%d par expands: %d contracts, " | |
| 255 "len = %d, max_len = %d\n.", | |
| 256 n_par_expands, n_par_contracts, par_expand_list_len, | |
| 257 max_par_expand_list_len); | |
| 258 } | |
| 259 } | |
| 260 #endif | |
| 261 | |
| 262 class PosParPRT: public PerRegionTable { | |
| 263 PerRegionTable** _par_tables; | |
| 264 | |
| 265 enum SomePrivateConstants { | |
| 266 ReserveParTableExpansion = 1 | |
| 267 }; | |
| 268 | |
| 269 void par_contract() { | |
| 270 assert(_par_tables != NULL, "Precondition."); | |
| 271 int n = HeapRegionRemSet::num_par_rem_sets()-1; | |
| 272 for (int i = 0; i < n; i++) { | |
| 273 _par_tables[i]->union_bitmap_into(bm()); | |
| 274 PerRegionTable::free(_par_tables[i]); | |
| 275 _par_tables[i] = NULL; | |
| 276 } | |
| 277 #if PRT_COUNT_OCCUPIED | |
| 278 // We must recount the "occupied." | |
| 279 recount_occupied(); | |
| 280 #endif | |
| 281 FREE_C_HEAP_ARRAY(PerRegionTable*, _par_tables); | |
| 282 _par_tables = NULL; | |
| 283 #if COUNT_PAR_EXPANDS | |
| 284 Atomic::inc(&n_par_contracts); | |
| 285 Atomic::dec(&par_expand_list_len); | |
| 286 #endif | |
| 287 } | |
| 288 | |
| 289 static PerRegionTable** _par_table_fl; | |
| 290 | |
| 291 PosParPRT* _next; | |
| 292 | |
| 293 static PosParPRT* _free_list; | |
| 294 | |
| 295 PerRegionTable** par_tables() const { | |
| 296 assert(uintptr_t(NULL) == 0, "Assumption."); | |
| 297 if (uintptr_t(_par_tables) <= ReserveParTableExpansion) | |
| 298 return NULL; | |
| 299 else | |
| 300 return _par_tables; | |
| 301 } | |
| 302 | |
| 303 PosParPRT* _next_par_expanded; | |
| 304 PosParPRT* next_par_expanded() { return _next_par_expanded; } | |
| 305 void set_next_par_expanded(PosParPRT* ppprt) { _next_par_expanded = ppprt; } | |
| 306 static PosParPRT* _par_expanded_list; | |
| 307 | |
| 308 public: | |
| 309 | |
| 310 PosParPRT(HeapRegion* hr) : PerRegionTable(hr), _par_tables(NULL) {} | |
| 311 | |
| 312 jint occupied() const { | |
| 313 jint res = PerRegionTable::occupied(); | |
| 314 if (par_tables() != NULL) { | |
| 315 for (int i = 0; i < HeapRegionRemSet::num_par_rem_sets()-1; i++) { | |
| 316 res += par_tables()[i]->occupied(); | |
| 317 } | |
| 318 } | |
| 319 return res; | |
| 320 } | |
| 321 | |
| 322 void init(HeapRegion* hr) { | |
| 323 PerRegionTable::init(hr); | |
| 324 _next = NULL; | |
| 325 if (par_tables() != NULL) { | |
| 326 for (int i = 0; i < HeapRegionRemSet::num_par_rem_sets()-1; i++) { | |
| 327 par_tables()[i]->init(hr); | |
| 328 } | |
| 329 } | |
| 330 } | |
| 331 | |
| 332 static void free(PosParPRT* prt) { | |
| 333 while (true) { | |
| 334 PosParPRT* fl = _free_list; | |
| 335 prt->set_next(fl); | |
| 336 PosParPRT* res = | |
| 337 (PosParPRT*) | |
| 338 Atomic::cmpxchg_ptr(prt, &_free_list, fl); | |
| 339 if (res == fl) return; | |
| 340 } | |
| 341 ShouldNotReachHere(); | |
| 342 } | |
| 343 | |
| 344 static PosParPRT* alloc(HeapRegion* hr) { | |
| 345 PosParPRT* fl = _free_list; | |
| 346 while (fl != NULL) { | |
| 347 PosParPRT* nxt = fl->next(); | |
| 348 PosParPRT* res = | |
| 349 (PosParPRT*) | |
| 350 Atomic::cmpxchg_ptr(nxt, &_free_list, fl); | |
| 351 if (res == fl) { | |
| 352 fl->init(hr); | |
| 353 return fl; | |
| 354 } else { | |
| 355 fl = _free_list; | |
| 356 } | |
| 357 } | |
| 358 assert(fl == NULL, "Loop condition."); | |
| 359 return new PosParPRT(hr); | |
| 360 } | |
| 361 | |
| 362 PosParPRT* next() const { return _next; } | |
| 363 void set_next(PosParPRT* nxt) { _next = nxt; } | |
| 364 PosParPRT** next_addr() { return &_next; } | |
| 365 | |
|
1259
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
366 bool should_expand(int tid) { |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
367 return par_tables() == NULL && tid > 0 && hr()->is_gc_alloc_region(); |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
368 } |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
369 |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
370 void par_expand() { |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
371 int n = HeapRegionRemSet::num_par_rem_sets()-1; |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
372 if (n <= 0) return; |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
373 if (_par_tables == NULL) { |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
374 PerRegionTable* res = |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
375 (PerRegionTable*) |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
376 Atomic::cmpxchg_ptr((PerRegionTable*)ReserveParTableExpansion, |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
377 &_par_tables, NULL); |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
378 if (res != NULL) return; |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
379 // Otherwise, we reserved the right to do the expansion. |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
380 |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
381 PerRegionTable** ptables = NEW_C_HEAP_ARRAY(PerRegionTable*, n); |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
382 for (int i = 0; i < n; i++) { |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
383 PerRegionTable* ptable = PerRegionTable::alloc(hr()); |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
384 ptables[i] = ptable; |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
385 } |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
386 // Here we do not need an atomic. |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
387 _par_tables = ptables; |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
388 #if COUNT_PAR_EXPANDS |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
389 print_par_expand(); |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
390 #endif |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
391 // We must put this table on the expanded list. |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
392 PosParPRT* exp_head = _par_expanded_list; |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
393 while (true) { |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
394 set_next_par_expanded(exp_head); |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
395 PosParPRT* res = |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
396 (PosParPRT*) |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
397 Atomic::cmpxchg_ptr(this, &_par_expanded_list, exp_head); |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
398 if (res == exp_head) return; |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
399 // Otherwise. |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
400 exp_head = res; |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
401 } |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
402 ShouldNotReachHere(); |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
403 } |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
404 } |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
405 |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
406 void add_reference(OopOrNarrowOopStar from, int tid) { |
| 342 | 407 // Expand if necessary. |
| 408 PerRegionTable** pt = par_tables(); | |
| 409 if (pt != NULL) { | |
| 410 // We always have to assume that mods to table 0 are in parallel, | |
| 411 // because of the claiming scheme in parallel expansion. A thread | |
| 412 // with tid != 0 that finds the table to be NULL, but doesn't succeed | |
| 413 // in claiming the right of expanding it, will end up in the else | |
| 414 // clause of the above if test. That thread could be delayed, and a | |
| 415 // thread 0 add reference could see the table expanded, and come | |
| 416 // here. Both threads would be adding in parallel. But we get to | |
| 417 // not use atomics for tids > 0. | |
| 418 if (tid == 0) { | |
| 419 PerRegionTable::add_reference(from); | |
| 420 } else { | |
| 421 pt[tid-1]->seq_add_reference(from); | |
| 422 } | |
| 423 } else { | |
| 424 // Not expanded -- add to the base table. | |
| 425 PerRegionTable::add_reference(from); | |
| 426 } | |
| 427 } | |
| 428 | |
| 429 void scrub(CardTableModRefBS* ctbs, BitMap* card_bm) { | |
| 430 assert(_par_tables == NULL, "Precondition"); | |
| 431 PerRegionTable::scrub(ctbs, card_bm); | |
| 432 } | |
| 433 | |
| 434 size_t mem_size() const { | |
| 435 size_t res = | |
| 436 PerRegionTable::mem_size() + sizeof(this) - sizeof(PerRegionTable); | |
| 437 if (_par_tables != NULL) { | |
| 438 for (int i = 0; i < HeapRegionRemSet::num_par_rem_sets()-1; i++) { | |
| 439 res += _par_tables[i]->mem_size(); | |
| 440 } | |
| 441 } | |
| 442 return res; | |
| 443 } | |
| 444 | |
| 445 static size_t fl_mem_size() { | |
| 446 PosParPRT* cur = _free_list; | |
| 447 size_t res = 0; | |
| 448 while (cur != NULL) { | |
| 449 res += sizeof(PosParPRT); | |
| 450 cur = cur->next(); | |
| 451 } | |
| 452 return res; | |
| 453 } | |
| 454 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
455 bool contains_reference(OopOrNarrowOopStar from) const { |
| 342 | 456 if (PerRegionTable::contains_reference(from)) return true; |
| 457 if (_par_tables != NULL) { | |
| 458 for (int i = 0; i < HeapRegionRemSet::num_par_rem_sets()-1; i++) { | |
| 459 if (_par_tables[i]->contains_reference(from)) return true; | |
| 460 } | |
| 461 } | |
| 462 return false; | |
| 463 } | |
| 464 | |
| 465 static void par_contract_all(); | |
| 466 }; | |
| 467 | |
| 468 void PosParPRT::par_contract_all() { | |
| 469 PosParPRT* hd = _par_expanded_list; | |
| 470 while (hd != NULL) { | |
| 471 PosParPRT* nxt = hd->next_par_expanded(); | |
| 472 PosParPRT* res = | |
| 473 (PosParPRT*) | |
| 474 Atomic::cmpxchg_ptr(nxt, &_par_expanded_list, hd); | |
| 475 if (res == hd) { | |
| 476 // We claimed the right to contract this table. | |
| 477 hd->set_next_par_expanded(NULL); | |
| 478 hd->par_contract(); | |
| 479 hd = _par_expanded_list; | |
| 480 } else { | |
| 481 hd = res; | |
| 482 } | |
| 483 } | |
| 484 } | |
| 485 | |
| 486 PosParPRT* PosParPRT::_free_list = NULL; | |
| 487 PosParPRT* PosParPRT::_par_expanded_list = NULL; | |
| 488 | |
| 489 jint OtherRegionsTable::_cache_probes = 0; | |
| 490 jint OtherRegionsTable::_cache_hits = 0; | |
| 491 | |
| 492 size_t OtherRegionsTable::_max_fine_entries = 0; | |
| 493 size_t OtherRegionsTable::_mod_max_fine_entries_mask = 0; | |
| 494 #if SAMPLE_FOR_EVICTION | |
| 495 size_t OtherRegionsTable::_fine_eviction_stride = 0; | |
| 496 size_t OtherRegionsTable::_fine_eviction_sample_size = 0; | |
| 497 #endif | |
| 498 | |
| 499 OtherRegionsTable::OtherRegionsTable(HeapRegion* hr) : | |
| 500 _g1h(G1CollectedHeap::heap()), | |
| 501 _m(Mutex::leaf, "An OtherRegionsTable lock", true), | |
| 502 _hr(hr), | |
| 503 _coarse_map(G1CollectedHeap::heap()->max_regions(), | |
| 504 false /* in-resource-area */), | |
| 505 _fine_grain_regions(NULL), | |
| 506 _n_fine_entries(0), _n_coarse_entries(0), | |
| 507 #if SAMPLE_FOR_EVICTION | |
| 508 _fine_eviction_start(0), | |
| 509 #endif | |
| 510 _sparse_table(hr) | |
| 511 { | |
| 512 typedef PosParPRT* PosParPRTPtr; | |
| 513 if (_max_fine_entries == 0) { | |
| 514 assert(_mod_max_fine_entries_mask == 0, "Both or none."); | |
|
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
515 size_t max_entries_log = (size_t)log2_long((jlong)G1RSetRegionEntries); |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
516 _max_fine_entries = (size_t)(1 << max_entries_log); |
| 342 | 517 _mod_max_fine_entries_mask = _max_fine_entries - 1; |
| 518 #if SAMPLE_FOR_EVICTION | |
| 519 assert(_fine_eviction_sample_size == 0 | |
| 520 && _fine_eviction_stride == 0, "All init at same time."); | |
|
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
521 _fine_eviction_sample_size = MAX2((size_t)4, max_entries_log); |
| 342 | 522 _fine_eviction_stride = _max_fine_entries / _fine_eviction_sample_size; |
| 523 #endif | |
| 524 } | |
| 525 _fine_grain_regions = new PosParPRTPtr[_max_fine_entries]; | |
| 526 if (_fine_grain_regions == NULL) | |
| 527 vm_exit_out_of_memory(sizeof(void*)*_max_fine_entries, | |
| 528 "Failed to allocate _fine_grain_entries."); | |
| 529 for (size_t i = 0; i < _max_fine_entries; i++) { | |
| 530 _fine_grain_regions[i] = NULL; | |
| 531 } | |
| 532 } | |
| 533 | |
| 534 int** OtherRegionsTable::_from_card_cache = NULL; | |
| 535 size_t OtherRegionsTable::_from_card_cache_max_regions = 0; | |
| 536 size_t OtherRegionsTable::_from_card_cache_mem_size = 0; | |
| 537 | |
| 538 void OtherRegionsTable::init_from_card_cache(size_t max_regions) { | |
| 539 _from_card_cache_max_regions = max_regions; | |
| 540 | |
| 541 int n_par_rs = HeapRegionRemSet::num_par_rem_sets(); | |
| 542 _from_card_cache = NEW_C_HEAP_ARRAY(int*, n_par_rs); | |
| 543 for (int i = 0; i < n_par_rs; i++) { | |
| 544 _from_card_cache[i] = NEW_C_HEAP_ARRAY(int, max_regions); | |
| 545 for (size_t j = 0; j < max_regions; j++) { | |
| 546 _from_card_cache[i][j] = -1; // An invalid value. | |
| 547 } | |
| 548 } | |
| 549 _from_card_cache_mem_size = n_par_rs * max_regions * sizeof(int); | |
| 550 } | |
| 551 | |
| 552 void OtherRegionsTable::shrink_from_card_cache(size_t new_n_regs) { | |
| 553 for (int i = 0; i < HeapRegionRemSet::num_par_rem_sets(); i++) { | |
| 554 assert(new_n_regs <= _from_card_cache_max_regions, "Must be within max."); | |
| 555 for (size_t j = new_n_regs; j < _from_card_cache_max_regions; j++) { | |
| 556 _from_card_cache[i][j] = -1; // An invalid value. | |
| 557 } | |
| 558 } | |
| 559 } | |
| 560 | |
| 561 #ifndef PRODUCT | |
| 562 void OtherRegionsTable::print_from_card_cache() { | |
| 563 for (int i = 0; i < HeapRegionRemSet::num_par_rem_sets(); i++) { | |
| 564 for (size_t j = 0; j < _from_card_cache_max_regions; j++) { | |
| 565 gclog_or_tty->print_cr("_from_card_cache[%d][%d] = %d.", | |
| 566 i, j, _from_card_cache[i][j]); | |
| 567 } | |
| 568 } | |
| 569 } | |
| 570 #endif | |
| 571 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
572 void OtherRegionsTable::add_reference(OopOrNarrowOopStar from, int tid) { |
| 342 | 573 size_t cur_hrs_ind = hr()->hrs_index(); |
| 574 | |
| 575 #if HRRS_VERBOSE | |
| 576 gclog_or_tty->print_cr("ORT::add_reference_work(" PTR_FORMAT "->" PTR_FORMAT ").", | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
577 from, |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
578 UseCompressedOops |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
579 ? oopDesc::load_decode_heap_oop((narrowOop*)from) |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
580 : oopDesc::load_decode_heap_oop((oop*)from)); |
| 342 | 581 #endif |
| 582 | |
| 583 int from_card = (int)(uintptr_t(from) >> CardTableModRefBS::card_shift); | |
| 584 | |
| 585 #if HRRS_VERBOSE | |
| 586 gclog_or_tty->print_cr("Table for [" PTR_FORMAT "...): card %d (cache = %d)", | |
| 587 hr()->bottom(), from_card, | |
| 588 _from_card_cache[tid][cur_hrs_ind]); | |
| 589 #endif | |
| 590 | |
| 591 #define COUNT_CACHE 0 | |
| 592 #if COUNT_CACHE | |
| 593 jint p = Atomic::add(1, &_cache_probes); | |
| 594 if ((p % 10000) == 0) { | |
| 595 jint hits = _cache_hits; | |
| 596 gclog_or_tty->print_cr("%d/%d = %5.2f%% RS cache hits.", | |
| 597 _cache_hits, p, 100.0* (float)hits/(float)p); | |
| 598 } | |
| 599 #endif | |
| 600 if (from_card == _from_card_cache[tid][cur_hrs_ind]) { | |
| 601 #if HRRS_VERBOSE | |
| 602 gclog_or_tty->print_cr(" from-card cache hit."); | |
| 603 #endif | |
| 604 #if COUNT_CACHE | |
| 605 Atomic::inc(&_cache_hits); | |
| 606 #endif | |
| 607 assert(contains_reference(from), "We just added it!"); | |
| 608 return; | |
| 609 } else { | |
| 610 _from_card_cache[tid][cur_hrs_ind] = from_card; | |
| 611 } | |
| 612 | |
| 613 // Note that this may be a continued H region. | |
| 614 HeapRegion* from_hr = _g1h->heap_region_containing_raw(from); | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
615 RegionIdx_t from_hrs_ind = (RegionIdx_t) from_hr->hrs_index(); |
| 342 | 616 |
| 617 // If the region is already coarsened, return. | |
| 618 if (_coarse_map.at(from_hrs_ind)) { | |
| 619 #if HRRS_VERBOSE | |
| 620 gclog_or_tty->print_cr(" coarse map hit."); | |
| 621 #endif | |
| 622 assert(contains_reference(from), "We just added it!"); | |
| 623 return; | |
| 624 } | |
| 625 | |
| 626 // Otherwise find a per-region table to add it to. | |
| 627 size_t ind = from_hrs_ind & _mod_max_fine_entries_mask; | |
| 628 PosParPRT* prt = find_region_table(ind, from_hr); | |
| 629 if (prt == NULL) { | |
| 630 MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag); | |
| 631 // Confirm that it's really not there... | |
| 632 prt = find_region_table(ind, from_hr); | |
| 633 if (prt == NULL) { | |
| 634 | |
| 635 uintptr_t from_hr_bot_card_index = | |
| 636 uintptr_t(from_hr->bottom()) | |
| 637 >> CardTableModRefBS::card_shift; | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
638 CardIdx_t card_index = from_card - from_hr_bot_card_index; |
|
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
639 assert(0 <= card_index && card_index < HeapRegion::CardsPerRegion, |
| 342 | 640 "Must be in range."); |
| 641 if (G1HRRSUseSparseTable && | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
642 _sparse_table.add_card(from_hrs_ind, card_index)) { |
| 342 | 643 if (G1RecordHRRSOops) { |
| 644 HeapRegionRemSet::record(hr(), from); | |
| 645 #if HRRS_VERBOSE | |
| 646 gclog_or_tty->print(" Added card " PTR_FORMAT " to region " | |
| 647 "[" PTR_FORMAT "...) for ref " PTR_FORMAT ".\n", | |
| 648 align_size_down(uintptr_t(from), | |
| 649 CardTableModRefBS::card_size), | |
| 650 hr()->bottom(), from); | |
| 651 #endif | |
| 652 } | |
| 653 #if HRRS_VERBOSE | |
| 654 gclog_or_tty->print_cr(" added card to sparse table."); | |
| 655 #endif | |
| 656 assert(contains_reference_locked(from), "We just added it!"); | |
| 657 return; | |
| 658 } else { | |
| 659 #if HRRS_VERBOSE | |
| 660 gclog_or_tty->print_cr(" [tid %d] sparse table entry " | |
| 661 "overflow(f: %d, t: %d)", | |
| 662 tid, from_hrs_ind, cur_hrs_ind); | |
| 663 #endif | |
| 664 } | |
| 665 | |
| 666 if (_n_fine_entries == _max_fine_entries) { | |
| 667 prt = delete_region_table(); | |
| 668 } else { | |
| 669 prt = PosParPRT::alloc(from_hr); | |
| 670 } | |
| 671 prt->init(from_hr); | |
| 672 | |
| 673 PosParPRT* first_prt = _fine_grain_regions[ind]; | |
| 674 prt->set_next(first_prt); // XXX Maybe move to init? | |
| 675 _fine_grain_regions[ind] = prt; | |
| 676 _n_fine_entries++; | |
| 677 | |
| 678 if (G1HRRSUseSparseTable) { | |
|
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
679 // Transfer from sparse to fine-grain. |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
680 SparsePRTEntry *sprt_entry = _sparse_table.get_entry(from_hrs_ind); |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
681 assert(sprt_entry != NULL, "There should have been an entry"); |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
682 for (int i = 0; i < SparsePRTEntry::cards_num(); i++) { |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
683 CardIdx_t c = sprt_entry->card(i); |
| 342 | 684 if (c != SparsePRTEntry::NullEntry) { |
| 685 prt->add_card(c); | |
| 686 } | |
| 687 } | |
| 688 // Now we can delete the sparse entry. | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
689 bool res = _sparse_table.delete_entry(from_hrs_ind); |
| 342 | 690 assert(res, "It should have been there."); |
| 691 } | |
| 692 } | |
| 693 assert(prt != NULL && prt->hr() == from_hr, "consequence"); | |
| 694 } | |
| 695 // Note that we can't assert "prt->hr() == from_hr", because of the | |
| 696 // possibility of concurrent reuse. But see head comment of | |
| 697 // OtherRegionsTable for why this is OK. | |
| 698 assert(prt != NULL, "Inv"); | |
| 699 | |
|
1259
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
700 if (prt->should_expand(tid)) { |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
701 MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag); |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
702 HeapRegion* prt_hr = prt->hr(); |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
703 if (prt_hr == from_hr) { |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
704 // Make sure the table still corresponds to the same region |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
705 prt->par_expand(); |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
706 prt->add_reference(from, tid); |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
707 } |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
708 // else: The table has been concurrently coarsened, evicted, and |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
709 // the table data structure re-used for another table. So, we |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
710 // don't need to add the reference any more given that the table |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
711 // has been coarsened and the whole region will be scanned anyway. |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
712 } else { |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
713 prt->add_reference(from, tid); |
|
9eee977dd1a9
6802453: G1: hr()->is_in_reserved(from),"Precondition."
tonyp
parents:
942
diff
changeset
|
714 } |
| 342 | 715 if (G1RecordHRRSOops) { |
| 716 HeapRegionRemSet::record(hr(), from); | |
| 717 #if HRRS_VERBOSE | |
| 718 gclog_or_tty->print("Added card " PTR_FORMAT " to region " | |
| 719 "[" PTR_FORMAT "...) for ref " PTR_FORMAT ".\n", | |
| 720 align_size_down(uintptr_t(from), | |
| 721 CardTableModRefBS::card_size), | |
| 722 hr()->bottom(), from); | |
| 723 #endif | |
| 724 } | |
| 725 assert(contains_reference(from), "We just added it!"); | |
| 726 } | |
| 727 | |
| 728 PosParPRT* | |
| 729 OtherRegionsTable::find_region_table(size_t ind, HeapRegion* hr) const { | |
| 730 assert(0 <= ind && ind < _max_fine_entries, "Preconditions."); | |
| 731 PosParPRT* prt = _fine_grain_regions[ind]; | |
| 732 while (prt != NULL && prt->hr() != hr) { | |
| 733 prt = prt->next(); | |
| 734 } | |
| 735 // Loop postcondition is the method postcondition. | |
| 736 return prt; | |
| 737 } | |
| 738 | |
| 739 | |
| 740 #define DRT_CENSUS 0 | |
| 741 | |
| 742 #if DRT_CENSUS | |
| 743 static const int HistoSize = 6; | |
| 744 static int global_histo[HistoSize] = { 0, 0, 0, 0, 0, 0 }; | |
| 745 static int coarsenings = 0; | |
| 746 static int occ_sum = 0; | |
| 747 #endif | |
| 748 | |
| 749 jint OtherRegionsTable::_n_coarsenings = 0; | |
| 750 | |
| 751 PosParPRT* OtherRegionsTable::delete_region_table() { | |
| 752 #if DRT_CENSUS | |
| 753 int histo[HistoSize] = { 0, 0, 0, 0, 0, 0 }; | |
| 754 const int histo_limits[] = { 1, 4, 16, 64, 256, 2048 }; | |
| 755 #endif | |
| 756 | |
| 757 assert(_m.owned_by_self(), "Precondition"); | |
| 758 assert(_n_fine_entries == _max_fine_entries, "Precondition"); | |
| 759 PosParPRT* max = NULL; | |
| 760 jint max_occ = 0; | |
| 761 PosParPRT** max_prev; | |
| 762 size_t max_ind; | |
| 763 | |
| 764 #if SAMPLE_FOR_EVICTION | |
| 765 size_t i = _fine_eviction_start; | |
| 766 for (size_t k = 0; k < _fine_eviction_sample_size; k++) { | |
| 767 size_t ii = i; | |
| 768 // Make sure we get a non-NULL sample. | |
| 769 while (_fine_grain_regions[ii] == NULL) { | |
| 770 ii++; | |
| 771 if (ii == _max_fine_entries) ii = 0; | |
| 772 guarantee(ii != i, "We must find one."); | |
| 773 } | |
| 774 PosParPRT** prev = &_fine_grain_regions[ii]; | |
| 775 PosParPRT* cur = *prev; | |
| 776 while (cur != NULL) { | |
| 777 jint cur_occ = cur->occupied(); | |
| 778 if (max == NULL || cur_occ > max_occ) { | |
| 779 max = cur; | |
| 780 max_prev = prev; | |
| 781 max_ind = i; | |
| 782 max_occ = cur_occ; | |
| 783 } | |
| 784 prev = cur->next_addr(); | |
| 785 cur = cur->next(); | |
| 786 } | |
| 787 i = i + _fine_eviction_stride; | |
| 788 if (i >= _n_fine_entries) i = i - _n_fine_entries; | |
| 789 } | |
| 790 _fine_eviction_start++; | |
| 791 if (_fine_eviction_start >= _n_fine_entries) | |
| 792 _fine_eviction_start -= _n_fine_entries; | |
| 793 #else | |
| 794 for (int i = 0; i < _max_fine_entries; i++) { | |
| 795 PosParPRT** prev = &_fine_grain_regions[i]; | |
| 796 PosParPRT* cur = *prev; | |
| 797 while (cur != NULL) { | |
| 798 jint cur_occ = cur->occupied(); | |
| 799 #if DRT_CENSUS | |
| 800 for (int k = 0; k < HistoSize; k++) { | |
| 801 if (cur_occ <= histo_limits[k]) { | |
| 802 histo[k]++; global_histo[k]++; break; | |
| 803 } | |
| 804 } | |
| 805 #endif | |
| 806 if (max == NULL || cur_occ > max_occ) { | |
| 807 max = cur; | |
| 808 max_prev = prev; | |
| 809 max_ind = i; | |
| 810 max_occ = cur_occ; | |
| 811 } | |
| 812 prev = cur->next_addr(); | |
| 813 cur = cur->next(); | |
| 814 } | |
| 815 } | |
| 816 #endif | |
| 817 // XXX | |
| 818 guarantee(max != NULL, "Since _n_fine_entries > 0"); | |
| 819 #if DRT_CENSUS | |
| 820 gclog_or_tty->print_cr("In a coarsening: histo of occs:"); | |
| 821 for (int k = 0; k < HistoSize; k++) { | |
| 822 gclog_or_tty->print_cr(" <= %4d: %5d.", histo_limits[k], histo[k]); | |
| 823 } | |
| 824 coarsenings++; | |
| 825 occ_sum += max_occ; | |
| 826 if ((coarsenings % 100) == 0) { | |
| 827 gclog_or_tty->print_cr("\ncoarsenings = %d; global summary:", coarsenings); | |
| 828 for (int k = 0; k < HistoSize; k++) { | |
| 829 gclog_or_tty->print_cr(" <= %4d: %5d.", histo_limits[k], global_histo[k]); | |
| 830 } | |
| 831 gclog_or_tty->print_cr("Avg occ of deleted region = %6.2f.", | |
| 832 (float)occ_sum/(float)coarsenings); | |
| 833 } | |
| 834 #endif | |
| 835 | |
| 836 // Set the corresponding coarse bit. | |
| 3766 | 837 size_t max_hrs_index = max->hr()->hrs_index(); |
| 342 | 838 if (!_coarse_map.at(max_hrs_index)) { |
| 839 _coarse_map.at_put(max_hrs_index, true); | |
| 840 _n_coarse_entries++; | |
| 841 #if 0 | |
| 842 gclog_or_tty->print("Coarsened entry in region [" PTR_FORMAT "...] " | |
| 843 "for region [" PTR_FORMAT "...] (%d coarse entries).\n", | |
| 844 hr()->bottom(), | |
| 845 max->hr()->bottom(), | |
| 846 _n_coarse_entries); | |
| 847 #endif | |
| 848 } | |
| 849 | |
| 850 // Unsplice. | |
| 851 *max_prev = max->next(); | |
| 852 Atomic::inc(&_n_coarsenings); | |
| 853 _n_fine_entries--; | |
| 854 return max; | |
| 855 } | |
| 856 | |
| 857 | |
| 858 // At present, this must be called stop-world single-threaded. | |
| 859 void OtherRegionsTable::scrub(CardTableModRefBS* ctbs, | |
| 860 BitMap* region_bm, BitMap* card_bm) { | |
| 861 // First eliminated garbage regions from the coarse map. | |
| 862 if (G1RSScrubVerbose) | |
| 3766 | 863 gclog_or_tty->print_cr("Scrubbing region "SIZE_FORMAT":", |
| 864 hr()->hrs_index()); | |
| 342 | 865 |
| 866 assert(_coarse_map.size() == region_bm->size(), "Precondition"); | |
| 867 if (G1RSScrubVerbose) | |
| 868 gclog_or_tty->print(" Coarse map: before = %d...", _n_coarse_entries); | |
| 869 _coarse_map.set_intersection(*region_bm); | |
| 870 _n_coarse_entries = _coarse_map.count_one_bits(); | |
| 871 if (G1RSScrubVerbose) | |
| 872 gclog_or_tty->print_cr(" after = %d.", _n_coarse_entries); | |
| 873 | |
| 874 // Now do the fine-grained maps. | |
| 875 for (size_t i = 0; i < _max_fine_entries; i++) { | |
| 876 PosParPRT* cur = _fine_grain_regions[i]; | |
| 877 PosParPRT** prev = &_fine_grain_regions[i]; | |
| 878 while (cur != NULL) { | |
| 879 PosParPRT* nxt = cur->next(); | |
| 880 // If the entire region is dead, eliminate. | |
| 881 if (G1RSScrubVerbose) | |
| 3766 | 882 gclog_or_tty->print_cr(" For other region "SIZE_FORMAT":", |
| 883 cur->hr()->hrs_index()); | |
| 342 | 884 if (!region_bm->at(cur->hr()->hrs_index())) { |
| 885 *prev = nxt; | |
| 886 cur->set_next(NULL); | |
| 887 _n_fine_entries--; | |
| 888 if (G1RSScrubVerbose) | |
| 889 gclog_or_tty->print_cr(" deleted via region map."); | |
| 890 PosParPRT::free(cur); | |
| 891 } else { | |
| 892 // Do fine-grain elimination. | |
| 893 if (G1RSScrubVerbose) | |
| 894 gclog_or_tty->print(" occ: before = %4d.", cur->occupied()); | |
| 895 cur->scrub(ctbs, card_bm); | |
| 896 if (G1RSScrubVerbose) | |
| 897 gclog_or_tty->print_cr(" after = %4d.", cur->occupied()); | |
| 898 // Did that empty the table completely? | |
| 899 if (cur->occupied() == 0) { | |
| 900 *prev = nxt; | |
| 901 cur->set_next(NULL); | |
| 902 _n_fine_entries--; | |
| 903 PosParPRT::free(cur); | |
| 904 } else { | |
| 905 prev = cur->next_addr(); | |
| 906 } | |
| 907 } | |
| 908 cur = nxt; | |
| 909 } | |
| 910 } | |
| 911 // Since we may have deleted a from_card_cache entry from the RS, clear | |
| 912 // the FCC. | |
| 913 clear_fcc(); | |
| 914 } | |
| 915 | |
| 916 | |
| 917 size_t OtherRegionsTable::occupied() const { | |
| 918 // Cast away const in this case. | |
| 919 MutexLockerEx x((Mutex*)&_m, Mutex::_no_safepoint_check_flag); | |
| 920 size_t sum = occ_fine(); | |
| 921 sum += occ_sparse(); | |
| 922 sum += occ_coarse(); | |
| 923 return sum; | |
| 924 } | |
| 925 | |
| 926 size_t OtherRegionsTable::occ_fine() const { | |
| 927 size_t sum = 0; | |
| 928 for (size_t i = 0; i < _max_fine_entries; i++) { | |
| 929 PosParPRT* cur = _fine_grain_regions[i]; | |
| 930 while (cur != NULL) { | |
| 931 sum += cur->occupied(); | |
| 932 cur = cur->next(); | |
| 933 } | |
| 934 } | |
| 935 return sum; | |
| 936 } | |
| 937 | |
| 938 size_t OtherRegionsTable::occ_coarse() const { | |
|
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
939 return (_n_coarse_entries * HeapRegion::CardsPerRegion); |
| 342 | 940 } |
| 941 | |
| 942 size_t OtherRegionsTable::occ_sparse() const { | |
| 943 return _sparse_table.occupied(); | |
| 944 } | |
| 945 | |
| 946 size_t OtherRegionsTable::mem_size() const { | |
| 947 // Cast away const in this case. | |
| 948 MutexLockerEx x((Mutex*)&_m, Mutex::_no_safepoint_check_flag); | |
| 949 size_t sum = 0; | |
| 950 for (size_t i = 0; i < _max_fine_entries; i++) { | |
| 951 PosParPRT* cur = _fine_grain_regions[i]; | |
| 952 while (cur != NULL) { | |
| 953 sum += cur->mem_size(); | |
| 954 cur = cur->next(); | |
| 955 } | |
| 956 } | |
| 957 sum += (sizeof(PosParPRT*) * _max_fine_entries); | |
| 958 sum += (_coarse_map.size_in_words() * HeapWordSize); | |
| 959 sum += (_sparse_table.mem_size()); | |
| 960 sum += sizeof(*this) - sizeof(_sparse_table); // Avoid double counting above. | |
| 961 return sum; | |
| 962 } | |
| 963 | |
| 964 size_t OtherRegionsTable::static_mem_size() { | |
| 965 return _from_card_cache_mem_size; | |
| 966 } | |
| 967 | |
| 968 size_t OtherRegionsTable::fl_mem_size() { | |
| 969 return PerRegionTable::fl_mem_size() + PosParPRT::fl_mem_size(); | |
| 970 } | |
| 971 | |
| 972 void OtherRegionsTable::clear_fcc() { | |
| 973 for (int i = 0; i < HeapRegionRemSet::num_par_rem_sets(); i++) { | |
| 974 _from_card_cache[i][hr()->hrs_index()] = -1; | |
| 975 } | |
| 976 } | |
| 977 | |
| 978 void OtherRegionsTable::clear() { | |
| 979 MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag); | |
| 980 for (size_t i = 0; i < _max_fine_entries; i++) { | |
| 981 PosParPRT* cur = _fine_grain_regions[i]; | |
| 982 while (cur != NULL) { | |
| 983 PosParPRT* nxt = cur->next(); | |
| 984 PosParPRT::free(cur); | |
| 985 cur = nxt; | |
| 986 } | |
| 987 _fine_grain_regions[i] = NULL; | |
| 988 } | |
| 989 _sparse_table.clear(); | |
| 990 _coarse_map.clear(); | |
| 991 _n_fine_entries = 0; | |
| 992 _n_coarse_entries = 0; | |
| 993 | |
| 994 clear_fcc(); | |
| 995 } | |
| 996 | |
| 997 void OtherRegionsTable::clear_incoming_entry(HeapRegion* from_hr) { | |
| 998 MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag); | |
| 3766 | 999 size_t hrs_ind = from_hr->hrs_index(); |
| 342 | 1000 size_t ind = hrs_ind & _mod_max_fine_entries_mask; |
| 1001 if (del_single_region_table(ind, from_hr)) { | |
| 1002 assert(!_coarse_map.at(hrs_ind), "Inv"); | |
| 1003 } else { | |
| 1004 _coarse_map.par_at_put(hrs_ind, 0); | |
| 1005 } | |
| 1006 // Check to see if any of the fcc entries come from here. | |
| 3766 | 1007 size_t hr_ind = hr()->hrs_index(); |
| 342 | 1008 for (int tid = 0; tid < HeapRegionRemSet::num_par_rem_sets(); tid++) { |
| 1009 int fcc_ent = _from_card_cache[tid][hr_ind]; | |
| 1010 if (fcc_ent != -1) { | |
| 1011 HeapWord* card_addr = (HeapWord*) | |
| 1012 (uintptr_t(fcc_ent) << CardTableModRefBS::card_shift); | |
| 1013 if (hr()->is_in_reserved(card_addr)) { | |
| 1014 // Clear the from card cache. | |
| 1015 _from_card_cache[tid][hr_ind] = -1; | |
| 1016 } | |
| 1017 } | |
| 1018 } | |
| 1019 } | |
| 1020 | |
| 1021 bool OtherRegionsTable::del_single_region_table(size_t ind, | |
| 1022 HeapRegion* hr) { | |
| 1023 assert(0 <= ind && ind < _max_fine_entries, "Preconditions."); | |
| 1024 PosParPRT** prev_addr = &_fine_grain_regions[ind]; | |
| 1025 PosParPRT* prt = *prev_addr; | |
| 1026 while (prt != NULL && prt->hr() != hr) { | |
| 1027 prev_addr = prt->next_addr(); | |
| 1028 prt = prt->next(); | |
| 1029 } | |
| 1030 if (prt != NULL) { | |
| 1031 assert(prt->hr() == hr, "Loop postcondition."); | |
| 1032 *prev_addr = prt->next(); | |
| 1033 PosParPRT::free(prt); | |
| 1034 _n_fine_entries--; | |
| 1035 return true; | |
| 1036 } else { | |
| 1037 return false; | |
| 1038 } | |
| 1039 } | |
| 1040 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1041 bool OtherRegionsTable::contains_reference(OopOrNarrowOopStar from) const { |
| 342 | 1042 // Cast away const in this case. |
| 1043 MutexLockerEx x((Mutex*)&_m, Mutex::_no_safepoint_check_flag); | |
| 1044 return contains_reference_locked(from); | |
| 1045 } | |
| 1046 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1047 bool OtherRegionsTable::contains_reference_locked(OopOrNarrowOopStar from) const { |
| 342 | 1048 HeapRegion* hr = _g1h->heap_region_containing_raw(from); |
| 1049 if (hr == NULL) return false; | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
1050 RegionIdx_t hr_ind = (RegionIdx_t) hr->hrs_index(); |
| 342 | 1051 // Is this region in the coarse map? |
| 1052 if (_coarse_map.at(hr_ind)) return true; | |
| 1053 | |
| 1054 PosParPRT* prt = find_region_table(hr_ind & _mod_max_fine_entries_mask, | |
| 1055 hr); | |
| 1056 if (prt != NULL) { | |
| 1057 return prt->contains_reference(from); | |
| 1058 | |
| 1059 } else { | |
| 1060 uintptr_t from_card = | |
| 1061 (uintptr_t(from) >> CardTableModRefBS::card_shift); | |
| 1062 uintptr_t hr_bot_card_index = | |
| 1063 uintptr_t(hr->bottom()) >> CardTableModRefBS::card_shift; | |
| 1064 assert(from_card >= hr_bot_card_index, "Inv"); | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
1065 CardIdx_t card_index = from_card - hr_bot_card_index; |
|
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
1066 assert(0 <= card_index && card_index < HeapRegion::CardsPerRegion, |
|
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
1067 "Must be in range."); |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
1068 return _sparse_table.contains_card(hr_ind, card_index); |
| 342 | 1069 } |
| 1070 | |
| 1071 | |
| 1072 } | |
| 1073 | |
| 2173 | 1074 void |
| 1075 OtherRegionsTable::do_cleanup_work(HRRSCleanupTask* hrrs_cleanup_task) { | |
| 1076 _sparse_table.do_cleanup_work(hrrs_cleanup_task); | |
| 1077 } | |
| 1078 | |
|
795
215f81b4d9b3
6841831: G1: assert(contains_reference(from),"We just added it!") fires
iveresov
parents:
794
diff
changeset
|
1079 // Determines how many threads can add records to an rset in parallel. |
|
215f81b4d9b3
6841831: G1: assert(contains_reference(from),"We just added it!") fires
iveresov
parents:
794
diff
changeset
|
1080 // This can be done by either mutator threads together with the |
|
215f81b4d9b3
6841831: G1: assert(contains_reference(from),"We just added it!") fires
iveresov
parents:
794
diff
changeset
|
1081 // concurrent refinement threads or GC threads. |
| 342 | 1082 int HeapRegionRemSet::num_par_rem_sets() { |
|
795
215f81b4d9b3
6841831: G1: assert(contains_reference(from),"We just added it!") fires
iveresov
parents:
794
diff
changeset
|
1083 return (int)MAX2(DirtyCardQueueSet::num_par_ids() + ConcurrentG1Refine::thread_num(), ParallelGCThreads); |
| 342 | 1084 } |
| 1085 | |
| 1086 HeapRegionRemSet::HeapRegionRemSet(G1BlockOffsetSharedArray* bosa, | |
| 1087 HeapRegion* hr) | |
|
3777
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
1088 : _bosa(bosa), _other_regions(hr) { |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
1089 reset_for_par_iteration(); |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
1090 } |
| 342 | 1091 |
|
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1092 void HeapRegionRemSet::setup_remset_size() { |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1093 // Setup sparse and fine-grain tables sizes. |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1094 // table_size = base * (log(region_size / 1M) + 1) |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1095 int region_size_log_mb = MAX2((int)HeapRegion::LogOfHRGrainBytes - (int)LOG_M, 0); |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1096 if (FLAG_IS_DEFAULT(G1RSetSparseRegionEntries)) { |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1097 G1RSetSparseRegionEntries = G1RSetSparseRegionEntriesBase * (region_size_log_mb + 1); |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1098 } |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1099 if (FLAG_IS_DEFAULT(G1RSetRegionEntries)) { |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1100 G1RSetRegionEntries = G1RSetRegionEntriesBase * (region_size_log_mb + 1); |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1101 } |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1102 guarantee(G1RSetSparseRegionEntries > 0 && G1RSetRegionEntries > 0 , "Sanity"); |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1103 } |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1104 |
| 342 | 1105 bool HeapRegionRemSet::claim_iter() { |
| 1106 if (_iter_state != Unclaimed) return false; | |
| 1107 jint res = Atomic::cmpxchg(Claimed, (jint*)(&_iter_state), Unclaimed); | |
| 1108 return (res == Unclaimed); | |
| 1109 } | |
| 1110 | |
| 1111 void HeapRegionRemSet::set_iter_complete() { | |
| 1112 _iter_state = Complete; | |
| 1113 } | |
| 1114 | |
| 1115 bool HeapRegionRemSet::iter_is_complete() { | |
| 1116 return _iter_state == Complete; | |
| 1117 } | |
| 1118 | |
| 1119 void HeapRegionRemSet::init_iterator(HeapRegionRemSetIterator* iter) const { | |
| 1120 iter->initialize(this); | |
| 1121 } | |
| 1122 | |
| 1123 #ifndef PRODUCT | |
| 1124 void HeapRegionRemSet::print() const { | |
| 1125 HeapRegionRemSetIterator iter; | |
| 1126 init_iterator(&iter); | |
| 1127 size_t card_index; | |
| 1128 while (iter.has_next(card_index)) { | |
| 1129 HeapWord* card_start = | |
| 1130 G1CollectedHeap::heap()->bot_shared()->address_for_index(card_index); | |
|
3777
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
1131 gclog_or_tty->print_cr(" Card " PTR_FORMAT, card_start); |
| 342 | 1132 } |
| 1133 // XXX | |
| 1134 if (iter.n_yielded() != occupied()) { | |
| 1135 gclog_or_tty->print_cr("Yielded disagrees with occupied:"); | |
| 1136 gclog_or_tty->print_cr(" %6d yielded (%6d coarse, %6d fine).", | |
| 1137 iter.n_yielded(), | |
| 1138 iter.n_yielded_coarse(), iter.n_yielded_fine()); | |
| 1139 gclog_or_tty->print_cr(" %6d occ (%6d coarse, %6d fine).", | |
| 1140 occupied(), occ_coarse(), occ_fine()); | |
| 1141 } | |
| 1142 guarantee(iter.n_yielded() == occupied(), | |
| 1143 "We should have yielded all the represented cards."); | |
| 1144 } | |
| 1145 #endif | |
| 1146 | |
| 1147 void HeapRegionRemSet::cleanup() { | |
| 1148 SparsePRT::cleanup_all(); | |
| 1149 } | |
| 1150 | |
| 1151 void HeapRegionRemSet::par_cleanup() { | |
| 1152 PosParPRT::par_contract_all(); | |
| 1153 } | |
| 1154 | |
| 1155 void HeapRegionRemSet::clear() { | |
| 1156 _other_regions.clear(); | |
| 1157 assert(occupied() == 0, "Should be clear."); | |
|
3777
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
1158 reset_for_par_iteration(); |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
1159 } |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
1160 |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
1161 void HeapRegionRemSet::reset_for_par_iteration() { |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
1162 _iter_state = Unclaimed; |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
1163 _iter_claimed = 0; |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
1164 // It's good to check this to make sure that the two methods are in sync. |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
1165 assert(verify_ready_for_par_iteration(), "post-condition"); |
| 342 | 1166 } |
| 1167 | |
| 1168 void HeapRegionRemSet::scrub(CardTableModRefBS* ctbs, | |
| 1169 BitMap* region_bm, BitMap* card_bm) { | |
| 1170 _other_regions.scrub(ctbs, region_bm, card_bm); | |
| 1171 } | |
| 1172 | |
| 1173 //-------------------- Iteration -------------------- | |
| 1174 | |
| 1175 HeapRegionRemSetIterator:: | |
| 1176 HeapRegionRemSetIterator() : | |
| 1177 _hrrs(NULL), | |
| 1178 _g1h(G1CollectedHeap::heap()), | |
| 1179 _bosa(NULL), | |
|
1884
9f4848ebbabd
6992189: G1: inconsistent base used in sparse rem set iterator
tonyp
parents:
1552
diff
changeset
|
1180 _sparse_iter() { } |
| 342 | 1181 |
| 1182 void HeapRegionRemSetIterator::initialize(const HeapRegionRemSet* hrrs) { | |
| 1183 _hrrs = hrrs; | |
| 1184 _coarse_map = &_hrrs->_other_regions._coarse_map; | |
| 1185 _fine_grain_regions = _hrrs->_other_regions._fine_grain_regions; | |
| 1186 _bosa = _hrrs->bosa(); | |
| 1187 | |
| 1188 _is = Sparse; | |
| 1189 // Set these values so that we increment to the first region. | |
| 1190 _coarse_cur_region_index = -1; | |
|
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
1191 _coarse_cur_region_cur_card = (HeapRegion::CardsPerRegion-1);; |
| 342 | 1192 |
| 1193 _cur_region_cur_card = 0; | |
| 1194 | |
| 1195 _fine_array_index = -1; | |
| 1196 _fine_cur_prt = NULL; | |
| 1197 | |
| 1198 _n_yielded_coarse = 0; | |
| 1199 _n_yielded_fine = 0; | |
| 1200 _n_yielded_sparse = 0; | |
| 1201 | |
| 1202 _sparse_iter.init(&hrrs->_other_regions._sparse_table); | |
| 1203 } | |
| 1204 | |
| 1205 bool HeapRegionRemSetIterator::coarse_has_next(size_t& card_index) { | |
| 1206 if (_hrrs->_other_regions._n_coarse_entries == 0) return false; | |
| 1207 // Go to the next card. | |
| 1208 _coarse_cur_region_cur_card++; | |
| 1209 // Was the last the last card in the current region? | |
|
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
1210 if (_coarse_cur_region_cur_card == HeapRegion::CardsPerRegion) { |
| 342 | 1211 // Yes: find the next region. This may leave _coarse_cur_region_index |
| 1212 // Set to the last index, in which case there are no more coarse | |
| 1213 // regions. | |
| 1214 _coarse_cur_region_index = | |
| 1215 (int) _coarse_map->get_next_one_offset(_coarse_cur_region_index + 1); | |
| 1216 if ((size_t)_coarse_cur_region_index < _coarse_map->size()) { | |
| 1217 _coarse_cur_region_cur_card = 0; | |
| 1218 HeapWord* r_bot = | |
| 1219 _g1h->region_at(_coarse_cur_region_index)->bottom(); | |
| 1220 _cur_region_card_offset = _bosa->index_for(r_bot); | |
| 1221 } else { | |
| 1222 return false; | |
| 1223 } | |
| 1224 } | |
| 1225 // If we didn't return false above, then we can yield a card. | |
| 1226 card_index = _cur_region_card_offset + _coarse_cur_region_cur_card; | |
| 1227 return true; | |
| 1228 } | |
| 1229 | |
| 1230 void HeapRegionRemSetIterator::fine_find_next_non_null_prt() { | |
| 1231 // Otherwise, find the next bucket list in the array. | |
| 1232 _fine_array_index++; | |
| 1233 while (_fine_array_index < (int) OtherRegionsTable::_max_fine_entries) { | |
| 1234 _fine_cur_prt = _fine_grain_regions[_fine_array_index]; | |
| 1235 if (_fine_cur_prt != NULL) return; | |
| 1236 else _fine_array_index++; | |
| 1237 } | |
| 1238 assert(_fine_cur_prt == NULL, "Loop post"); | |
| 1239 } | |
| 1240 | |
| 1241 bool HeapRegionRemSetIterator::fine_has_next(size_t& card_index) { | |
| 1242 if (fine_has_next()) { | |
| 1243 _cur_region_cur_card = | |
| 1244 _fine_cur_prt->_bm.get_next_one_offset(_cur_region_cur_card + 1); | |
| 1245 } | |
| 1246 while (!fine_has_next()) { | |
|
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
1247 if (_cur_region_cur_card == (size_t) HeapRegion::CardsPerRegion) { |
| 342 | 1248 _cur_region_cur_card = 0; |
| 1249 _fine_cur_prt = _fine_cur_prt->next(); | |
| 1250 } | |
| 1251 if (_fine_cur_prt == NULL) { | |
| 1252 fine_find_next_non_null_prt(); | |
| 1253 if (_fine_cur_prt == NULL) return false; | |
| 1254 } | |
| 1255 assert(_fine_cur_prt != NULL && _cur_region_cur_card == 0, | |
| 1256 "inv."); | |
| 1257 HeapWord* r_bot = | |
| 1258 _fine_cur_prt->hr()->bottom(); | |
| 1259 _cur_region_card_offset = _bosa->index_for(r_bot); | |
| 1260 _cur_region_cur_card = _fine_cur_prt->_bm.get_next_one_offset(0); | |
| 1261 } | |
| 1262 assert(fine_has_next(), "Or else we exited the loop via the return."); | |
| 1263 card_index = _cur_region_card_offset + _cur_region_cur_card; | |
| 1264 return true; | |
| 1265 } | |
| 1266 | |
| 1267 bool HeapRegionRemSetIterator::fine_has_next() { | |
| 1268 return | |
| 1269 _fine_cur_prt != NULL && | |
|
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
1270 _cur_region_cur_card < (size_t) HeapRegion::CardsPerRegion; |
| 342 | 1271 } |
| 1272 | |
| 1273 bool HeapRegionRemSetIterator::has_next(size_t& card_index) { | |
| 1274 switch (_is) { | |
| 1275 case Sparse: | |
| 1276 if (_sparse_iter.has_next(card_index)) { | |
| 1277 _n_yielded_sparse++; | |
| 1278 return true; | |
| 1279 } | |
| 1280 // Otherwise, deliberate fall-through | |
| 1281 _is = Fine; | |
| 1282 case Fine: | |
| 1283 if (fine_has_next(card_index)) { | |
| 1284 _n_yielded_fine++; | |
| 1285 return true; | |
| 1286 } | |
| 1287 // Otherwise, deliberate fall-through | |
| 1288 _is = Coarse; | |
| 1289 case Coarse: | |
| 1290 if (coarse_has_next(card_index)) { | |
| 1291 _n_yielded_coarse++; | |
| 1292 return true; | |
| 1293 } | |
| 1294 // Otherwise... | |
| 1295 break; | |
| 1296 } | |
| 1297 assert(ParallelGCThreads > 1 || | |
| 1298 n_yielded() == _hrrs->occupied(), | |
| 1299 "Should have yielded all the cards in the rem set " | |
| 1300 "(in the non-par case)."); | |
| 1301 return false; | |
| 1302 } | |
| 1303 | |
| 1304 | |
| 1305 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1306 OopOrNarrowOopStar* HeapRegionRemSet::_recorded_oops = NULL; |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1307 HeapWord** HeapRegionRemSet::_recorded_cards = NULL; |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1308 HeapRegion** HeapRegionRemSet::_recorded_regions = NULL; |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1309 int HeapRegionRemSet::_n_recorded = 0; |
| 342 | 1310 |
| 1311 HeapRegionRemSet::Event* HeapRegionRemSet::_recorded_events = NULL; | |
| 1312 int* HeapRegionRemSet::_recorded_event_index = NULL; | |
| 1313 int HeapRegionRemSet::_n_recorded_events = 0; | |
| 1314 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1315 void HeapRegionRemSet::record(HeapRegion* hr, OopOrNarrowOopStar f) { |
| 342 | 1316 if (_recorded_oops == NULL) { |
| 1317 assert(_n_recorded == 0 | |
| 1318 && _recorded_cards == NULL | |
| 1319 && _recorded_regions == NULL, | |
| 1320 "Inv"); | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1321 _recorded_oops = NEW_C_HEAP_ARRAY(OopOrNarrowOopStar, MaxRecorded); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1322 _recorded_cards = NEW_C_HEAP_ARRAY(HeapWord*, MaxRecorded); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1323 _recorded_regions = NEW_C_HEAP_ARRAY(HeapRegion*, MaxRecorded); |
| 342 | 1324 } |
| 1325 if (_n_recorded == MaxRecorded) { | |
| 1326 gclog_or_tty->print_cr("Filled up 'recorded' (%d).", MaxRecorded); | |
| 1327 } else { | |
| 1328 _recorded_cards[_n_recorded] = | |
| 1329 (HeapWord*)align_size_down(uintptr_t(f), | |
| 1330 CardTableModRefBS::card_size); | |
| 1331 _recorded_oops[_n_recorded] = f; | |
| 1332 _recorded_regions[_n_recorded] = hr; | |
| 1333 _n_recorded++; | |
| 1334 } | |
| 1335 } | |
| 1336 | |
| 1337 void HeapRegionRemSet::record_event(Event evnt) { | |
| 1338 if (!G1RecordHRRSEvents) return; | |
| 1339 | |
| 1340 if (_recorded_events == NULL) { | |
| 1341 assert(_n_recorded_events == 0 | |
| 1342 && _recorded_event_index == NULL, | |
| 1343 "Inv"); | |
| 1344 _recorded_events = NEW_C_HEAP_ARRAY(Event, MaxRecordedEvents); | |
| 1345 _recorded_event_index = NEW_C_HEAP_ARRAY(int, MaxRecordedEvents); | |
| 1346 } | |
| 1347 if (_n_recorded_events == MaxRecordedEvents) { | |
| 1348 gclog_or_tty->print_cr("Filled up 'recorded_events' (%d).", MaxRecordedEvents); | |
| 1349 } else { | |
| 1350 _recorded_events[_n_recorded_events] = evnt; | |
| 1351 _recorded_event_index[_n_recorded_events] = _n_recorded; | |
| 1352 _n_recorded_events++; | |
| 1353 } | |
| 1354 } | |
| 1355 | |
| 1356 void HeapRegionRemSet::print_event(outputStream* str, Event evnt) { | |
| 1357 switch (evnt) { | |
| 1358 case Event_EvacStart: | |
| 1359 str->print("Evac Start"); | |
| 1360 break; | |
| 1361 case Event_EvacEnd: | |
| 1362 str->print("Evac End"); | |
| 1363 break; | |
| 1364 case Event_RSUpdateEnd: | |
| 1365 str->print("RS Update End"); | |
| 1366 break; | |
| 1367 } | |
| 1368 } | |
| 1369 | |
| 1370 void HeapRegionRemSet::print_recorded() { | |
| 1371 int cur_evnt = 0; | |
| 1372 Event cur_evnt_kind; | |
| 1373 int cur_evnt_ind = 0; | |
| 1374 if (_n_recorded_events > 0) { | |
| 1375 cur_evnt_kind = _recorded_events[cur_evnt]; | |
| 1376 cur_evnt_ind = _recorded_event_index[cur_evnt]; | |
| 1377 } | |
| 1378 | |
| 1379 for (int i = 0; i < _n_recorded; i++) { | |
| 1380 while (cur_evnt < _n_recorded_events && i == cur_evnt_ind) { | |
| 1381 gclog_or_tty->print("Event: "); | |
| 1382 print_event(gclog_or_tty, cur_evnt_kind); | |
| 1383 gclog_or_tty->print_cr(""); | |
| 1384 cur_evnt++; | |
| 1385 if (cur_evnt < MaxRecordedEvents) { | |
| 1386 cur_evnt_kind = _recorded_events[cur_evnt]; | |
| 1387 cur_evnt_ind = _recorded_event_index[cur_evnt]; | |
| 1388 } | |
| 1389 } | |
| 1390 gclog_or_tty->print("Added card " PTR_FORMAT " to region [" PTR_FORMAT "...]" | |
| 1391 " for ref " PTR_FORMAT ".\n", | |
| 1392 _recorded_cards[i], _recorded_regions[i]->bottom(), | |
| 1393 _recorded_oops[i]); | |
| 1394 } | |
| 1395 } | |
| 1396 | |
| 2173 | 1397 void HeapRegionRemSet::reset_for_cleanup_tasks() { |
| 1398 SparsePRT::reset_for_cleanup_tasks(); | |
| 1399 } | |
| 1400 | |
| 1401 void HeapRegionRemSet::do_cleanup_work(HRRSCleanupTask* hrrs_cleanup_task) { | |
| 1402 _other_regions.do_cleanup_work(hrrs_cleanup_task); | |
| 1403 } | |
| 1404 | |
| 1405 void | |
| 1406 HeapRegionRemSet::finish_cleanup_task(HRRSCleanupTask* hrrs_cleanup_task) { | |
| 1407 SparsePRT::finish_cleanup_task(hrrs_cleanup_task); | |
| 1408 } | |
| 1409 | |
| 342 | 1410 #ifndef PRODUCT |
| 1411 void HeapRegionRemSet::test() { | |
| 1412 os::sleep(Thread::current(), (jlong)5000, false); | |
| 1413 G1CollectedHeap* g1h = G1CollectedHeap::heap(); | |
| 1414 | |
|
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1415 // Run with "-XX:G1LogRSetRegionEntries=2", so that 1 and 5 end up in same |
| 342 | 1416 // hash bucket. |
| 1417 HeapRegion* hr0 = g1h->region_at(0); | |
| 1418 HeapRegion* hr1 = g1h->region_at(1); | |
| 1419 HeapRegion* hr2 = g1h->region_at(5); | |
| 1420 HeapRegion* hr3 = g1h->region_at(6); | |
| 1421 HeapRegion* hr4 = g1h->region_at(7); | |
| 1422 HeapRegion* hr5 = g1h->region_at(8); | |
| 1423 | |
| 1424 HeapWord* hr1_start = hr1->bottom(); | |
| 1425 HeapWord* hr1_mid = hr1_start + HeapRegion::GrainWords/2; | |
| 1426 HeapWord* hr1_last = hr1->end() - 1; | |
| 1427 | |
| 1428 HeapWord* hr2_start = hr2->bottom(); | |
| 1429 HeapWord* hr2_mid = hr2_start + HeapRegion::GrainWords/2; | |
| 1430 HeapWord* hr2_last = hr2->end() - 1; | |
| 1431 | |
| 1432 HeapWord* hr3_start = hr3->bottom(); | |
| 1433 HeapWord* hr3_mid = hr3_start + HeapRegion::GrainWords/2; | |
| 1434 HeapWord* hr3_last = hr3->end() - 1; | |
| 1435 | |
| 1436 HeapRegionRemSet* hrrs = hr0->rem_set(); | |
| 1437 | |
| 1438 // Make three references from region 0x101... | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1439 hrrs->add_reference((OopOrNarrowOopStar)hr1_start); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1440 hrrs->add_reference((OopOrNarrowOopStar)hr1_mid); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1441 hrrs->add_reference((OopOrNarrowOopStar)hr1_last); |
| 342 | 1442 |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1443 hrrs->add_reference((OopOrNarrowOopStar)hr2_start); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1444 hrrs->add_reference((OopOrNarrowOopStar)hr2_mid); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1445 hrrs->add_reference((OopOrNarrowOopStar)hr2_last); |
| 342 | 1446 |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1447 hrrs->add_reference((OopOrNarrowOopStar)hr3_start); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1448 hrrs->add_reference((OopOrNarrowOopStar)hr3_mid); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1449 hrrs->add_reference((OopOrNarrowOopStar)hr3_last); |
| 342 | 1450 |
| 1451 // Now cause a coarsening. | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1452 hrrs->add_reference((OopOrNarrowOopStar)hr4->bottom()); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1453 hrrs->add_reference((OopOrNarrowOopStar)hr5->bottom()); |
| 342 | 1454 |
| 1455 // Now, does iteration yield these three? | |
| 1456 HeapRegionRemSetIterator iter; | |
| 1457 hrrs->init_iterator(&iter); | |
| 1458 size_t sum = 0; | |
| 1459 size_t card_index; | |
| 1460 while (iter.has_next(card_index)) { | |
| 1461 HeapWord* card_start = | |
| 1462 G1CollectedHeap::heap()->bot_shared()->address_for_index(card_index); | |
| 1463 gclog_or_tty->print_cr(" Card " PTR_FORMAT ".", card_start); | |
| 1464 sum++; | |
| 1465 } | |
| 1466 guarantee(sum == 11 - 3 + 2048, "Failure"); | |
| 1467 guarantee(sum == hrrs->occupied(), "Failure"); | |
| 1468 } | |
| 1469 #endif |