Mercurial > hg > truffle
annotate src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp @ 8733:9def4075da6d
8008079: G1: Add nextObject routine to CMBitMapRO and replace nextWord
Summary: Update the task local finger to the start of the next object when marking aborts, in order to avoid the redundant scanning of all 0's when the marking task restarts, if otherwise updating to the next word. In addition, reuse the routine nextObject() in routine iterate().
Reviewed-by: johnc, ysr
Contributed-by: tamao <tao.mao@oracle.com>
| author | tamao |
|---|---|
| date | Tue, 05 Mar 2013 15:36:56 -0800 |
| parents | 859cd1a76f8a |
| children | 6f817ce50129 5c93c1f61226 |
| rev | line source |
|---|---|
| 342 | 1 /* |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
2 * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. |
| 342 | 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| 4 * | |
| 5 * This code is free software; you can redistribute it and/or modify it | |
| 6 * under the terms of the GNU General Public License version 2 only, as | |
| 7 * published by the Free Software Foundation. | |
| 8 * | |
| 9 * This code is distributed in the hope that it will be useful, but WITHOUT | |
| 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
| 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
| 12 * version 2 for more details (a copy is included in the LICENSE file that | |
| 13 * accompanied this code). | |
| 14 * | |
| 15 * You should have received a copy of the GNU General Public License version | |
| 16 * 2 along with this work; if not, write to the Free Software Foundation, | |
| 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |
| 18 * | |
|
1552
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1390
diff
changeset
|
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
|
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1390
diff
changeset
|
20 * or visit www.oracle.com if you need additional information or have any |
|
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1390
diff
changeset
|
21 * questions. |
| 342 | 22 * |
| 23 */ | |
| 24 | |
| 1972 | 25 #include "precompiled.hpp" |
| 26 #include "gc_implementation/g1/concurrentG1Refine.hpp" | |
| 27 #include "gc_implementation/g1/g1BlockOffsetTable.inline.hpp" | |
| 28 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp" | |
| 29 #include "gc_implementation/g1/heapRegionRemSet.hpp" | |
| 30 #include "gc_implementation/g1/heapRegionSeq.inline.hpp" | |
| 31 #include "memory/allocation.hpp" | |
| 32 #include "memory/space.inline.hpp" | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
33 #include "oops/oop.inline.hpp" |
| 1972 | 34 #include "utilities/bitMap.inline.hpp" |
| 35 #include "utilities/globalDefinitions.hpp" | |
| 342 | 36 |
| 6197 | 37 class PerRegionTable: public CHeapObj<mtGC> { |
| 342 | 38 friend class OtherRegionsTable; |
| 39 friend class HeapRegionRemSetIterator; | |
| 40 | |
| 41 HeapRegion* _hr; | |
| 42 BitMap _bm; | |
| 43 jint _occupied; | |
| 44 | |
| 6253 | 45 // next pointer for free/allocated 'all' list |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
46 PerRegionTable* _next; |
| 342 | 47 |
| 6253 | 48 // prev pointer for the allocated 'all' list |
| 49 PerRegionTable* _prev; | |
| 342 | 50 |
| 6253 | 51 // next pointer in collision list |
| 52 PerRegionTable * _collision_list_next; | |
| 342 | 53 |
| 6253 | 54 // Global free list of PRTs |
| 55 static PerRegionTable* _free_list; | |
| 342 | 56 |
| 57 protected: | |
| 58 // We need access in order to union things into the base table. | |
| 59 BitMap* bm() { return &_bm; } | |
| 60 | |
| 61 void recount_occupied() { | |
| 62 _occupied = (jint) bm()->count_one_bits(); | |
| 63 } | |
| 64 | |
| 65 PerRegionTable(HeapRegion* hr) : | |
| 66 _hr(hr), | |
| 67 _occupied(0), | |
| 6253 | 68 _bm(HeapRegion::CardsPerRegion, false /* in-resource-area */), |
| 69 _collision_list_next(NULL), _next(NULL), _prev(NULL) | |
| 342 | 70 {} |
| 71 | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
72 void add_card_work(CardIdx_t from_card, bool par) { |
| 342 | 73 if (!_bm.at(from_card)) { |
| 74 if (par) { | |
| 75 if (_bm.par_at_put(from_card, 1)) { | |
| 76 Atomic::inc(&_occupied); | |
| 77 } | |
| 78 } else { | |
| 79 _bm.at_put(from_card, 1); | |
| 80 _occupied++; | |
| 81 } | |
| 82 } | |
| 83 } | |
| 84 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
85 void add_reference_work(OopOrNarrowOopStar from, bool par) { |
| 342 | 86 // Must make this robust in case "from" is not in "_hr", because of |
| 87 // concurrency. | |
| 88 | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
89 if (G1TraceHeapRegionRememberedSet) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
90 gclog_or_tty->print_cr(" PRT::Add_reference_work(" PTR_FORMAT "->" PTR_FORMAT").", |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
91 from, |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
92 UseCompressedOops |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
93 ? oopDesc::load_decode_heap_oop((narrowOop*)from) |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
94 : oopDesc::load_decode_heap_oop((oop*)from)); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
95 } |
| 342 | 96 |
| 97 HeapRegion* loc_hr = hr(); | |
| 98 // If the test below fails, then this table was reused concurrently | |
| 99 // with this operation. This is OK, since the old table was coarsened, | |
| 100 // and adding a bit to the new table is never incorrect. | |
|
4020
5e5d4821bf07
7097516: G1: assert(0<= from_card && from_card<HeapRegion::CardsPerRegion) failed: Must be in range.
brutisso
parents:
3986
diff
changeset
|
101 // If the table used to belong to a continues humongous region and is |
|
5e5d4821bf07
7097516: G1: assert(0<= from_card && from_card<HeapRegion::CardsPerRegion) failed: Must be in range.
brutisso
parents:
3986
diff
changeset
|
102 // now reused for the corresponding start humongous region, we need to |
|
5e5d4821bf07
7097516: G1: assert(0<= from_card && from_card<HeapRegion::CardsPerRegion) failed: Must be in range.
brutisso
parents:
3986
diff
changeset
|
103 // make sure that we detect this. Thus, we call is_in_reserved_raw() |
|
5e5d4821bf07
7097516: G1: assert(0<= from_card && from_card<HeapRegion::CardsPerRegion) failed: Must be in range.
brutisso
parents:
3986
diff
changeset
|
104 // instead of just is_in_reserved() here. |
|
5e5d4821bf07
7097516: G1: assert(0<= from_card && from_card<HeapRegion::CardsPerRegion) failed: Must be in range.
brutisso
parents:
3986
diff
changeset
|
105 if (loc_hr->is_in_reserved_raw(from)) { |
| 342 | 106 size_t hw_offset = pointer_delta((HeapWord*)from, loc_hr->bottom()); |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
107 CardIdx_t from_card = (CardIdx_t) |
|
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
108 hw_offset >> (CardTableModRefBS::card_shift - LogHeapWordSize); |
| 342 | 109 |
|
3986
65a8ff39a6da
7095194: G1: HeapRegion::GrainBytes, GrainWords, and CardsPerRegion should be size_t
johnc
parents:
3830
diff
changeset
|
110 assert(0 <= from_card && (size_t)from_card < HeapRegion::CardsPerRegion, |
|
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
111 "Must be in range."); |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
112 add_card_work(from_card, par); |
| 342 | 113 } |
| 114 } | |
| 115 | |
| 116 public: | |
| 117 | |
| 118 HeapRegion* hr() const { return _hr; } | |
| 119 | |
| 120 jint occupied() const { | |
| 121 // Overkill, but if we ever need it... | |
| 122 // guarantee(_occupied == _bm.count_one_bits(), "Check"); | |
| 123 return _occupied; | |
| 124 } | |
| 125 | |
| 6253 | 126 void init(HeapRegion* hr, bool clear_links_to_all_list) { |
| 127 if (clear_links_to_all_list) { | |
| 128 set_next(NULL); | |
| 129 set_prev(NULL); | |
| 130 } | |
| 342 | 131 _hr = hr; |
| 6253 | 132 _collision_list_next = NULL; |
| 342 | 133 _occupied = 0; |
| 134 _bm.clear(); | |
| 135 } | |
| 136 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
137 void add_reference(OopOrNarrowOopStar from) { |
| 342 | 138 add_reference_work(from, /*parallel*/ true); |
| 139 } | |
| 140 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
141 void seq_add_reference(OopOrNarrowOopStar from) { |
| 342 | 142 add_reference_work(from, /*parallel*/ false); |
| 143 } | |
| 144 | |
| 145 void scrub(CardTableModRefBS* ctbs, BitMap* card_bm) { | |
| 146 HeapWord* hr_bot = hr()->bottom(); | |
|
489
2494ab195856
6653214: MemoryPoolMXBean.setUsageThreshold() does not support large heap sizes.
swamyv
parents:
342
diff
changeset
|
147 size_t hr_first_card_index = ctbs->index_for(hr_bot); |
| 342 | 148 bm()->set_intersection_at_offset(*card_bm, hr_first_card_index); |
| 149 recount_occupied(); | |
| 150 } | |
| 151 | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
152 void add_card(CardIdx_t from_card_index) { |
| 342 | 153 add_card_work(from_card_index, /*parallel*/ true); |
| 154 } | |
| 155 | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
156 void seq_add_card(CardIdx_t from_card_index) { |
| 342 | 157 add_card_work(from_card_index, /*parallel*/ false); |
| 158 } | |
| 159 | |
| 160 // (Destructively) union the bitmap of the current table into the given | |
| 161 // bitmap (which is assumed to be of the same size.) | |
| 162 void union_bitmap_into(BitMap* bm) { | |
| 163 bm->set_union(_bm); | |
| 164 } | |
| 165 | |
| 166 // Mem size in bytes. | |
| 167 size_t mem_size() const { | |
| 168 return sizeof(this) + _bm.size_in_words() * HeapWordSize; | |
| 169 } | |
| 170 | |
| 171 // Requires "from" to be in "hr()". | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
172 bool contains_reference(OopOrNarrowOopStar from) const { |
| 342 | 173 assert(hr()->is_in_reserved(from), "Precondition."); |
| 174 size_t card_ind = pointer_delta(from, hr()->bottom(), | |
| 175 CardTableModRefBS::card_size); | |
| 176 return _bm.at(card_ind); | |
| 177 } | |
| 178 | |
| 6253 | 179 // Bulk-free the PRTs from prt to last, assumes that they are |
| 180 // linked together using their _next field. | |
| 181 static void bulk_free(PerRegionTable* prt, PerRegionTable* last) { | |
| 342 | 182 while (true) { |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
183 PerRegionTable* fl = _free_list; |
| 6253 | 184 last->set_next(fl); |
| 185 PerRegionTable* res = (PerRegionTable*) Atomic::cmpxchg_ptr(prt, &_free_list, fl); | |
| 186 if (res == fl) { | |
| 187 return; | |
| 188 } | |
| 342 | 189 } |
| 190 ShouldNotReachHere(); | |
| 191 } | |
| 192 | |
| 6253 | 193 static void free(PerRegionTable* prt) { |
| 194 bulk_free(prt, prt); | |
| 195 } | |
| 196 | |
| 197 // Returns an initialized PerRegionTable instance. | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
198 static PerRegionTable* alloc(HeapRegion* hr) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
199 PerRegionTable* fl = _free_list; |
| 342 | 200 while (fl != NULL) { |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
201 PerRegionTable* nxt = fl->next(); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
202 PerRegionTable* res = |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
203 (PerRegionTable*) |
| 342 | 204 Atomic::cmpxchg_ptr(nxt, &_free_list, fl); |
| 205 if (res == fl) { | |
| 6253 | 206 fl->init(hr, true); |
| 342 | 207 return fl; |
| 208 } else { | |
| 209 fl = _free_list; | |
| 210 } | |
| 211 } | |
| 212 assert(fl == NULL, "Loop condition."); | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
213 return new PerRegionTable(hr); |
| 342 | 214 } |
| 215 | |
| 6253 | 216 PerRegionTable* next() const { return _next; } |
| 217 void set_next(PerRegionTable* next) { _next = next; } | |
| 218 PerRegionTable* prev() const { return _prev; } | |
| 219 void set_prev(PerRegionTable* prev) { _prev = prev; } | |
| 220 | |
| 221 // Accessor and Modification routines for the pointer for the | |
| 222 // singly linked collision list that links the PRTs within the | |
| 223 // OtherRegionsTable::_fine_grain_regions hash table. | |
| 224 // | |
| 225 // It might be useful to also make the collision list doubly linked | |
| 226 // to avoid iteration over the collisions list during scrubbing/deletion. | |
| 227 // OTOH there might not be many collisions. | |
| 228 | |
| 229 PerRegionTable* collision_list_next() const { | |
| 230 return _collision_list_next; | |
| 231 } | |
| 232 | |
| 233 void set_collision_list_next(PerRegionTable* next) { | |
| 234 _collision_list_next = next; | |
| 235 } | |
| 236 | |
| 237 PerRegionTable** collision_list_next_addr() { | |
| 238 return &_collision_list_next; | |
| 239 } | |
| 240 | |
| 342 | 241 static size_t fl_mem_size() { |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
242 PerRegionTable* cur = _free_list; |
| 342 | 243 size_t res = 0; |
| 244 while (cur != NULL) { | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
245 res += sizeof(PerRegionTable); |
| 342 | 246 cur = cur->next(); |
| 247 } | |
| 248 return res; | |
| 249 } | |
| 250 }; | |
| 251 | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
252 PerRegionTable* PerRegionTable::_free_list = NULL; |
| 342 | 253 |
| 254 size_t OtherRegionsTable::_max_fine_entries = 0; | |
| 255 size_t OtherRegionsTable::_mod_max_fine_entries_mask = 0; | |
| 256 size_t OtherRegionsTable::_fine_eviction_stride = 0; | |
| 257 size_t OtherRegionsTable::_fine_eviction_sample_size = 0; | |
| 258 | |
| 259 OtherRegionsTable::OtherRegionsTable(HeapRegion* hr) : | |
| 260 _g1h(G1CollectedHeap::heap()), | |
| 261 _m(Mutex::leaf, "An OtherRegionsTable lock", true), | |
| 262 _hr(hr), | |
| 263 _coarse_map(G1CollectedHeap::heap()->max_regions(), | |
| 264 false /* in-resource-area */), | |
| 265 _fine_grain_regions(NULL), | |
| 6253 | 266 _first_all_fine_prts(NULL), _last_all_fine_prts(NULL), |
| 342 | 267 _n_fine_entries(0), _n_coarse_entries(0), |
| 268 _fine_eviction_start(0), | |
| 269 _sparse_table(hr) | |
| 270 { | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
271 typedef PerRegionTable* PerRegionTablePtr; |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
272 |
| 342 | 273 if (_max_fine_entries == 0) { |
| 274 assert(_mod_max_fine_entries_mask == 0, "Both or none."); | |
|
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
275 size_t max_entries_log = (size_t)log2_long((jlong)G1RSetRegionEntries); |
|
6750
859cd1a76f8a
7197906: BlockOffsetArray::power_to_cards_back() needs to handle > 32 bit shifts
brutisso
parents:
6253
diff
changeset
|
276 _max_fine_entries = (size_t)1 << max_entries_log; |
| 342 | 277 _mod_max_fine_entries_mask = _max_fine_entries - 1; |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
278 |
| 342 | 279 assert(_fine_eviction_sample_size == 0 |
| 280 && _fine_eviction_stride == 0, "All init at same time."); | |
|
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
281 _fine_eviction_sample_size = MAX2((size_t)4, max_entries_log); |
| 342 | 282 _fine_eviction_stride = _max_fine_entries / _fine_eviction_sample_size; |
| 283 } | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
284 |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
285 _fine_grain_regions = new PerRegionTablePtr[_max_fine_entries]; |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
286 |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
287 if (_fine_grain_regions == NULL) { |
| 342 | 288 vm_exit_out_of_memory(sizeof(void*)*_max_fine_entries, |
| 289 "Failed to allocate _fine_grain_entries."); | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
290 } |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
291 |
| 342 | 292 for (size_t i = 0; i < _max_fine_entries; i++) { |
| 293 _fine_grain_regions[i] = NULL; | |
| 294 } | |
| 295 } | |
| 296 | |
| 6253 | 297 void OtherRegionsTable::link_to_all(PerRegionTable* prt) { |
| 298 // We always append to the beginning of the list for convenience; | |
| 299 // the order of entries in this list does not matter. | |
| 300 if (_first_all_fine_prts != NULL) { | |
| 301 assert(_first_all_fine_prts->prev() == NULL, "invariant"); | |
| 302 _first_all_fine_prts->set_prev(prt); | |
| 303 prt->set_next(_first_all_fine_prts); | |
| 304 } else { | |
| 305 // this is the first element we insert. Adjust the "last" pointer | |
| 306 _last_all_fine_prts = prt; | |
| 307 assert(prt->next() == NULL, "just checking"); | |
| 308 } | |
| 309 // the new element is always the first element without a predecessor | |
| 310 prt->set_prev(NULL); | |
| 311 _first_all_fine_prts = prt; | |
| 312 | |
| 313 assert(prt->prev() == NULL, "just checking"); | |
| 314 assert(_first_all_fine_prts == prt, "just checking"); | |
| 315 assert((_first_all_fine_prts == NULL && _last_all_fine_prts == NULL) || | |
| 316 (_first_all_fine_prts != NULL && _last_all_fine_prts != NULL), | |
| 317 "just checking"); | |
| 318 assert(_last_all_fine_prts == NULL || _last_all_fine_prts->next() == NULL, | |
| 319 "just checking"); | |
| 320 assert(_first_all_fine_prts == NULL || _first_all_fine_prts->prev() == NULL, | |
| 321 "just checking"); | |
| 322 } | |
| 323 | |
| 324 void OtherRegionsTable::unlink_from_all(PerRegionTable* prt) { | |
| 325 if (prt->prev() != NULL) { | |
| 326 assert(_first_all_fine_prts != prt, "just checking"); | |
| 327 prt->prev()->set_next(prt->next()); | |
| 328 // removing the last element in the list? | |
| 329 if (_last_all_fine_prts == prt) { | |
| 330 _last_all_fine_prts = prt->prev(); | |
| 331 } | |
| 332 } else { | |
| 333 assert(_first_all_fine_prts == prt, "just checking"); | |
| 334 _first_all_fine_prts = prt->next(); | |
| 335 // list is empty now? | |
| 336 if (_first_all_fine_prts == NULL) { | |
| 337 _last_all_fine_prts = NULL; | |
| 338 } | |
| 339 } | |
| 340 | |
| 341 if (prt->next() != NULL) { | |
| 342 prt->next()->set_prev(prt->prev()); | |
| 343 } | |
| 344 | |
| 345 prt->set_next(NULL); | |
| 346 prt->set_prev(NULL); | |
| 347 | |
| 348 assert((_first_all_fine_prts == NULL && _last_all_fine_prts == NULL) || | |
| 349 (_first_all_fine_prts != NULL && _last_all_fine_prts != NULL), | |
| 350 "just checking"); | |
| 351 assert(_last_all_fine_prts == NULL || _last_all_fine_prts->next() == NULL, | |
| 352 "just checking"); | |
| 353 assert(_first_all_fine_prts == NULL || _first_all_fine_prts->prev() == NULL, | |
| 354 "just checking"); | |
| 355 } | |
| 356 | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
357 int** OtherRegionsTable::_from_card_cache = NULL; |
| 342 | 358 size_t OtherRegionsTable::_from_card_cache_max_regions = 0; |
| 359 size_t OtherRegionsTable::_from_card_cache_mem_size = 0; | |
| 360 | |
| 361 void OtherRegionsTable::init_from_card_cache(size_t max_regions) { | |
| 362 _from_card_cache_max_regions = max_regions; | |
| 363 | |
| 364 int n_par_rs = HeapRegionRemSet::num_par_rem_sets(); | |
| 6197 | 365 _from_card_cache = NEW_C_HEAP_ARRAY(int*, n_par_rs, mtGC); |
| 342 | 366 for (int i = 0; i < n_par_rs; i++) { |
| 6197 | 367 _from_card_cache[i] = NEW_C_HEAP_ARRAY(int, max_regions, mtGC); |
| 342 | 368 for (size_t j = 0; j < max_regions; j++) { |
| 369 _from_card_cache[i][j] = -1; // An invalid value. | |
| 370 } | |
| 371 } | |
| 372 _from_card_cache_mem_size = n_par_rs * max_regions * sizeof(int); | |
| 373 } | |
| 374 | |
| 375 void OtherRegionsTable::shrink_from_card_cache(size_t new_n_regs) { | |
| 376 for (int i = 0; i < HeapRegionRemSet::num_par_rem_sets(); i++) { | |
| 377 assert(new_n_regs <= _from_card_cache_max_regions, "Must be within max."); | |
| 378 for (size_t j = new_n_regs; j < _from_card_cache_max_regions; j++) { | |
| 379 _from_card_cache[i][j] = -1; // An invalid value. | |
| 380 } | |
| 381 } | |
| 382 } | |
| 383 | |
| 384 #ifndef PRODUCT | |
| 385 void OtherRegionsTable::print_from_card_cache() { | |
| 386 for (int i = 0; i < HeapRegionRemSet::num_par_rem_sets(); i++) { | |
| 387 for (size_t j = 0; j < _from_card_cache_max_regions; j++) { | |
| 388 gclog_or_tty->print_cr("_from_card_cache[%d][%d] = %d.", | |
| 389 i, j, _from_card_cache[i][j]); | |
| 390 } | |
| 391 } | |
| 392 } | |
| 393 #endif | |
| 394 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
395 void OtherRegionsTable::add_reference(OopOrNarrowOopStar from, int tid) { |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
396 size_t cur_hrs_ind = (size_t) hr()->hrs_index(); |
| 342 | 397 |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
398 if (G1TraceHeapRegionRememberedSet) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
399 gclog_or_tty->print_cr("ORT::add_reference_work(" PTR_FORMAT "->" PTR_FORMAT ").", |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
400 from, |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
401 UseCompressedOops |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
402 ? oopDesc::load_decode_heap_oop((narrowOop*)from) |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
403 : oopDesc::load_decode_heap_oop((oop*)from)); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
404 } |
| 342 | 405 |
| 406 int from_card = (int)(uintptr_t(from) >> CardTableModRefBS::card_shift); | |
| 407 | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
408 if (G1TraceHeapRegionRememberedSet) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
409 gclog_or_tty->print_cr("Table for [" PTR_FORMAT "...): card %d (cache = %d)", |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
410 hr()->bottom(), from_card, |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
411 _from_card_cache[tid][cur_hrs_ind]); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
412 } |
| 342 | 413 |
| 414 if (from_card == _from_card_cache[tid][cur_hrs_ind]) { | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
415 if (G1TraceHeapRegionRememberedSet) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
416 gclog_or_tty->print_cr(" from-card cache hit."); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
417 } |
| 342 | 418 assert(contains_reference(from), "We just added it!"); |
| 419 return; | |
| 420 } else { | |
| 421 _from_card_cache[tid][cur_hrs_ind] = from_card; | |
| 422 } | |
| 423 | |
| 424 // Note that this may be a continued H region. | |
| 425 HeapRegion* from_hr = _g1h->heap_region_containing_raw(from); | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
426 RegionIdx_t from_hrs_ind = (RegionIdx_t) from_hr->hrs_index(); |
| 342 | 427 |
| 428 // If the region is already coarsened, return. | |
| 429 if (_coarse_map.at(from_hrs_ind)) { | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
430 if (G1TraceHeapRegionRememberedSet) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
431 gclog_or_tty->print_cr(" coarse map hit."); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
432 } |
| 342 | 433 assert(contains_reference(from), "We just added it!"); |
| 434 return; | |
| 435 } | |
| 436 | |
| 437 // Otherwise find a per-region table to add it to. | |
| 438 size_t ind = from_hrs_ind & _mod_max_fine_entries_mask; | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
439 PerRegionTable* prt = find_region_table(ind, from_hr); |
| 342 | 440 if (prt == NULL) { |
| 441 MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag); | |
| 442 // Confirm that it's really not there... | |
| 443 prt = find_region_table(ind, from_hr); | |
| 444 if (prt == NULL) { | |
| 445 | |
| 446 uintptr_t from_hr_bot_card_index = | |
| 447 uintptr_t(from_hr->bottom()) | |
| 448 >> CardTableModRefBS::card_shift; | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
449 CardIdx_t card_index = from_card - from_hr_bot_card_index; |
|
3986
65a8ff39a6da
7095194: G1: HeapRegion::GrainBytes, GrainWords, and CardsPerRegion should be size_t
johnc
parents:
3830
diff
changeset
|
450 assert(0 <= card_index && (size_t)card_index < HeapRegion::CardsPerRegion, |
| 342 | 451 "Must be in range."); |
| 452 if (G1HRRSUseSparseTable && | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
453 _sparse_table.add_card(from_hrs_ind, card_index)) { |
| 342 | 454 if (G1RecordHRRSOops) { |
| 455 HeapRegionRemSet::record(hr(), from); | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
456 if (G1TraceHeapRegionRememberedSet) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
457 gclog_or_tty->print(" Added card " PTR_FORMAT " to region " |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
458 "[" PTR_FORMAT "...) for ref " PTR_FORMAT ".\n", |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
459 align_size_down(uintptr_t(from), |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
460 CardTableModRefBS::card_size), |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
461 hr()->bottom(), from); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
462 } |
| 342 | 463 } |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
464 if (G1TraceHeapRegionRememberedSet) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
465 gclog_or_tty->print_cr(" added card to sparse table."); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
466 } |
| 342 | 467 assert(contains_reference_locked(from), "We just added it!"); |
| 468 return; | |
| 469 } else { | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
470 if (G1TraceHeapRegionRememberedSet) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
471 gclog_or_tty->print_cr(" [tid %d] sparse table entry " |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
472 "overflow(f: %d, t: %d)", |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
473 tid, from_hrs_ind, cur_hrs_ind); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
474 } |
| 342 | 475 } |
| 476 | |
| 477 if (_n_fine_entries == _max_fine_entries) { | |
| 478 prt = delete_region_table(); | |
| 6253 | 479 // There is no need to clear the links to the 'all' list here: |
| 480 // prt will be reused immediately, i.e. remain in the 'all' list. | |
| 481 prt->init(from_hr, false /* clear_links_to_all_list */); | |
| 342 | 482 } else { |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
483 prt = PerRegionTable::alloc(from_hr); |
| 6253 | 484 link_to_all(prt); |
| 342 | 485 } |
| 486 | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
487 PerRegionTable* first_prt = _fine_grain_regions[ind]; |
| 6253 | 488 prt->set_collision_list_next(first_prt); |
| 342 | 489 _fine_grain_regions[ind] = prt; |
| 490 _n_fine_entries++; | |
| 491 | |
| 492 if (G1HRRSUseSparseTable) { | |
|
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
493 // Transfer from sparse to fine-grain. |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
494 SparsePRTEntry *sprt_entry = _sparse_table.get_entry(from_hrs_ind); |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
495 assert(sprt_entry != NULL, "There should have been an entry"); |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
496 for (int i = 0; i < SparsePRTEntry::cards_num(); i++) { |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
497 CardIdx_t c = sprt_entry->card(i); |
| 342 | 498 if (c != SparsePRTEntry::NullEntry) { |
| 499 prt->add_card(c); | |
| 500 } | |
| 501 } | |
| 502 // Now we can delete the sparse entry. | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
503 bool res = _sparse_table.delete_entry(from_hrs_ind); |
| 342 | 504 assert(res, "It should have been there."); |
| 505 } | |
| 506 } | |
| 507 assert(prt != NULL && prt->hr() == from_hr, "consequence"); | |
| 508 } | |
| 509 // Note that we can't assert "prt->hr() == from_hr", because of the | |
| 510 // possibility of concurrent reuse. But see head comment of | |
| 511 // OtherRegionsTable for why this is OK. | |
| 512 assert(prt != NULL, "Inv"); | |
| 513 | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
514 prt->add_reference(from); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
515 |
| 342 | 516 if (G1RecordHRRSOops) { |
| 517 HeapRegionRemSet::record(hr(), from); | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
518 if (G1TraceHeapRegionRememberedSet) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
519 gclog_or_tty->print("Added card " PTR_FORMAT " to region " |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
520 "[" PTR_FORMAT "...) for ref " PTR_FORMAT ".\n", |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
521 align_size_down(uintptr_t(from), |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
522 CardTableModRefBS::card_size), |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
523 hr()->bottom(), from); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
524 } |
| 342 | 525 } |
| 526 assert(contains_reference(from), "We just added it!"); | |
| 527 } | |
| 528 | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
529 PerRegionTable* |
| 342 | 530 OtherRegionsTable::find_region_table(size_t ind, HeapRegion* hr) const { |
| 531 assert(0 <= ind && ind < _max_fine_entries, "Preconditions."); | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
532 PerRegionTable* prt = _fine_grain_regions[ind]; |
| 342 | 533 while (prt != NULL && prt->hr() != hr) { |
| 6253 | 534 prt = prt->collision_list_next(); |
| 342 | 535 } |
| 536 // Loop postcondition is the method postcondition. | |
| 537 return prt; | |
| 538 } | |
| 539 | |
| 540 jint OtherRegionsTable::_n_coarsenings = 0; | |
| 541 | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
542 PerRegionTable* OtherRegionsTable::delete_region_table() { |
| 342 | 543 assert(_m.owned_by_self(), "Precondition"); |
| 544 assert(_n_fine_entries == _max_fine_entries, "Precondition"); | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
545 PerRegionTable* max = NULL; |
| 342 | 546 jint max_occ = 0; |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
547 PerRegionTable** max_prev; |
| 342 | 548 size_t max_ind; |
| 549 | |
| 550 size_t i = _fine_eviction_start; | |
| 551 for (size_t k = 0; k < _fine_eviction_sample_size; k++) { | |
| 552 size_t ii = i; | |
| 553 // Make sure we get a non-NULL sample. | |
| 554 while (_fine_grain_regions[ii] == NULL) { | |
| 555 ii++; | |
| 556 if (ii == _max_fine_entries) ii = 0; | |
| 557 guarantee(ii != i, "We must find one."); | |
| 558 } | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
559 PerRegionTable** prev = &_fine_grain_regions[ii]; |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
560 PerRegionTable* cur = *prev; |
| 342 | 561 while (cur != NULL) { |
| 562 jint cur_occ = cur->occupied(); | |
| 563 if (max == NULL || cur_occ > max_occ) { | |
| 564 max = cur; | |
| 565 max_prev = prev; | |
| 566 max_ind = i; | |
| 567 max_occ = cur_occ; | |
| 568 } | |
| 6253 | 569 prev = cur->collision_list_next_addr(); |
| 570 cur = cur->collision_list_next(); | |
| 342 | 571 } |
| 572 i = i + _fine_eviction_stride; | |
| 573 if (i >= _n_fine_entries) i = i - _n_fine_entries; | |
| 574 } | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
575 |
| 342 | 576 _fine_eviction_start++; |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
577 |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
578 if (_fine_eviction_start >= _n_fine_entries) { |
| 342 | 579 _fine_eviction_start -= _n_fine_entries; |
| 580 } | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
581 |
| 342 | 582 guarantee(max != NULL, "Since _n_fine_entries > 0"); |
| 583 | |
| 584 // Set the corresponding coarse bit. | |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
585 size_t max_hrs_index = (size_t) max->hr()->hrs_index(); |
| 342 | 586 if (!_coarse_map.at(max_hrs_index)) { |
| 587 _coarse_map.at_put(max_hrs_index, true); | |
| 588 _n_coarse_entries++; | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
589 if (G1TraceHeapRegionRememberedSet) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
590 gclog_or_tty->print("Coarsened entry in region [" PTR_FORMAT "...] " |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
591 "for region [" PTR_FORMAT "...] (%d coarse entries).\n", |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
592 hr()->bottom(), |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
593 max->hr()->bottom(), |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
594 _n_coarse_entries); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
595 } |
| 342 | 596 } |
| 597 | |
| 598 // Unsplice. | |
| 6253 | 599 *max_prev = max->collision_list_next(); |
| 342 | 600 Atomic::inc(&_n_coarsenings); |
| 601 _n_fine_entries--; | |
| 602 return max; | |
| 603 } | |
| 604 | |
| 605 | |
| 606 // At present, this must be called stop-world single-threaded. | |
| 607 void OtherRegionsTable::scrub(CardTableModRefBS* ctbs, | |
| 608 BitMap* region_bm, BitMap* card_bm) { | |
| 609 // First eliminated garbage regions from the coarse map. | |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
610 if (G1RSScrubVerbose) { |
|
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
611 gclog_or_tty->print_cr("Scrubbing region %u:", hr()->hrs_index()); |
|
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
612 } |
| 342 | 613 |
| 614 assert(_coarse_map.size() == region_bm->size(), "Precondition"); | |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
615 if (G1RSScrubVerbose) { |
|
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
616 gclog_or_tty->print(" Coarse map: before = "SIZE_FORMAT"...", |
|
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
617 _n_coarse_entries); |
|
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
618 } |
| 342 | 619 _coarse_map.set_intersection(*region_bm); |
| 620 _n_coarse_entries = _coarse_map.count_one_bits(); | |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
621 if (G1RSScrubVerbose) { |
|
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
622 gclog_or_tty->print_cr(" after = "SIZE_FORMAT".", _n_coarse_entries); |
|
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
623 } |
| 342 | 624 |
| 625 // Now do the fine-grained maps. | |
| 626 for (size_t i = 0; i < _max_fine_entries; i++) { | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
627 PerRegionTable* cur = _fine_grain_regions[i]; |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
628 PerRegionTable** prev = &_fine_grain_regions[i]; |
| 342 | 629 while (cur != NULL) { |
| 6253 | 630 PerRegionTable* nxt = cur->collision_list_next(); |
| 342 | 631 // If the entire region is dead, eliminate. |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
632 if (G1RSScrubVerbose) { |
|
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
633 gclog_or_tty->print_cr(" For other region %u:", |
| 3766 | 634 cur->hr()->hrs_index()); |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
635 } |
|
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
636 if (!region_bm->at((size_t) cur->hr()->hrs_index())) { |
| 342 | 637 *prev = nxt; |
| 6253 | 638 cur->set_collision_list_next(NULL); |
| 342 | 639 _n_fine_entries--; |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
640 if (G1RSScrubVerbose) { |
| 342 | 641 gclog_or_tty->print_cr(" deleted via region map."); |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
642 } |
| 6253 | 643 unlink_from_all(cur); |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
644 PerRegionTable::free(cur); |
| 342 | 645 } else { |
| 646 // Do fine-grain elimination. | |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
647 if (G1RSScrubVerbose) { |
| 342 | 648 gclog_or_tty->print(" occ: before = %4d.", cur->occupied()); |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
649 } |
| 342 | 650 cur->scrub(ctbs, card_bm); |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
651 if (G1RSScrubVerbose) { |
| 342 | 652 gclog_or_tty->print_cr(" after = %4d.", cur->occupied()); |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
653 } |
| 342 | 654 // Did that empty the table completely? |
| 655 if (cur->occupied() == 0) { | |
| 656 *prev = nxt; | |
| 6253 | 657 cur->set_collision_list_next(NULL); |
| 342 | 658 _n_fine_entries--; |
| 6253 | 659 unlink_from_all(cur); |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
660 PerRegionTable::free(cur); |
| 342 | 661 } else { |
| 6253 | 662 prev = cur->collision_list_next_addr(); |
| 342 | 663 } |
| 664 } | |
| 665 cur = nxt; | |
| 666 } | |
| 667 } | |
| 668 // Since we may have deleted a from_card_cache entry from the RS, clear | |
| 669 // the FCC. | |
| 670 clear_fcc(); | |
| 671 } | |
| 672 | |
| 673 | |
| 674 size_t OtherRegionsTable::occupied() const { | |
| 675 // Cast away const in this case. | |
| 676 MutexLockerEx x((Mutex*)&_m, Mutex::_no_safepoint_check_flag); | |
| 677 size_t sum = occ_fine(); | |
| 678 sum += occ_sparse(); | |
| 679 sum += occ_coarse(); | |
| 680 return sum; | |
| 681 } | |
| 682 | |
| 683 size_t OtherRegionsTable::occ_fine() const { | |
| 684 size_t sum = 0; | |
| 6253 | 685 |
| 686 size_t num = 0; | |
| 687 PerRegionTable * cur = _first_all_fine_prts; | |
| 688 while (cur != NULL) { | |
| 689 sum += cur->occupied(); | |
| 690 cur = cur->next(); | |
| 691 num++; | |
| 342 | 692 } |
| 6253 | 693 guarantee(num == _n_fine_entries, "just checking"); |
| 342 | 694 return sum; |
| 695 } | |
| 696 | |
| 697 size_t OtherRegionsTable::occ_coarse() const { | |
|
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
698 return (_n_coarse_entries * HeapRegion::CardsPerRegion); |
| 342 | 699 } |
| 700 | |
| 701 size_t OtherRegionsTable::occ_sparse() const { | |
| 702 return _sparse_table.occupied(); | |
| 703 } | |
| 704 | |
| 705 size_t OtherRegionsTable::mem_size() const { | |
| 706 // Cast away const in this case. | |
| 707 MutexLockerEx x((Mutex*)&_m, Mutex::_no_safepoint_check_flag); | |
| 708 size_t sum = 0; | |
| 6253 | 709 PerRegionTable * cur = _first_all_fine_prts; |
| 710 while (cur != NULL) { | |
| 711 sum += cur->mem_size(); | |
| 712 cur = cur->next(); | |
| 342 | 713 } |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
714 sum += (sizeof(PerRegionTable*) * _max_fine_entries); |
| 342 | 715 sum += (_coarse_map.size_in_words() * HeapWordSize); |
| 716 sum += (_sparse_table.mem_size()); | |
| 717 sum += sizeof(*this) - sizeof(_sparse_table); // Avoid double counting above. | |
| 718 return sum; | |
| 719 } | |
| 720 | |
| 721 size_t OtherRegionsTable::static_mem_size() { | |
| 722 return _from_card_cache_mem_size; | |
| 723 } | |
| 724 | |
| 725 size_t OtherRegionsTable::fl_mem_size() { | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
726 return PerRegionTable::fl_mem_size(); |
| 342 | 727 } |
| 728 | |
| 729 void OtherRegionsTable::clear_fcc() { | |
| 6253 | 730 size_t hrs_idx = hr()->hrs_index(); |
| 342 | 731 for (int i = 0; i < HeapRegionRemSet::num_par_rem_sets(); i++) { |
| 6253 | 732 _from_card_cache[i][hrs_idx] = -1; |
| 342 | 733 } |
| 734 } | |
| 735 | |
| 736 void OtherRegionsTable::clear() { | |
| 737 MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag); | |
| 6253 | 738 // if there are no entries, skip this step |
| 739 if (_first_all_fine_prts != NULL) { | |
| 740 guarantee(_first_all_fine_prts != NULL && _last_all_fine_prts != NULL, "just checking"); | |
| 741 PerRegionTable::bulk_free(_first_all_fine_prts, _last_all_fine_prts); | |
| 742 memset(_fine_grain_regions, 0, _max_fine_entries * sizeof(_fine_grain_regions[0])); | |
| 743 } else { | |
| 744 guarantee(_first_all_fine_prts == NULL && _last_all_fine_prts == NULL, "just checking"); | |
| 342 | 745 } |
| 6253 | 746 |
| 747 _first_all_fine_prts = _last_all_fine_prts = NULL; | |
| 342 | 748 _sparse_table.clear(); |
| 749 _coarse_map.clear(); | |
| 750 _n_fine_entries = 0; | |
| 751 _n_coarse_entries = 0; | |
| 752 | |
| 753 clear_fcc(); | |
| 754 } | |
| 755 | |
| 756 void OtherRegionsTable::clear_incoming_entry(HeapRegion* from_hr) { | |
| 757 MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag); | |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
758 size_t hrs_ind = (size_t) from_hr->hrs_index(); |
| 342 | 759 size_t ind = hrs_ind & _mod_max_fine_entries_mask; |
| 760 if (del_single_region_table(ind, from_hr)) { | |
| 761 assert(!_coarse_map.at(hrs_ind), "Inv"); | |
| 762 } else { | |
| 763 _coarse_map.par_at_put(hrs_ind, 0); | |
| 764 } | |
| 765 // Check to see if any of the fcc entries come from here. | |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
766 size_t hr_ind = (size_t) hr()->hrs_index(); |
| 342 | 767 for (int tid = 0; tid < HeapRegionRemSet::num_par_rem_sets(); tid++) { |
| 768 int fcc_ent = _from_card_cache[tid][hr_ind]; | |
| 769 if (fcc_ent != -1) { | |
| 770 HeapWord* card_addr = (HeapWord*) | |
| 771 (uintptr_t(fcc_ent) << CardTableModRefBS::card_shift); | |
| 772 if (hr()->is_in_reserved(card_addr)) { | |
| 773 // Clear the from card cache. | |
| 774 _from_card_cache[tid][hr_ind] = -1; | |
| 775 } | |
| 776 } | |
| 777 } | |
| 778 } | |
| 779 | |
| 780 bool OtherRegionsTable::del_single_region_table(size_t ind, | |
| 781 HeapRegion* hr) { | |
| 782 assert(0 <= ind && ind < _max_fine_entries, "Preconditions."); | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
783 PerRegionTable** prev_addr = &_fine_grain_regions[ind]; |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
784 PerRegionTable* prt = *prev_addr; |
| 342 | 785 while (prt != NULL && prt->hr() != hr) { |
| 6253 | 786 prev_addr = prt->collision_list_next_addr(); |
| 787 prt = prt->collision_list_next(); | |
| 342 | 788 } |
| 789 if (prt != NULL) { | |
| 790 assert(prt->hr() == hr, "Loop postcondition."); | |
| 6253 | 791 *prev_addr = prt->collision_list_next(); |
| 792 unlink_from_all(prt); | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
793 PerRegionTable::free(prt); |
| 342 | 794 _n_fine_entries--; |
| 795 return true; | |
| 796 } else { | |
| 797 return false; | |
| 798 } | |
| 799 } | |
| 800 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
801 bool OtherRegionsTable::contains_reference(OopOrNarrowOopStar from) const { |
| 342 | 802 // Cast away const in this case. |
| 803 MutexLockerEx x((Mutex*)&_m, Mutex::_no_safepoint_check_flag); | |
| 804 return contains_reference_locked(from); | |
| 805 } | |
| 806 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
807 bool OtherRegionsTable::contains_reference_locked(OopOrNarrowOopStar from) const { |
| 342 | 808 HeapRegion* hr = _g1h->heap_region_containing_raw(from); |
| 809 if (hr == NULL) return false; | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
810 RegionIdx_t hr_ind = (RegionIdx_t) hr->hrs_index(); |
| 342 | 811 // Is this region in the coarse map? |
| 812 if (_coarse_map.at(hr_ind)) return true; | |
| 813 | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
814 PerRegionTable* prt = find_region_table(hr_ind & _mod_max_fine_entries_mask, |
| 342 | 815 hr); |
| 816 if (prt != NULL) { | |
| 817 return prt->contains_reference(from); | |
| 818 | |
| 819 } else { | |
| 820 uintptr_t from_card = | |
| 821 (uintptr_t(from) >> CardTableModRefBS::card_shift); | |
| 822 uintptr_t hr_bot_card_index = | |
| 823 uintptr_t(hr->bottom()) >> CardTableModRefBS::card_shift; | |
| 824 assert(from_card >= hr_bot_card_index, "Inv"); | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
825 CardIdx_t card_index = from_card - hr_bot_card_index; |
|
3986
65a8ff39a6da
7095194: G1: HeapRegion::GrainBytes, GrainWords, and CardsPerRegion should be size_t
johnc
parents:
3830
diff
changeset
|
826 assert(0 <= card_index && (size_t)card_index < HeapRegion::CardsPerRegion, |
|
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
827 "Must be in range."); |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
828 return _sparse_table.contains_card(hr_ind, card_index); |
| 342 | 829 } |
| 830 | |
| 831 | |
| 832 } | |
| 833 | |
| 2173 | 834 void |
| 835 OtherRegionsTable::do_cleanup_work(HRRSCleanupTask* hrrs_cleanup_task) { | |
| 836 _sparse_table.do_cleanup_work(hrrs_cleanup_task); | |
| 837 } | |
| 838 | |
|
795
215f81b4d9b3
6841831: G1: assert(contains_reference(from),"We just added it!") fires
iveresov
parents:
794
diff
changeset
|
839 // Determines how many threads can add records to an rset in parallel. |
|
215f81b4d9b3
6841831: G1: assert(contains_reference(from),"We just added it!") fires
iveresov
parents:
794
diff
changeset
|
840 // This can be done by either mutator threads together with the |
|
215f81b4d9b3
6841831: G1: assert(contains_reference(from),"We just added it!") fires
iveresov
parents:
794
diff
changeset
|
841 // concurrent refinement threads or GC threads. |
| 342 | 842 int HeapRegionRemSet::num_par_rem_sets() { |
|
795
215f81b4d9b3
6841831: G1: assert(contains_reference(from),"We just added it!") fires
iveresov
parents:
794
diff
changeset
|
843 return (int)MAX2(DirtyCardQueueSet::num_par_ids() + ConcurrentG1Refine::thread_num(), ParallelGCThreads); |
| 342 | 844 } |
| 845 | |
| 846 HeapRegionRemSet::HeapRegionRemSet(G1BlockOffsetSharedArray* bosa, | |
| 847 HeapRegion* hr) | |
|
3777
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
848 : _bosa(bosa), _other_regions(hr) { |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
849 reset_for_par_iteration(); |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
850 } |
| 342 | 851 |
|
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
852 void HeapRegionRemSet::setup_remset_size() { |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
853 // Setup sparse and fine-grain tables sizes. |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
854 // table_size = base * (log(region_size / 1M) + 1) |
|
6107
5c8bd7c16119
7171936: LOG_G incorrectly defined in globalDefinitions.hpp
brutisso
parents:
6010
diff
changeset
|
855 const int LOG_M = 20; |
|
5c8bd7c16119
7171936: LOG_G incorrectly defined in globalDefinitions.hpp
brutisso
parents:
6010
diff
changeset
|
856 int region_size_log_mb = MAX2(HeapRegion::LogOfHRGrainBytes - LOG_M, 0); |
|
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
857 if (FLAG_IS_DEFAULT(G1RSetSparseRegionEntries)) { |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
858 G1RSetSparseRegionEntries = G1RSetSparseRegionEntriesBase * (region_size_log_mb + 1); |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
859 } |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
860 if (FLAG_IS_DEFAULT(G1RSetRegionEntries)) { |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
861 G1RSetRegionEntries = G1RSetRegionEntriesBase * (region_size_log_mb + 1); |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
862 } |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
863 guarantee(G1RSetSparseRegionEntries > 0 && G1RSetRegionEntries > 0 , "Sanity"); |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
864 } |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
865 |
| 342 | 866 bool HeapRegionRemSet::claim_iter() { |
| 867 if (_iter_state != Unclaimed) return false; | |
| 868 jint res = Atomic::cmpxchg(Claimed, (jint*)(&_iter_state), Unclaimed); | |
| 869 return (res == Unclaimed); | |
| 870 } | |
| 871 | |
| 872 void HeapRegionRemSet::set_iter_complete() { | |
| 873 _iter_state = Complete; | |
| 874 } | |
| 875 | |
| 876 bool HeapRegionRemSet::iter_is_complete() { | |
| 877 return _iter_state == Complete; | |
| 878 } | |
| 879 | |
| 880 void HeapRegionRemSet::init_iterator(HeapRegionRemSetIterator* iter) const { | |
| 881 iter->initialize(this); | |
| 882 } | |
| 883 | |
| 884 #ifndef PRODUCT | |
| 885 void HeapRegionRemSet::print() const { | |
| 886 HeapRegionRemSetIterator iter; | |
| 887 init_iterator(&iter); | |
| 888 size_t card_index; | |
| 889 while (iter.has_next(card_index)) { | |
| 890 HeapWord* card_start = | |
| 891 G1CollectedHeap::heap()->bot_shared()->address_for_index(card_index); | |
|
3777
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
892 gclog_or_tty->print_cr(" Card " PTR_FORMAT, card_start); |
| 342 | 893 } |
| 894 if (iter.n_yielded() != occupied()) { | |
| 895 gclog_or_tty->print_cr("Yielded disagrees with occupied:"); | |
| 896 gclog_or_tty->print_cr(" %6d yielded (%6d coarse, %6d fine).", | |
| 897 iter.n_yielded(), | |
| 898 iter.n_yielded_coarse(), iter.n_yielded_fine()); | |
| 899 gclog_or_tty->print_cr(" %6d occ (%6d coarse, %6d fine).", | |
| 900 occupied(), occ_coarse(), occ_fine()); | |
| 901 } | |
| 902 guarantee(iter.n_yielded() == occupied(), | |
| 903 "We should have yielded all the represented cards."); | |
| 904 } | |
| 905 #endif | |
| 906 | |
| 907 void HeapRegionRemSet::cleanup() { | |
| 908 SparsePRT::cleanup_all(); | |
| 909 } | |
| 910 | |
| 911 void HeapRegionRemSet::clear() { | |
| 912 _other_regions.clear(); | |
| 913 assert(occupied() == 0, "Should be clear."); | |
|
3777
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
914 reset_for_par_iteration(); |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
915 } |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
916 |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
917 void HeapRegionRemSet::reset_for_par_iteration() { |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
918 _iter_state = Unclaimed; |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
919 _iter_claimed = 0; |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
920 // It's good to check this to make sure that the two methods are in sync. |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
921 assert(verify_ready_for_par_iteration(), "post-condition"); |
| 342 | 922 } |
| 923 | |
| 924 void HeapRegionRemSet::scrub(CardTableModRefBS* ctbs, | |
| 925 BitMap* region_bm, BitMap* card_bm) { | |
| 926 _other_regions.scrub(ctbs, region_bm, card_bm); | |
| 927 } | |
| 928 | |
| 929 //-------------------- Iteration -------------------- | |
| 930 | |
| 931 HeapRegionRemSetIterator:: | |
| 932 HeapRegionRemSetIterator() : | |
| 933 _hrrs(NULL), | |
| 934 _g1h(G1CollectedHeap::heap()), | |
| 935 _bosa(NULL), | |
|
1884
9f4848ebbabd
6992189: G1: inconsistent base used in sparse rem set iterator
tonyp
parents:
1552
diff
changeset
|
936 _sparse_iter() { } |
| 342 | 937 |
| 938 void HeapRegionRemSetIterator::initialize(const HeapRegionRemSet* hrrs) { | |
| 939 _hrrs = hrrs; | |
| 940 _coarse_map = &_hrrs->_other_regions._coarse_map; | |
| 941 _fine_grain_regions = _hrrs->_other_regions._fine_grain_regions; | |
| 942 _bosa = _hrrs->bosa(); | |
| 943 | |
| 944 _is = Sparse; | |
| 945 // Set these values so that we increment to the first region. | |
| 946 _coarse_cur_region_index = -1; | |
|
3986
65a8ff39a6da
7095194: G1: HeapRegion::GrainBytes, GrainWords, and CardsPerRegion should be size_t
johnc
parents:
3830
diff
changeset
|
947 _coarse_cur_region_cur_card = (HeapRegion::CardsPerRegion-1); |
| 342 | 948 |
| 949 _cur_region_cur_card = 0; | |
| 950 | |
| 951 _fine_array_index = -1; | |
| 952 _fine_cur_prt = NULL; | |
| 953 | |
| 954 _n_yielded_coarse = 0; | |
| 955 _n_yielded_fine = 0; | |
| 956 _n_yielded_sparse = 0; | |
| 957 | |
| 958 _sparse_iter.init(&hrrs->_other_regions._sparse_table); | |
| 959 } | |
| 960 | |
| 961 bool HeapRegionRemSetIterator::coarse_has_next(size_t& card_index) { | |
| 962 if (_hrrs->_other_regions._n_coarse_entries == 0) return false; | |
| 963 // Go to the next card. | |
| 964 _coarse_cur_region_cur_card++; | |
| 965 // Was the last the last card in the current region? | |
|
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
966 if (_coarse_cur_region_cur_card == HeapRegion::CardsPerRegion) { |
| 342 | 967 // Yes: find the next region. This may leave _coarse_cur_region_index |
| 968 // Set to the last index, in which case there are no more coarse | |
| 969 // regions. | |
| 970 _coarse_cur_region_index = | |
| 971 (int) _coarse_map->get_next_one_offset(_coarse_cur_region_index + 1); | |
| 972 if ((size_t)_coarse_cur_region_index < _coarse_map->size()) { | |
| 973 _coarse_cur_region_cur_card = 0; | |
| 974 HeapWord* r_bot = | |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
975 _g1h->region_at((uint) _coarse_cur_region_index)->bottom(); |
| 342 | 976 _cur_region_card_offset = _bosa->index_for(r_bot); |
| 977 } else { | |
| 978 return false; | |
| 979 } | |
| 980 } | |
| 981 // If we didn't return false above, then we can yield a card. | |
| 982 card_index = _cur_region_card_offset + _coarse_cur_region_cur_card; | |
| 983 return true; | |
| 984 } | |
| 985 | |
| 986 void HeapRegionRemSetIterator::fine_find_next_non_null_prt() { | |
| 987 // Otherwise, find the next bucket list in the array. | |
| 988 _fine_array_index++; | |
| 989 while (_fine_array_index < (int) OtherRegionsTable::_max_fine_entries) { | |
| 990 _fine_cur_prt = _fine_grain_regions[_fine_array_index]; | |
| 991 if (_fine_cur_prt != NULL) return; | |
| 992 else _fine_array_index++; | |
| 993 } | |
| 994 assert(_fine_cur_prt == NULL, "Loop post"); | |
| 995 } | |
| 996 | |
| 997 bool HeapRegionRemSetIterator::fine_has_next(size_t& card_index) { | |
| 998 if (fine_has_next()) { | |
| 999 _cur_region_cur_card = | |
| 1000 _fine_cur_prt->_bm.get_next_one_offset(_cur_region_cur_card + 1); | |
| 1001 } | |
| 1002 while (!fine_has_next()) { | |
|
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
1003 if (_cur_region_cur_card == (size_t) HeapRegion::CardsPerRegion) { |
| 342 | 1004 _cur_region_cur_card = 0; |
| 6253 | 1005 _fine_cur_prt = _fine_cur_prt->collision_list_next(); |
| 342 | 1006 } |
| 1007 if (_fine_cur_prt == NULL) { | |
| 1008 fine_find_next_non_null_prt(); | |
| 1009 if (_fine_cur_prt == NULL) return false; | |
| 1010 } | |
| 1011 assert(_fine_cur_prt != NULL && _cur_region_cur_card == 0, | |
| 1012 "inv."); | |
| 1013 HeapWord* r_bot = | |
| 1014 _fine_cur_prt->hr()->bottom(); | |
| 1015 _cur_region_card_offset = _bosa->index_for(r_bot); | |
| 1016 _cur_region_cur_card = _fine_cur_prt->_bm.get_next_one_offset(0); | |
| 1017 } | |
| 1018 assert(fine_has_next(), "Or else we exited the loop via the return."); | |
| 1019 card_index = _cur_region_card_offset + _cur_region_cur_card; | |
| 1020 return true; | |
| 1021 } | |
| 1022 | |
| 1023 bool HeapRegionRemSetIterator::fine_has_next() { | |
| 1024 return | |
| 1025 _fine_cur_prt != NULL && | |
|
3986
65a8ff39a6da
7095194: G1: HeapRegion::GrainBytes, GrainWords, and CardsPerRegion should be size_t
johnc
parents:
3830
diff
changeset
|
1026 _cur_region_cur_card < HeapRegion::CardsPerRegion; |
| 342 | 1027 } |
| 1028 | |
| 1029 bool HeapRegionRemSetIterator::has_next(size_t& card_index) { | |
| 1030 switch (_is) { | |
| 1031 case Sparse: | |
| 1032 if (_sparse_iter.has_next(card_index)) { | |
| 1033 _n_yielded_sparse++; | |
| 1034 return true; | |
| 1035 } | |
| 1036 // Otherwise, deliberate fall-through | |
| 1037 _is = Fine; | |
| 1038 case Fine: | |
| 1039 if (fine_has_next(card_index)) { | |
| 1040 _n_yielded_fine++; | |
| 1041 return true; | |
| 1042 } | |
| 1043 // Otherwise, deliberate fall-through | |
| 1044 _is = Coarse; | |
| 1045 case Coarse: | |
| 1046 if (coarse_has_next(card_index)) { | |
| 1047 _n_yielded_coarse++; | |
| 1048 return true; | |
| 1049 } | |
| 1050 // Otherwise... | |
| 1051 break; | |
| 1052 } | |
| 1053 assert(ParallelGCThreads > 1 || | |
| 1054 n_yielded() == _hrrs->occupied(), | |
| 1055 "Should have yielded all the cards in the rem set " | |
| 1056 "(in the non-par case)."); | |
| 1057 return false; | |
| 1058 } | |
| 1059 | |
| 1060 | |
| 1061 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1062 OopOrNarrowOopStar* HeapRegionRemSet::_recorded_oops = NULL; |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1063 HeapWord** HeapRegionRemSet::_recorded_cards = NULL; |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1064 HeapRegion** HeapRegionRemSet::_recorded_regions = NULL; |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1065 int HeapRegionRemSet::_n_recorded = 0; |
| 342 | 1066 |
| 1067 HeapRegionRemSet::Event* HeapRegionRemSet::_recorded_events = NULL; | |
| 1068 int* HeapRegionRemSet::_recorded_event_index = NULL; | |
| 1069 int HeapRegionRemSet::_n_recorded_events = 0; | |
| 1070 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1071 void HeapRegionRemSet::record(HeapRegion* hr, OopOrNarrowOopStar f) { |
| 342 | 1072 if (_recorded_oops == NULL) { |
| 1073 assert(_n_recorded == 0 | |
| 1074 && _recorded_cards == NULL | |
| 1075 && _recorded_regions == NULL, | |
| 1076 "Inv"); | |
| 6197 | 1077 _recorded_oops = NEW_C_HEAP_ARRAY(OopOrNarrowOopStar, MaxRecorded, mtGC); |
| 1078 _recorded_cards = NEW_C_HEAP_ARRAY(HeapWord*, MaxRecorded, mtGC); | |
| 1079 _recorded_regions = NEW_C_HEAP_ARRAY(HeapRegion*, MaxRecorded, mtGC); | |
| 342 | 1080 } |
| 1081 if (_n_recorded == MaxRecorded) { | |
| 1082 gclog_or_tty->print_cr("Filled up 'recorded' (%d).", MaxRecorded); | |
| 1083 } else { | |
| 1084 _recorded_cards[_n_recorded] = | |
| 1085 (HeapWord*)align_size_down(uintptr_t(f), | |
| 1086 CardTableModRefBS::card_size); | |
| 1087 _recorded_oops[_n_recorded] = f; | |
| 1088 _recorded_regions[_n_recorded] = hr; | |
| 1089 _n_recorded++; | |
| 1090 } | |
| 1091 } | |
| 1092 | |
| 1093 void HeapRegionRemSet::record_event(Event evnt) { | |
| 1094 if (!G1RecordHRRSEvents) return; | |
| 1095 | |
| 1096 if (_recorded_events == NULL) { | |
| 1097 assert(_n_recorded_events == 0 | |
| 1098 && _recorded_event_index == NULL, | |
| 1099 "Inv"); | |
| 6197 | 1100 _recorded_events = NEW_C_HEAP_ARRAY(Event, MaxRecordedEvents, mtGC); |
| 1101 _recorded_event_index = NEW_C_HEAP_ARRAY(int, MaxRecordedEvents, mtGC); | |
| 342 | 1102 } |
| 1103 if (_n_recorded_events == MaxRecordedEvents) { | |
| 1104 gclog_or_tty->print_cr("Filled up 'recorded_events' (%d).", MaxRecordedEvents); | |
| 1105 } else { | |
| 1106 _recorded_events[_n_recorded_events] = evnt; | |
| 1107 _recorded_event_index[_n_recorded_events] = _n_recorded; | |
| 1108 _n_recorded_events++; | |
| 1109 } | |
| 1110 } | |
| 1111 | |
| 1112 void HeapRegionRemSet::print_event(outputStream* str, Event evnt) { | |
| 1113 switch (evnt) { | |
| 1114 case Event_EvacStart: | |
| 1115 str->print("Evac Start"); | |
| 1116 break; | |
| 1117 case Event_EvacEnd: | |
| 1118 str->print("Evac End"); | |
| 1119 break; | |
| 1120 case Event_RSUpdateEnd: | |
| 1121 str->print("RS Update End"); | |
| 1122 break; | |
| 1123 } | |
| 1124 } | |
| 1125 | |
| 1126 void HeapRegionRemSet::print_recorded() { | |
| 1127 int cur_evnt = 0; | |
| 1128 Event cur_evnt_kind; | |
| 1129 int cur_evnt_ind = 0; | |
| 1130 if (_n_recorded_events > 0) { | |
| 1131 cur_evnt_kind = _recorded_events[cur_evnt]; | |
| 1132 cur_evnt_ind = _recorded_event_index[cur_evnt]; | |
| 1133 } | |
| 1134 | |
| 1135 for (int i = 0; i < _n_recorded; i++) { | |
| 1136 while (cur_evnt < _n_recorded_events && i == cur_evnt_ind) { | |
| 1137 gclog_or_tty->print("Event: "); | |
| 1138 print_event(gclog_or_tty, cur_evnt_kind); | |
| 1139 gclog_or_tty->print_cr(""); | |
| 1140 cur_evnt++; | |
| 1141 if (cur_evnt < MaxRecordedEvents) { | |
| 1142 cur_evnt_kind = _recorded_events[cur_evnt]; | |
| 1143 cur_evnt_ind = _recorded_event_index[cur_evnt]; | |
| 1144 } | |
| 1145 } | |
| 1146 gclog_or_tty->print("Added card " PTR_FORMAT " to region [" PTR_FORMAT "...]" | |
| 1147 " for ref " PTR_FORMAT ".\n", | |
| 1148 _recorded_cards[i], _recorded_regions[i]->bottom(), | |
| 1149 _recorded_oops[i]); | |
| 1150 } | |
| 1151 } | |
| 1152 | |
| 2173 | 1153 void HeapRegionRemSet::reset_for_cleanup_tasks() { |
| 1154 SparsePRT::reset_for_cleanup_tasks(); | |
| 1155 } | |
| 1156 | |
| 1157 void HeapRegionRemSet::do_cleanup_work(HRRSCleanupTask* hrrs_cleanup_task) { | |
| 1158 _other_regions.do_cleanup_work(hrrs_cleanup_task); | |
| 1159 } | |
| 1160 | |
| 1161 void | |
| 1162 HeapRegionRemSet::finish_cleanup_task(HRRSCleanupTask* hrrs_cleanup_task) { | |
| 1163 SparsePRT::finish_cleanup_task(hrrs_cleanup_task); | |
| 1164 } | |
| 1165 | |
| 342 | 1166 #ifndef PRODUCT |
| 1167 void HeapRegionRemSet::test() { | |
| 1168 os::sleep(Thread::current(), (jlong)5000, false); | |
| 1169 G1CollectedHeap* g1h = G1CollectedHeap::heap(); | |
| 1170 | |
|
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1171 // Run with "-XX:G1LogRSetRegionEntries=2", so that 1 and 5 end up in same |
| 342 | 1172 // hash bucket. |
| 1173 HeapRegion* hr0 = g1h->region_at(0); | |
| 1174 HeapRegion* hr1 = g1h->region_at(1); | |
| 1175 HeapRegion* hr2 = g1h->region_at(5); | |
| 1176 HeapRegion* hr3 = g1h->region_at(6); | |
| 1177 HeapRegion* hr4 = g1h->region_at(7); | |
| 1178 HeapRegion* hr5 = g1h->region_at(8); | |
| 1179 | |
| 1180 HeapWord* hr1_start = hr1->bottom(); | |
| 1181 HeapWord* hr1_mid = hr1_start + HeapRegion::GrainWords/2; | |
| 1182 HeapWord* hr1_last = hr1->end() - 1; | |
| 1183 | |
| 1184 HeapWord* hr2_start = hr2->bottom(); | |
| 1185 HeapWord* hr2_mid = hr2_start + HeapRegion::GrainWords/2; | |
| 1186 HeapWord* hr2_last = hr2->end() - 1; | |
| 1187 | |
| 1188 HeapWord* hr3_start = hr3->bottom(); | |
| 1189 HeapWord* hr3_mid = hr3_start + HeapRegion::GrainWords/2; | |
| 1190 HeapWord* hr3_last = hr3->end() - 1; | |
| 1191 | |
| 1192 HeapRegionRemSet* hrrs = hr0->rem_set(); | |
| 1193 | |
| 1194 // Make three references from region 0x101... | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1195 hrrs->add_reference((OopOrNarrowOopStar)hr1_start); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1196 hrrs->add_reference((OopOrNarrowOopStar)hr1_mid); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1197 hrrs->add_reference((OopOrNarrowOopStar)hr1_last); |
| 342 | 1198 |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1199 hrrs->add_reference((OopOrNarrowOopStar)hr2_start); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1200 hrrs->add_reference((OopOrNarrowOopStar)hr2_mid); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1201 hrrs->add_reference((OopOrNarrowOopStar)hr2_last); |
| 342 | 1202 |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1203 hrrs->add_reference((OopOrNarrowOopStar)hr3_start); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1204 hrrs->add_reference((OopOrNarrowOopStar)hr3_mid); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1205 hrrs->add_reference((OopOrNarrowOopStar)hr3_last); |
| 342 | 1206 |
| 1207 // Now cause a coarsening. | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1208 hrrs->add_reference((OopOrNarrowOopStar)hr4->bottom()); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1209 hrrs->add_reference((OopOrNarrowOopStar)hr5->bottom()); |
| 342 | 1210 |
| 1211 // Now, does iteration yield these three? | |
| 1212 HeapRegionRemSetIterator iter; | |
| 1213 hrrs->init_iterator(&iter); | |
| 1214 size_t sum = 0; | |
| 1215 size_t card_index; | |
| 1216 while (iter.has_next(card_index)) { | |
| 1217 HeapWord* card_start = | |
| 1218 G1CollectedHeap::heap()->bot_shared()->address_for_index(card_index); | |
| 1219 gclog_or_tty->print_cr(" Card " PTR_FORMAT ".", card_start); | |
| 1220 sum++; | |
| 1221 } | |
| 1222 guarantee(sum == 11 - 3 + 2048, "Failure"); | |
| 1223 guarantee(sum == hrrs->occupied(), "Failure"); | |
| 1224 } | |
| 1225 #endif |