Mercurial > hg > graal-jvmci-8
annotate src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp @ 20211:82693fb204a5
8038930: G1CodeRootSet::test fails with assert(_num_chunks_handed_out == 0) failed: No elements must have been handed out yet
Summary: The test incorrectly assumed that it had been started with no other previous compilation activity. Fix this by allowing multiple code root free chunk lists, and use one separate from the global one to perform the test.
Reviewed-by: brutisso
| author | tschatzl |
|---|---|
| date | Wed, 16 Apr 2014 10:14:50 +0200 |
| parents | 78bbf4d43a14 |
| children | ff7b317d2af8 |
| rev | line source |
|---|---|
| 342 | 1 /* |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
2 * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved. |
| 342 | 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| 4 * | |
| 5 * This code is free software; you can redistribute it and/or modify it | |
| 6 * under the terms of the GNU General Public License version 2 only, as | |
| 7 * published by the Free Software Foundation. | |
| 8 * | |
| 9 * This code is distributed in the hope that it will be useful, but WITHOUT | |
| 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
| 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
| 12 * version 2 for more details (a copy is included in the LICENSE file that | |
| 13 * accompanied this code). | |
| 14 * | |
| 15 * You should have received a copy of the GNU General Public License version | |
| 16 * 2 along with this work; if not, write to the Free Software Foundation, | |
| 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |
| 18 * | |
|
1552
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1390
diff
changeset
|
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
|
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1390
diff
changeset
|
20 * or visit www.oracle.com if you need additional information or have any |
|
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1390
diff
changeset
|
21 * questions. |
| 342 | 22 * |
| 23 */ | |
| 24 | |
| 1972 | 25 #include "precompiled.hpp" |
| 26 #include "gc_implementation/g1/concurrentG1Refine.hpp" | |
| 27 #include "gc_implementation/g1/g1BlockOffsetTable.inline.hpp" | |
| 28 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp" | |
| 29 #include "gc_implementation/g1/heapRegionRemSet.hpp" | |
| 30 #include "gc_implementation/g1/heapRegionSeq.inline.hpp" | |
| 31 #include "memory/allocation.hpp" | |
|
17754
d7070f371770
8035815: Cache-align and pad the from card cache
tschatzl
parents:
17753
diff
changeset
|
32 #include "memory/padded.inline.hpp" |
| 1972 | 33 #include "memory/space.inline.hpp" |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
34 #include "oops/oop.inline.hpp" |
| 1972 | 35 #include "utilities/bitMap.inline.hpp" |
| 36 #include "utilities/globalDefinitions.hpp" | |
| 12080 | 37 #include "utilities/growableArray.hpp" |
| 342 | 38 |
|
17937
78bbf4d43a14
8037816: Fix for 8036122 breaks build with Xcode5/clang
drchase
parents:
17844
diff
changeset
|
39 PRAGMA_FORMAT_MUTE_WARNINGS_FOR_GCC |
|
78bbf4d43a14
8037816: Fix for 8036122 breaks build with Xcode5/clang
drchase
parents:
17844
diff
changeset
|
40 |
| 6197 | 41 class PerRegionTable: public CHeapObj<mtGC> { |
| 342 | 42 friend class OtherRegionsTable; |
| 43 friend class HeapRegionRemSetIterator; | |
| 44 | |
| 45 HeapRegion* _hr; | |
| 46 BitMap _bm; | |
| 47 jint _occupied; | |
| 48 | |
| 6253 | 49 // next pointer for free/allocated 'all' list |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
50 PerRegionTable* _next; |
| 342 | 51 |
| 6253 | 52 // prev pointer for the allocated 'all' list |
| 53 PerRegionTable* _prev; | |
| 342 | 54 |
| 6253 | 55 // next pointer in collision list |
| 56 PerRegionTable * _collision_list_next; | |
| 342 | 57 |
| 6253 | 58 // Global free list of PRTs |
| 59 static PerRegionTable* _free_list; | |
| 342 | 60 |
| 61 protected: | |
| 62 // We need access in order to union things into the base table. | |
| 63 BitMap* bm() { return &_bm; } | |
| 64 | |
| 65 void recount_occupied() { | |
| 66 _occupied = (jint) bm()->count_one_bits(); | |
| 67 } | |
| 68 | |
| 69 PerRegionTable(HeapRegion* hr) : | |
| 70 _hr(hr), | |
| 71 _occupied(0), | |
| 6253 | 72 _bm(HeapRegion::CardsPerRegion, false /* in-resource-area */), |
| 73 _collision_list_next(NULL), _next(NULL), _prev(NULL) | |
| 342 | 74 {} |
| 75 | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
76 void add_card_work(CardIdx_t from_card, bool par) { |
| 342 | 77 if (!_bm.at(from_card)) { |
| 78 if (par) { | |
| 79 if (_bm.par_at_put(from_card, 1)) { | |
| 80 Atomic::inc(&_occupied); | |
| 81 } | |
| 82 } else { | |
| 83 _bm.at_put(from_card, 1); | |
| 84 _occupied++; | |
| 85 } | |
| 86 } | |
| 87 } | |
| 88 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
89 void add_reference_work(OopOrNarrowOopStar from, bool par) { |
| 342 | 90 // Must make this robust in case "from" is not in "_hr", because of |
| 91 // concurrency. | |
| 92 | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
93 if (G1TraceHeapRegionRememberedSet) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
94 gclog_or_tty->print_cr(" PRT::Add_reference_work(" PTR_FORMAT "->" PTR_FORMAT").", |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
95 from, |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
96 UseCompressedOops |
|
12316
190899198332
7195622: CheckUnhandledOops has limited usefulness now
hseigel
parents:
12080
diff
changeset
|
97 ? (void *)oopDesc::load_decode_heap_oop((narrowOop*)from) |
|
190899198332
7195622: CheckUnhandledOops has limited usefulness now
hseigel
parents:
12080
diff
changeset
|
98 : (void *)oopDesc::load_decode_heap_oop((oop*)from)); |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
99 } |
| 342 | 100 |
| 101 HeapRegion* loc_hr = hr(); | |
| 102 // If the test below fails, then this table was reused concurrently | |
| 103 // with this operation. This is OK, since the old table was coarsened, | |
| 104 // and adding a bit to the new table is never incorrect. | |
|
4020
5e5d4821bf07
7097516: G1: assert(0<= from_card && from_card<HeapRegion::CardsPerRegion) failed: Must be in range.
brutisso
parents:
3986
diff
changeset
|
105 // If the table used to belong to a continues humongous region and is |
|
5e5d4821bf07
7097516: G1: assert(0<= from_card && from_card<HeapRegion::CardsPerRegion) failed: Must be in range.
brutisso
parents:
3986
diff
changeset
|
106 // now reused for the corresponding start humongous region, we need to |
|
5e5d4821bf07
7097516: G1: assert(0<= from_card && from_card<HeapRegion::CardsPerRegion) failed: Must be in range.
brutisso
parents:
3986
diff
changeset
|
107 // make sure that we detect this. Thus, we call is_in_reserved_raw() |
|
5e5d4821bf07
7097516: G1: assert(0<= from_card && from_card<HeapRegion::CardsPerRegion) failed: Must be in range.
brutisso
parents:
3986
diff
changeset
|
108 // instead of just is_in_reserved() here. |
|
5e5d4821bf07
7097516: G1: assert(0<= from_card && from_card<HeapRegion::CardsPerRegion) failed: Must be in range.
brutisso
parents:
3986
diff
changeset
|
109 if (loc_hr->is_in_reserved_raw(from)) { |
| 342 | 110 size_t hw_offset = pointer_delta((HeapWord*)from, loc_hr->bottom()); |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
111 CardIdx_t from_card = (CardIdx_t) |
|
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
112 hw_offset >> (CardTableModRefBS::card_shift - LogHeapWordSize); |
| 342 | 113 |
|
3986
65a8ff39a6da
7095194: G1: HeapRegion::GrainBytes, GrainWords, and CardsPerRegion should be size_t
johnc
parents:
3830
diff
changeset
|
114 assert(0 <= from_card && (size_t)from_card < HeapRegion::CardsPerRegion, |
|
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
115 "Must be in range."); |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
116 add_card_work(from_card, par); |
| 342 | 117 } |
| 118 } | |
| 119 | |
| 120 public: | |
| 121 | |
| 122 HeapRegion* hr() const { return _hr; } | |
| 123 | |
| 124 jint occupied() const { | |
| 125 // Overkill, but if we ever need it... | |
| 126 // guarantee(_occupied == _bm.count_one_bits(), "Check"); | |
| 127 return _occupied; | |
| 128 } | |
| 129 | |
| 6253 | 130 void init(HeapRegion* hr, bool clear_links_to_all_list) { |
| 131 if (clear_links_to_all_list) { | |
| 132 set_next(NULL); | |
| 133 set_prev(NULL); | |
| 134 } | |
| 342 | 135 _hr = hr; |
| 6253 | 136 _collision_list_next = NULL; |
| 342 | 137 _occupied = 0; |
| 138 _bm.clear(); | |
| 139 } | |
| 140 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
141 void add_reference(OopOrNarrowOopStar from) { |
| 342 | 142 add_reference_work(from, /*parallel*/ true); |
| 143 } | |
| 144 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
145 void seq_add_reference(OopOrNarrowOopStar from) { |
| 342 | 146 add_reference_work(from, /*parallel*/ false); |
| 147 } | |
| 148 | |
| 149 void scrub(CardTableModRefBS* ctbs, BitMap* card_bm) { | |
| 150 HeapWord* hr_bot = hr()->bottom(); | |
|
489
2494ab195856
6653214: MemoryPoolMXBean.setUsageThreshold() does not support large heap sizes.
swamyv
parents:
342
diff
changeset
|
151 size_t hr_first_card_index = ctbs->index_for(hr_bot); |
| 342 | 152 bm()->set_intersection_at_offset(*card_bm, hr_first_card_index); |
| 153 recount_occupied(); | |
| 154 } | |
| 155 | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
156 void add_card(CardIdx_t from_card_index) { |
| 342 | 157 add_card_work(from_card_index, /*parallel*/ true); |
| 158 } | |
| 159 | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
160 void seq_add_card(CardIdx_t from_card_index) { |
| 342 | 161 add_card_work(from_card_index, /*parallel*/ false); |
| 162 } | |
| 163 | |
| 164 // (Destructively) union the bitmap of the current table into the given | |
| 165 // bitmap (which is assumed to be of the same size.) | |
| 166 void union_bitmap_into(BitMap* bm) { | |
| 167 bm->set_union(_bm); | |
| 168 } | |
| 169 | |
| 170 // Mem size in bytes. | |
| 171 size_t mem_size() const { | |
| 172 return sizeof(this) + _bm.size_in_words() * HeapWordSize; | |
| 173 } | |
| 174 | |
| 175 // Requires "from" to be in "hr()". | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
176 bool contains_reference(OopOrNarrowOopStar from) const { |
| 342 | 177 assert(hr()->is_in_reserved(from), "Precondition."); |
| 178 size_t card_ind = pointer_delta(from, hr()->bottom(), | |
| 179 CardTableModRefBS::card_size); | |
| 180 return _bm.at(card_ind); | |
| 181 } | |
| 182 | |
| 6253 | 183 // Bulk-free the PRTs from prt to last, assumes that they are |
| 184 // linked together using their _next field. | |
| 185 static void bulk_free(PerRegionTable* prt, PerRegionTable* last) { | |
| 342 | 186 while (true) { |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
187 PerRegionTable* fl = _free_list; |
| 6253 | 188 last->set_next(fl); |
| 189 PerRegionTable* res = (PerRegionTable*) Atomic::cmpxchg_ptr(prt, &_free_list, fl); | |
| 190 if (res == fl) { | |
| 191 return; | |
| 192 } | |
| 342 | 193 } |
| 194 ShouldNotReachHere(); | |
| 195 } | |
| 196 | |
| 6253 | 197 static void free(PerRegionTable* prt) { |
| 198 bulk_free(prt, prt); | |
| 199 } | |
| 200 | |
| 201 // Returns an initialized PerRegionTable instance. | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
202 static PerRegionTable* alloc(HeapRegion* hr) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
203 PerRegionTable* fl = _free_list; |
| 342 | 204 while (fl != NULL) { |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
205 PerRegionTable* nxt = fl->next(); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
206 PerRegionTable* res = |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
207 (PerRegionTable*) |
| 342 | 208 Atomic::cmpxchg_ptr(nxt, &_free_list, fl); |
| 209 if (res == fl) { | |
| 6253 | 210 fl->init(hr, true); |
| 342 | 211 return fl; |
| 212 } else { | |
| 213 fl = _free_list; | |
| 214 } | |
| 215 } | |
| 216 assert(fl == NULL, "Loop condition."); | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
217 return new PerRegionTable(hr); |
| 342 | 218 } |
| 219 | |
| 6253 | 220 PerRegionTable* next() const { return _next; } |
| 221 void set_next(PerRegionTable* next) { _next = next; } | |
| 222 PerRegionTable* prev() const { return _prev; } | |
| 223 void set_prev(PerRegionTable* prev) { _prev = prev; } | |
| 224 | |
| 225 // Accessor and Modification routines for the pointer for the | |
| 226 // singly linked collision list that links the PRTs within the | |
| 227 // OtherRegionsTable::_fine_grain_regions hash table. | |
| 228 // | |
| 229 // It might be useful to also make the collision list doubly linked | |
| 230 // to avoid iteration over the collisions list during scrubbing/deletion. | |
| 231 // OTOH there might not be many collisions. | |
| 232 | |
| 233 PerRegionTable* collision_list_next() const { | |
| 234 return _collision_list_next; | |
| 235 } | |
| 236 | |
| 237 void set_collision_list_next(PerRegionTable* next) { | |
| 238 _collision_list_next = next; | |
| 239 } | |
| 240 | |
| 241 PerRegionTable** collision_list_next_addr() { | |
| 242 return &_collision_list_next; | |
| 243 } | |
| 244 | |
| 342 | 245 static size_t fl_mem_size() { |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
246 PerRegionTable* cur = _free_list; |
| 342 | 247 size_t res = 0; |
| 248 while (cur != NULL) { | |
|
10333
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
249 res += cur->mem_size(); |
| 342 | 250 cur = cur->next(); |
| 251 } | |
| 252 return res; | |
| 253 } | |
|
10333
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
254 |
|
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
255 static void test_fl_mem_size(); |
| 342 | 256 }; |
| 257 | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
258 PerRegionTable* PerRegionTable::_free_list = NULL; |
| 342 | 259 |
| 260 size_t OtherRegionsTable::_max_fine_entries = 0; | |
| 261 size_t OtherRegionsTable::_mod_max_fine_entries_mask = 0; | |
| 262 size_t OtherRegionsTable::_fine_eviction_stride = 0; | |
| 263 size_t OtherRegionsTable::_fine_eviction_sample_size = 0; | |
| 264 | |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
265 OtherRegionsTable::OtherRegionsTable(HeapRegion* hr, Mutex* m) : |
| 342 | 266 _g1h(G1CollectedHeap::heap()), |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
267 _hr(hr), _m(m), |
| 342 | 268 _coarse_map(G1CollectedHeap::heap()->max_regions(), |
| 269 false /* in-resource-area */), | |
| 270 _fine_grain_regions(NULL), | |
| 6253 | 271 _first_all_fine_prts(NULL), _last_all_fine_prts(NULL), |
| 342 | 272 _n_fine_entries(0), _n_coarse_entries(0), |
| 273 _fine_eviction_start(0), | |
| 274 _sparse_table(hr) | |
| 275 { | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
276 typedef PerRegionTable* PerRegionTablePtr; |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
277 |
| 342 | 278 if (_max_fine_entries == 0) { |
| 279 assert(_mod_max_fine_entries_mask == 0, "Both or none."); | |
|
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
280 size_t max_entries_log = (size_t)log2_long((jlong)G1RSetRegionEntries); |
|
6750
859cd1a76f8a
7197906: BlockOffsetArray::power_to_cards_back() needs to handle > 32 bit shifts
brutisso
parents:
6253
diff
changeset
|
281 _max_fine_entries = (size_t)1 << max_entries_log; |
| 342 | 282 _mod_max_fine_entries_mask = _max_fine_entries - 1; |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
283 |
| 342 | 284 assert(_fine_eviction_sample_size == 0 |
| 285 && _fine_eviction_stride == 0, "All init at same time."); | |
|
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
286 _fine_eviction_sample_size = MAX2((size_t)4, max_entries_log); |
| 342 | 287 _fine_eviction_stride = _max_fine_entries / _fine_eviction_sample_size; |
| 288 } | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
289 |
|
10271
f9be75d21404
8012902: remove use of global operator new - take 2
minqi
parents:
10193
diff
changeset
|
290 _fine_grain_regions = NEW_C_HEAP_ARRAY3(PerRegionTablePtr, _max_fine_entries, |
|
f9be75d21404
8012902: remove use of global operator new - take 2
minqi
parents:
10193
diff
changeset
|
291 mtGC, 0, AllocFailStrategy::RETURN_NULL); |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
292 |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
293 if (_fine_grain_regions == NULL) { |
|
10161
746b070f5022
8011661: Insufficient memory message says "malloc" when sometimes it should say "mmap"
ccheung
parents:
10135
diff
changeset
|
294 vm_exit_out_of_memory(sizeof(void*)*_max_fine_entries, OOM_MALLOC_ERROR, |
| 342 | 295 "Failed to allocate _fine_grain_entries."); |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
296 } |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
297 |
| 342 | 298 for (size_t i = 0; i < _max_fine_entries; i++) { |
| 299 _fine_grain_regions[i] = NULL; | |
| 300 } | |
| 301 } | |
| 302 | |
| 6253 | 303 void OtherRegionsTable::link_to_all(PerRegionTable* prt) { |
| 304 // We always append to the beginning of the list for convenience; | |
| 305 // the order of entries in this list does not matter. | |
| 306 if (_first_all_fine_prts != NULL) { | |
| 307 assert(_first_all_fine_prts->prev() == NULL, "invariant"); | |
| 308 _first_all_fine_prts->set_prev(prt); | |
| 309 prt->set_next(_first_all_fine_prts); | |
| 310 } else { | |
| 311 // this is the first element we insert. Adjust the "last" pointer | |
| 312 _last_all_fine_prts = prt; | |
| 313 assert(prt->next() == NULL, "just checking"); | |
| 314 } | |
| 315 // the new element is always the first element without a predecessor | |
| 316 prt->set_prev(NULL); | |
| 317 _first_all_fine_prts = prt; | |
| 318 | |
| 319 assert(prt->prev() == NULL, "just checking"); | |
| 320 assert(_first_all_fine_prts == prt, "just checking"); | |
| 321 assert((_first_all_fine_prts == NULL && _last_all_fine_prts == NULL) || | |
| 322 (_first_all_fine_prts != NULL && _last_all_fine_prts != NULL), | |
| 323 "just checking"); | |
| 324 assert(_last_all_fine_prts == NULL || _last_all_fine_prts->next() == NULL, | |
| 325 "just checking"); | |
| 326 assert(_first_all_fine_prts == NULL || _first_all_fine_prts->prev() == NULL, | |
| 327 "just checking"); | |
| 328 } | |
| 329 | |
| 330 void OtherRegionsTable::unlink_from_all(PerRegionTable* prt) { | |
| 331 if (prt->prev() != NULL) { | |
| 332 assert(_first_all_fine_prts != prt, "just checking"); | |
| 333 prt->prev()->set_next(prt->next()); | |
| 334 // removing the last element in the list? | |
| 335 if (_last_all_fine_prts == prt) { | |
| 336 _last_all_fine_prts = prt->prev(); | |
| 337 } | |
| 338 } else { | |
| 339 assert(_first_all_fine_prts == prt, "just checking"); | |
| 340 _first_all_fine_prts = prt->next(); | |
| 341 // list is empty now? | |
| 342 if (_first_all_fine_prts == NULL) { | |
| 343 _last_all_fine_prts = NULL; | |
| 344 } | |
| 345 } | |
| 346 | |
| 347 if (prt->next() != NULL) { | |
| 348 prt->next()->set_prev(prt->prev()); | |
| 349 } | |
| 350 | |
| 351 prt->set_next(NULL); | |
| 352 prt->set_prev(NULL); | |
| 353 | |
| 354 assert((_first_all_fine_prts == NULL && _last_all_fine_prts == NULL) || | |
| 355 (_first_all_fine_prts != NULL && _last_all_fine_prts != NULL), | |
| 356 "just checking"); | |
| 357 assert(_last_all_fine_prts == NULL || _last_all_fine_prts->next() == NULL, | |
| 358 "just checking"); | |
| 359 assert(_first_all_fine_prts == NULL || _first_all_fine_prts->prev() == NULL, | |
| 360 "just checking"); | |
| 361 } | |
| 362 | |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
363 int** FromCardCache::_cache = NULL; |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
364 uint FromCardCache::_max_regions = 0; |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
365 size_t FromCardCache::_static_mem_size = 0; |
| 342 | 366 |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
367 void FromCardCache::initialize(uint n_par_rs, uint max_num_regions) { |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
368 guarantee(_cache == NULL, "Should not call this multiple times"); |
| 342 | 369 |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
370 _max_regions = max_num_regions; |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
371 _cache = Padded2DArray<int, mtGC>::create_unfreeable(n_par_rs, |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
372 _max_regions, |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
373 &_static_mem_size); |
|
17754
d7070f371770
8035815: Cache-align and pad the from card cache
tschatzl
parents:
17753
diff
changeset
|
374 |
|
d7070f371770
8035815: Cache-align and pad the from card cache
tschatzl
parents:
17753
diff
changeset
|
375 for (uint i = 0; i < n_par_rs; i++) { |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
376 for (uint j = 0; j < _max_regions; j++) { |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
377 set(i, j, InvalidCard); |
| 342 | 378 } |
| 379 } | |
| 380 } | |
| 381 | |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
382 void FromCardCache::shrink(uint new_num_regions) { |
|
17754
d7070f371770
8035815: Cache-align and pad the from card cache
tschatzl
parents:
17753
diff
changeset
|
383 for (uint i = 0; i < HeapRegionRemSet::num_par_rem_sets(); i++) { |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
384 assert(new_num_regions <= _max_regions, "Must be within max."); |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
385 for (uint j = new_num_regions; j < _max_regions; j++) { |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
386 set(i, j, InvalidCard); |
| 342 | 387 } |
| 388 } | |
| 389 } | |
| 390 | |
| 391 #ifndef PRODUCT | |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
392 void FromCardCache::print(outputStream* out) { |
|
17754
d7070f371770
8035815: Cache-align and pad the from card cache
tschatzl
parents:
17753
diff
changeset
|
393 for (uint i = 0; i < HeapRegionRemSet::num_par_rem_sets(); i++) { |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
394 for (uint j = 0; j < _max_regions; j++) { |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
395 out->print_cr("_from_card_cache["UINT32_FORMAT"]["UINT32_FORMAT"] = "INT32_FORMAT".", |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
396 i, j, at(i, j)); |
| 342 | 397 } |
| 398 } | |
| 399 } | |
| 400 #endif | |
| 401 | |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
402 void FromCardCache::clear(uint region_idx) { |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
403 uint num_par_remsets = HeapRegionRemSet::num_par_rem_sets(); |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
404 for (uint i = 0; i < num_par_remsets; i++) { |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
405 set(i, region_idx, InvalidCard); |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
406 } |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
407 } |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
408 |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
409 void OtherRegionsTable::init_from_card_cache(uint max_regions) { |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
410 FromCardCache::initialize(HeapRegionRemSet::num_par_rem_sets(), max_regions); |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
411 } |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
412 |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
413 void OtherRegionsTable::shrink_from_card_cache(uint new_num_regions) { |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
414 FromCardCache::shrink(new_num_regions); |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
415 } |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
416 |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
417 void OtherRegionsTable::print_from_card_cache() { |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
418 FromCardCache::print(); |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
419 } |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
420 |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
421 void OtherRegionsTable::add_reference(OopOrNarrowOopStar from, int tid) { |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
422 uint cur_hrs_ind = hr()->hrs_index(); |
| 342 | 423 |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
424 if (G1TraceHeapRegionRememberedSet) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
425 gclog_or_tty->print_cr("ORT::add_reference_work(" PTR_FORMAT "->" PTR_FORMAT ").", |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
426 from, |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
427 UseCompressedOops |
|
12316
190899198332
7195622: CheckUnhandledOops has limited usefulness now
hseigel
parents:
12080
diff
changeset
|
428 ? (void *)oopDesc::load_decode_heap_oop((narrowOop*)from) |
|
190899198332
7195622: CheckUnhandledOops has limited usefulness now
hseigel
parents:
12080
diff
changeset
|
429 : (void *)oopDesc::load_decode_heap_oop((oop*)from)); |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
430 } |
| 342 | 431 |
| 432 int from_card = (int)(uintptr_t(from) >> CardTableModRefBS::card_shift); | |
| 433 | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
434 if (G1TraceHeapRegionRememberedSet) { |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
435 gclog_or_tty->print_cr("Table for [" PTR_FORMAT "...): card %d (cache = "INT32_FORMAT")", |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
436 hr()->bottom(), from_card, |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
437 FromCardCache::at((uint)tid, cur_hrs_ind)); |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
438 } |
| 342 | 439 |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
440 if (FromCardCache::contains_or_replace((uint)tid, cur_hrs_ind, from_card)) { |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
441 if (G1TraceHeapRegionRememberedSet) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
442 gclog_or_tty->print_cr(" from-card cache hit."); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
443 } |
| 342 | 444 assert(contains_reference(from), "We just added it!"); |
| 445 return; | |
| 446 } | |
| 447 | |
| 448 // Note that this may be a continued H region. | |
| 449 HeapRegion* from_hr = _g1h->heap_region_containing_raw(from); | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
450 RegionIdx_t from_hrs_ind = (RegionIdx_t) from_hr->hrs_index(); |
| 342 | 451 |
| 452 // If the region is already coarsened, return. | |
| 453 if (_coarse_map.at(from_hrs_ind)) { | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
454 if (G1TraceHeapRegionRememberedSet) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
455 gclog_or_tty->print_cr(" coarse map hit."); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
456 } |
| 342 | 457 assert(contains_reference(from), "We just added it!"); |
| 458 return; | |
| 459 } | |
| 460 | |
| 461 // Otherwise find a per-region table to add it to. | |
| 462 size_t ind = from_hrs_ind & _mod_max_fine_entries_mask; | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
463 PerRegionTable* prt = find_region_table(ind, from_hr); |
| 342 | 464 if (prt == NULL) { |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
465 MutexLockerEx x(_m, Mutex::_no_safepoint_check_flag); |
| 342 | 466 // Confirm that it's really not there... |
| 467 prt = find_region_table(ind, from_hr); | |
| 468 if (prt == NULL) { | |
| 469 | |
| 470 uintptr_t from_hr_bot_card_index = | |
| 471 uintptr_t(from_hr->bottom()) | |
| 472 >> CardTableModRefBS::card_shift; | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
473 CardIdx_t card_index = from_card - from_hr_bot_card_index; |
|
3986
65a8ff39a6da
7095194: G1: HeapRegion::GrainBytes, GrainWords, and CardsPerRegion should be size_t
johnc
parents:
3830
diff
changeset
|
474 assert(0 <= card_index && (size_t)card_index < HeapRegion::CardsPerRegion, |
| 342 | 475 "Must be in range."); |
| 476 if (G1HRRSUseSparseTable && | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
477 _sparse_table.add_card(from_hrs_ind, card_index)) { |
| 342 | 478 if (G1RecordHRRSOops) { |
| 479 HeapRegionRemSet::record(hr(), from); | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
480 if (G1TraceHeapRegionRememberedSet) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
481 gclog_or_tty->print(" Added card " PTR_FORMAT " to region " |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
482 "[" PTR_FORMAT "...) for ref " PTR_FORMAT ".\n", |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
483 align_size_down(uintptr_t(from), |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
484 CardTableModRefBS::card_size), |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
485 hr()->bottom(), from); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
486 } |
| 342 | 487 } |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
488 if (G1TraceHeapRegionRememberedSet) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
489 gclog_or_tty->print_cr(" added card to sparse table."); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
490 } |
| 342 | 491 assert(contains_reference_locked(from), "We just added it!"); |
| 492 return; | |
| 493 } else { | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
494 if (G1TraceHeapRegionRememberedSet) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
495 gclog_or_tty->print_cr(" [tid %d] sparse table entry " |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
496 "overflow(f: %d, t: %d)", |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
497 tid, from_hrs_ind, cur_hrs_ind); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
498 } |
| 342 | 499 } |
| 500 | |
| 501 if (_n_fine_entries == _max_fine_entries) { | |
| 502 prt = delete_region_table(); | |
| 6253 | 503 // There is no need to clear the links to the 'all' list here: |
| 504 // prt will be reused immediately, i.e. remain in the 'all' list. | |
| 505 prt->init(from_hr, false /* clear_links_to_all_list */); | |
| 342 | 506 } else { |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
507 prt = PerRegionTable::alloc(from_hr); |
| 6253 | 508 link_to_all(prt); |
| 342 | 509 } |
| 510 | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
511 PerRegionTable* first_prt = _fine_grain_regions[ind]; |
| 6253 | 512 prt->set_collision_list_next(first_prt); |
| 342 | 513 _fine_grain_regions[ind] = prt; |
| 514 _n_fine_entries++; | |
| 515 | |
| 516 if (G1HRRSUseSparseTable) { | |
|
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
517 // Transfer from sparse to fine-grain. |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
518 SparsePRTEntry *sprt_entry = _sparse_table.get_entry(from_hrs_ind); |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
519 assert(sprt_entry != NULL, "There should have been an entry"); |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
520 for (int i = 0; i < SparsePRTEntry::cards_num(); i++) { |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
521 CardIdx_t c = sprt_entry->card(i); |
| 342 | 522 if (c != SparsePRTEntry::NullEntry) { |
| 523 prt->add_card(c); | |
| 524 } | |
| 525 } | |
| 526 // Now we can delete the sparse entry. | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
527 bool res = _sparse_table.delete_entry(from_hrs_ind); |
| 342 | 528 assert(res, "It should have been there."); |
| 529 } | |
| 530 } | |
| 531 assert(prt != NULL && prt->hr() == from_hr, "consequence"); | |
| 532 } | |
| 533 // Note that we can't assert "prt->hr() == from_hr", because of the | |
| 534 // possibility of concurrent reuse. But see head comment of | |
| 535 // OtherRegionsTable for why this is OK. | |
| 536 assert(prt != NULL, "Inv"); | |
| 537 | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
538 prt->add_reference(from); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
539 |
| 342 | 540 if (G1RecordHRRSOops) { |
| 541 HeapRegionRemSet::record(hr(), from); | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
542 if (G1TraceHeapRegionRememberedSet) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
543 gclog_or_tty->print("Added card " PTR_FORMAT " to region " |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
544 "[" PTR_FORMAT "...) for ref " PTR_FORMAT ".\n", |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
545 align_size_down(uintptr_t(from), |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
546 CardTableModRefBS::card_size), |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
547 hr()->bottom(), from); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
548 } |
| 342 | 549 } |
| 550 assert(contains_reference(from), "We just added it!"); | |
| 551 } | |
| 552 | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
553 PerRegionTable* |
| 342 | 554 OtherRegionsTable::find_region_table(size_t ind, HeapRegion* hr) const { |
| 555 assert(0 <= ind && ind < _max_fine_entries, "Preconditions."); | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
556 PerRegionTable* prt = _fine_grain_regions[ind]; |
| 342 | 557 while (prt != NULL && prt->hr() != hr) { |
| 6253 | 558 prt = prt->collision_list_next(); |
| 342 | 559 } |
| 560 // Loop postcondition is the method postcondition. | |
| 561 return prt; | |
| 562 } | |
| 563 | |
| 564 jint OtherRegionsTable::_n_coarsenings = 0; | |
| 565 | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
566 PerRegionTable* OtherRegionsTable::delete_region_table() { |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
567 assert(_m->owned_by_self(), "Precondition"); |
| 342 | 568 assert(_n_fine_entries == _max_fine_entries, "Precondition"); |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
569 PerRegionTable* max = NULL; |
| 342 | 570 jint max_occ = 0; |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
571 PerRegionTable** max_prev; |
| 342 | 572 size_t max_ind; |
| 573 | |
| 574 size_t i = _fine_eviction_start; | |
| 575 for (size_t k = 0; k < _fine_eviction_sample_size; k++) { | |
| 576 size_t ii = i; | |
| 577 // Make sure we get a non-NULL sample. | |
| 578 while (_fine_grain_regions[ii] == NULL) { | |
| 579 ii++; | |
| 580 if (ii == _max_fine_entries) ii = 0; | |
| 581 guarantee(ii != i, "We must find one."); | |
| 582 } | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
583 PerRegionTable** prev = &_fine_grain_regions[ii]; |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
584 PerRegionTable* cur = *prev; |
| 342 | 585 while (cur != NULL) { |
| 586 jint cur_occ = cur->occupied(); | |
| 587 if (max == NULL || cur_occ > max_occ) { | |
| 588 max = cur; | |
| 589 max_prev = prev; | |
| 590 max_ind = i; | |
| 591 max_occ = cur_occ; | |
| 592 } | |
| 6253 | 593 prev = cur->collision_list_next_addr(); |
| 594 cur = cur->collision_list_next(); | |
| 342 | 595 } |
| 596 i = i + _fine_eviction_stride; | |
| 597 if (i >= _n_fine_entries) i = i - _n_fine_entries; | |
| 598 } | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
599 |
| 342 | 600 _fine_eviction_start++; |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
601 |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
602 if (_fine_eviction_start >= _n_fine_entries) { |
| 342 | 603 _fine_eviction_start -= _n_fine_entries; |
| 604 } | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
605 |
| 342 | 606 guarantee(max != NULL, "Since _n_fine_entries > 0"); |
| 607 | |
| 608 // Set the corresponding coarse bit. | |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
609 size_t max_hrs_index = (size_t) max->hr()->hrs_index(); |
| 342 | 610 if (!_coarse_map.at(max_hrs_index)) { |
| 611 _coarse_map.at_put(max_hrs_index, true); | |
| 612 _n_coarse_entries++; | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
613 if (G1TraceHeapRegionRememberedSet) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
614 gclog_or_tty->print("Coarsened entry in region [" PTR_FORMAT "...] " |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
615 "for region [" PTR_FORMAT "...] (%d coarse entries).\n", |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
616 hr()->bottom(), |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
617 max->hr()->bottom(), |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
618 _n_coarse_entries); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
619 } |
| 342 | 620 } |
| 621 | |
| 622 // Unsplice. | |
| 6253 | 623 *max_prev = max->collision_list_next(); |
| 342 | 624 Atomic::inc(&_n_coarsenings); |
| 625 _n_fine_entries--; | |
| 626 return max; | |
| 627 } | |
| 628 | |
| 629 | |
| 630 // At present, this must be called stop-world single-threaded. | |
| 631 void OtherRegionsTable::scrub(CardTableModRefBS* ctbs, | |
| 632 BitMap* region_bm, BitMap* card_bm) { | |
| 633 // First eliminated garbage regions from the coarse map. | |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
634 if (G1RSScrubVerbose) { |
|
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
635 gclog_or_tty->print_cr("Scrubbing region %u:", hr()->hrs_index()); |
|
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
636 } |
| 342 | 637 |
| 638 assert(_coarse_map.size() == region_bm->size(), "Precondition"); | |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
639 if (G1RSScrubVerbose) { |
|
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
640 gclog_or_tty->print(" Coarse map: before = "SIZE_FORMAT"...", |
|
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
641 _n_coarse_entries); |
|
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
642 } |
| 342 | 643 _coarse_map.set_intersection(*region_bm); |
| 644 _n_coarse_entries = _coarse_map.count_one_bits(); | |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
645 if (G1RSScrubVerbose) { |
|
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
646 gclog_or_tty->print_cr(" after = "SIZE_FORMAT".", _n_coarse_entries); |
|
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
647 } |
| 342 | 648 |
| 649 // Now do the fine-grained maps. | |
| 650 for (size_t i = 0; i < _max_fine_entries; i++) { | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
651 PerRegionTable* cur = _fine_grain_regions[i]; |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
652 PerRegionTable** prev = &_fine_grain_regions[i]; |
| 342 | 653 while (cur != NULL) { |
| 6253 | 654 PerRegionTable* nxt = cur->collision_list_next(); |
| 342 | 655 // If the entire region is dead, eliminate. |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
656 if (G1RSScrubVerbose) { |
|
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
657 gclog_or_tty->print_cr(" For other region %u:", |
| 3766 | 658 cur->hr()->hrs_index()); |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
659 } |
|
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
660 if (!region_bm->at((size_t) cur->hr()->hrs_index())) { |
| 342 | 661 *prev = nxt; |
| 6253 | 662 cur->set_collision_list_next(NULL); |
| 342 | 663 _n_fine_entries--; |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
664 if (G1RSScrubVerbose) { |
| 342 | 665 gclog_or_tty->print_cr(" deleted via region map."); |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
666 } |
| 6253 | 667 unlink_from_all(cur); |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
668 PerRegionTable::free(cur); |
| 342 | 669 } else { |
| 670 // Do fine-grain elimination. | |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
671 if (G1RSScrubVerbose) { |
| 342 | 672 gclog_or_tty->print(" occ: before = %4d.", cur->occupied()); |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
673 } |
| 342 | 674 cur->scrub(ctbs, card_bm); |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
675 if (G1RSScrubVerbose) { |
| 342 | 676 gclog_or_tty->print_cr(" after = %4d.", cur->occupied()); |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
677 } |
| 342 | 678 // Did that empty the table completely? |
| 679 if (cur->occupied() == 0) { | |
| 680 *prev = nxt; | |
| 6253 | 681 cur->set_collision_list_next(NULL); |
| 342 | 682 _n_fine_entries--; |
| 6253 | 683 unlink_from_all(cur); |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
684 PerRegionTable::free(cur); |
| 342 | 685 } else { |
| 6253 | 686 prev = cur->collision_list_next_addr(); |
| 342 | 687 } |
| 688 } | |
| 689 cur = nxt; | |
| 690 } | |
| 691 } | |
| 692 // Since we may have deleted a from_card_cache entry from the RS, clear | |
| 693 // the FCC. | |
| 694 clear_fcc(); | |
| 695 } | |
| 696 | |
| 697 | |
| 698 size_t OtherRegionsTable::occupied() const { | |
| 699 size_t sum = occ_fine(); | |
| 700 sum += occ_sparse(); | |
| 701 sum += occ_coarse(); | |
| 702 return sum; | |
| 703 } | |
| 704 | |
| 705 size_t OtherRegionsTable::occ_fine() const { | |
| 706 size_t sum = 0; | |
| 6253 | 707 |
| 708 size_t num = 0; | |
| 709 PerRegionTable * cur = _first_all_fine_prts; | |
| 710 while (cur != NULL) { | |
| 711 sum += cur->occupied(); | |
| 712 cur = cur->next(); | |
| 713 num++; | |
| 342 | 714 } |
| 6253 | 715 guarantee(num == _n_fine_entries, "just checking"); |
| 342 | 716 return sum; |
| 717 } | |
| 718 | |
| 719 size_t OtherRegionsTable::occ_coarse() const { | |
|
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
720 return (_n_coarse_entries * HeapRegion::CardsPerRegion); |
| 342 | 721 } |
| 722 | |
| 723 size_t OtherRegionsTable::occ_sparse() const { | |
| 724 return _sparse_table.occupied(); | |
| 725 } | |
| 726 | |
| 727 size_t OtherRegionsTable::mem_size() const { | |
| 728 size_t sum = 0; | |
|
10290
05a17f270c7e
8014240: G1: Add remembered set size information to output of G1PrintRegionLivenessInfo
tschatzl
parents:
10193
diff
changeset
|
729 // all PRTs are of the same size so it is sufficient to query only one of them. |
|
05a17f270c7e
8014240: G1: Add remembered set size information to output of G1PrintRegionLivenessInfo
tschatzl
parents:
10193
diff
changeset
|
730 if (_first_all_fine_prts != NULL) { |
|
05a17f270c7e
8014240: G1: Add remembered set size information to output of G1PrintRegionLivenessInfo
tschatzl
parents:
10193
diff
changeset
|
731 assert(_last_all_fine_prts != NULL && |
|
05a17f270c7e
8014240: G1: Add remembered set size information to output of G1PrintRegionLivenessInfo
tschatzl
parents:
10193
diff
changeset
|
732 _first_all_fine_prts->mem_size() == _last_all_fine_prts->mem_size(), "check that mem_size() is constant"); |
|
05a17f270c7e
8014240: G1: Add remembered set size information to output of G1PrintRegionLivenessInfo
tschatzl
parents:
10193
diff
changeset
|
733 sum += _first_all_fine_prts->mem_size() * _n_fine_entries; |
| 342 | 734 } |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
735 sum += (sizeof(PerRegionTable*) * _max_fine_entries); |
| 342 | 736 sum += (_coarse_map.size_in_words() * HeapWordSize); |
| 737 sum += (_sparse_table.mem_size()); | |
| 738 sum += sizeof(*this) - sizeof(_sparse_table); // Avoid double counting above. | |
| 739 return sum; | |
| 740 } | |
| 741 | |
| 742 size_t OtherRegionsTable::static_mem_size() { | |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
743 return FromCardCache::static_mem_size(); |
| 342 | 744 } |
| 745 | |
| 746 size_t OtherRegionsTable::fl_mem_size() { | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
747 return PerRegionTable::fl_mem_size(); |
| 342 | 748 } |
| 749 | |
| 750 void OtherRegionsTable::clear_fcc() { | |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
751 FromCardCache::clear(hr()->hrs_index()); |
| 342 | 752 } |
| 753 | |
| 754 void OtherRegionsTable::clear() { | |
| 6253 | 755 // if there are no entries, skip this step |
| 756 if (_first_all_fine_prts != NULL) { | |
| 757 guarantee(_first_all_fine_prts != NULL && _last_all_fine_prts != NULL, "just checking"); | |
| 758 PerRegionTable::bulk_free(_first_all_fine_prts, _last_all_fine_prts); | |
| 759 memset(_fine_grain_regions, 0, _max_fine_entries * sizeof(_fine_grain_regions[0])); | |
| 760 } else { | |
| 761 guarantee(_first_all_fine_prts == NULL && _last_all_fine_prts == NULL, "just checking"); | |
| 342 | 762 } |
| 6253 | 763 |
| 764 _first_all_fine_prts = _last_all_fine_prts = NULL; | |
| 342 | 765 _sparse_table.clear(); |
| 766 _coarse_map.clear(); | |
| 767 _n_fine_entries = 0; | |
| 768 _n_coarse_entries = 0; | |
| 769 | |
| 770 clear_fcc(); | |
| 771 } | |
| 772 | |
| 773 void OtherRegionsTable::clear_incoming_entry(HeapRegion* from_hr) { | |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
774 MutexLockerEx x(_m, Mutex::_no_safepoint_check_flag); |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
775 size_t hrs_ind = (size_t) from_hr->hrs_index(); |
| 342 | 776 size_t ind = hrs_ind & _mod_max_fine_entries_mask; |
| 777 if (del_single_region_table(ind, from_hr)) { | |
| 778 assert(!_coarse_map.at(hrs_ind), "Inv"); | |
| 779 } else { | |
| 780 _coarse_map.par_at_put(hrs_ind, 0); | |
| 781 } | |
| 782 // Check to see if any of the fcc entries come from here. | |
|
17754
d7070f371770
8035815: Cache-align and pad the from card cache
tschatzl
parents:
17753
diff
changeset
|
783 uint hr_ind = hr()->hrs_index(); |
|
d7070f371770
8035815: Cache-align and pad the from card cache
tschatzl
parents:
17753
diff
changeset
|
784 for (uint tid = 0; tid < HeapRegionRemSet::num_par_rem_sets(); tid++) { |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
785 int fcc_ent = FromCardCache::at(tid, hr_ind); |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
786 if (fcc_ent != FromCardCache::InvalidCard) { |
| 342 | 787 HeapWord* card_addr = (HeapWord*) |
| 788 (uintptr_t(fcc_ent) << CardTableModRefBS::card_shift); | |
| 789 if (hr()->is_in_reserved(card_addr)) { | |
| 790 // Clear the from card cache. | |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
791 FromCardCache::set(tid, hr_ind, FromCardCache::InvalidCard); |
| 342 | 792 } |
| 793 } | |
| 794 } | |
| 795 } | |
| 796 | |
| 797 bool OtherRegionsTable::del_single_region_table(size_t ind, | |
| 798 HeapRegion* hr) { | |
| 799 assert(0 <= ind && ind < _max_fine_entries, "Preconditions."); | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
800 PerRegionTable** prev_addr = &_fine_grain_regions[ind]; |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
801 PerRegionTable* prt = *prev_addr; |
| 342 | 802 while (prt != NULL && prt->hr() != hr) { |
| 6253 | 803 prev_addr = prt->collision_list_next_addr(); |
| 804 prt = prt->collision_list_next(); | |
| 342 | 805 } |
| 806 if (prt != NULL) { | |
| 807 assert(prt->hr() == hr, "Loop postcondition."); | |
| 6253 | 808 *prev_addr = prt->collision_list_next(); |
| 809 unlink_from_all(prt); | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
810 PerRegionTable::free(prt); |
| 342 | 811 _n_fine_entries--; |
| 812 return true; | |
| 813 } else { | |
| 814 return false; | |
| 815 } | |
| 816 } | |
| 817 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
818 bool OtherRegionsTable::contains_reference(OopOrNarrowOopStar from) const { |
| 342 | 819 // Cast away const in this case. |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
820 MutexLockerEx x((Mutex*)_m, Mutex::_no_safepoint_check_flag); |
| 342 | 821 return contains_reference_locked(from); |
| 822 } | |
| 823 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
824 bool OtherRegionsTable::contains_reference_locked(OopOrNarrowOopStar from) const { |
| 342 | 825 HeapRegion* hr = _g1h->heap_region_containing_raw(from); |
| 826 if (hr == NULL) return false; | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
827 RegionIdx_t hr_ind = (RegionIdx_t) hr->hrs_index(); |
| 342 | 828 // Is this region in the coarse map? |
| 829 if (_coarse_map.at(hr_ind)) return true; | |
| 830 | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
831 PerRegionTable* prt = find_region_table(hr_ind & _mod_max_fine_entries_mask, |
| 342 | 832 hr); |
| 833 if (prt != NULL) { | |
| 834 return prt->contains_reference(from); | |
| 835 | |
| 836 } else { | |
| 837 uintptr_t from_card = | |
| 838 (uintptr_t(from) >> CardTableModRefBS::card_shift); | |
| 839 uintptr_t hr_bot_card_index = | |
| 840 uintptr_t(hr->bottom()) >> CardTableModRefBS::card_shift; | |
| 841 assert(from_card >= hr_bot_card_index, "Inv"); | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
842 CardIdx_t card_index = from_card - hr_bot_card_index; |
|
3986
65a8ff39a6da
7095194: G1: HeapRegion::GrainBytes, GrainWords, and CardsPerRegion should be size_t
johnc
parents:
3830
diff
changeset
|
843 assert(0 <= card_index && (size_t)card_index < HeapRegion::CardsPerRegion, |
|
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
844 "Must be in range."); |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
845 return _sparse_table.contains_card(hr_ind, card_index); |
| 342 | 846 } |
| 847 } | |
| 848 | |
| 2173 | 849 void |
| 850 OtherRegionsTable::do_cleanup_work(HRRSCleanupTask* hrrs_cleanup_task) { | |
| 851 _sparse_table.do_cleanup_work(hrrs_cleanup_task); | |
| 852 } | |
| 853 | |
|
795
215f81b4d9b3
6841831: G1: assert(contains_reference(from),"We just added it!") fires
iveresov
parents:
794
diff
changeset
|
854 // Determines how many threads can add records to an rset in parallel. |
|
215f81b4d9b3
6841831: G1: assert(contains_reference(from),"We just added it!") fires
iveresov
parents:
794
diff
changeset
|
855 // This can be done by either mutator threads together with the |
|
215f81b4d9b3
6841831: G1: assert(contains_reference(from),"We just added it!") fires
iveresov
parents:
794
diff
changeset
|
856 // concurrent refinement threads or GC threads. |
|
17754
d7070f371770
8035815: Cache-align and pad the from card cache
tschatzl
parents:
17753
diff
changeset
|
857 uint HeapRegionRemSet::num_par_rem_sets() { |
|
17844
8847586c9037
8016302: Change type of the number of GC workers to unsigned int (2)
vkempik
parents:
17758
diff
changeset
|
858 return MAX2(DirtyCardQueueSet::num_par_ids() + ConcurrentG1Refine::thread_num(), (uint)ParallelGCThreads); |
| 342 | 859 } |
| 860 | |
| 861 HeapRegionRemSet::HeapRegionRemSet(G1BlockOffsetSharedArray* bosa, | |
| 862 HeapRegion* hr) | |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
863 : _bosa(bosa), |
|
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
864 _m(Mutex::leaf, FormatBuffer<128>("HeapRegionRemSet lock #"UINT32_FORMAT, hr->hrs_index()), true), |
|
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
865 _code_roots(), _other_regions(hr, &_m) { |
|
3777
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
866 reset_for_par_iteration(); |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
867 } |
| 342 | 868 |
|
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
869 void HeapRegionRemSet::setup_remset_size() { |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
870 // Setup sparse and fine-grain tables sizes. |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
871 // table_size = base * (log(region_size / 1M) + 1) |
|
6107
5c8bd7c16119
7171936: LOG_G incorrectly defined in globalDefinitions.hpp
brutisso
parents:
6010
diff
changeset
|
872 const int LOG_M = 20; |
|
5c8bd7c16119
7171936: LOG_G incorrectly defined in globalDefinitions.hpp
brutisso
parents:
6010
diff
changeset
|
873 int region_size_log_mb = MAX2(HeapRegion::LogOfHRGrainBytes - LOG_M, 0); |
|
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
874 if (FLAG_IS_DEFAULT(G1RSetSparseRegionEntries)) { |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
875 G1RSetSparseRegionEntries = G1RSetSparseRegionEntriesBase * (region_size_log_mb + 1); |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
876 } |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
877 if (FLAG_IS_DEFAULT(G1RSetRegionEntries)) { |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
878 G1RSetRegionEntries = G1RSetRegionEntriesBase * (region_size_log_mb + 1); |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
879 } |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
880 guarantee(G1RSetSparseRegionEntries > 0 && G1RSetRegionEntries > 0 , "Sanity"); |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
881 } |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
882 |
| 342 | 883 bool HeapRegionRemSet::claim_iter() { |
| 884 if (_iter_state != Unclaimed) return false; | |
| 885 jint res = Atomic::cmpxchg(Claimed, (jint*)(&_iter_state), Unclaimed); | |
| 886 return (res == Unclaimed); | |
| 887 } | |
| 888 | |
| 889 void HeapRegionRemSet::set_iter_complete() { | |
| 890 _iter_state = Complete; | |
| 891 } | |
| 892 | |
| 893 bool HeapRegionRemSet::iter_is_complete() { | |
| 894 return _iter_state == Complete; | |
| 895 } | |
| 896 | |
| 897 #ifndef PRODUCT | |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
898 void HeapRegionRemSet::print() { |
|
10182
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
899 HeapRegionRemSetIterator iter(this); |
| 342 | 900 size_t card_index; |
| 901 while (iter.has_next(card_index)) { | |
| 902 HeapWord* card_start = | |
| 903 G1CollectedHeap::heap()->bot_shared()->address_for_index(card_index); | |
|
3777
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
904 gclog_or_tty->print_cr(" Card " PTR_FORMAT, card_start); |
| 342 | 905 } |
| 906 if (iter.n_yielded() != occupied()) { | |
| 907 gclog_or_tty->print_cr("Yielded disagrees with occupied:"); | |
| 908 gclog_or_tty->print_cr(" %6d yielded (%6d coarse, %6d fine).", | |
| 909 iter.n_yielded(), | |
| 910 iter.n_yielded_coarse(), iter.n_yielded_fine()); | |
| 911 gclog_or_tty->print_cr(" %6d occ (%6d coarse, %6d fine).", | |
| 912 occupied(), occ_coarse(), occ_fine()); | |
| 913 } | |
| 914 guarantee(iter.n_yielded() == occupied(), | |
| 915 "We should have yielded all the represented cards."); | |
| 916 } | |
| 917 #endif | |
| 918 | |
| 919 void HeapRegionRemSet::cleanup() { | |
| 920 SparsePRT::cleanup_all(); | |
| 921 } | |
| 922 | |
| 923 void HeapRegionRemSet::clear() { | |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
924 MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag); |
|
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
925 clear_locked(); |
|
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
926 } |
| 12080 | 927 |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
928 void HeapRegionRemSet::clear_locked() { |
|
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
929 _code_roots.clear(); |
| 342 | 930 _other_regions.clear(); |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
931 assert(occupied_locked() == 0, "Should be clear."); |
|
3777
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
932 reset_for_par_iteration(); |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
933 } |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
934 |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
935 void HeapRegionRemSet::reset_for_par_iteration() { |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
936 _iter_state = Unclaimed; |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
937 _iter_claimed = 0; |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
938 // It's good to check this to make sure that the two methods are in sync. |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
939 assert(verify_ready_for_par_iteration(), "post-condition"); |
| 342 | 940 } |
| 941 | |
| 942 void HeapRegionRemSet::scrub(CardTableModRefBS* ctbs, | |
| 943 BitMap* region_bm, BitMap* card_bm) { | |
| 944 _other_regions.scrub(ctbs, region_bm, card_bm); | |
| 945 } | |
| 946 | |
| 12080 | 947 // Code roots support |
| 948 | |
| 949 void HeapRegionRemSet::add_strong_code_root(nmethod* nm) { | |
| 950 assert(nm != NULL, "sanity"); | |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
951 _code_roots.add(nm); |
| 12080 | 952 } |
| 953 | |
| 954 void HeapRegionRemSet::remove_strong_code_root(nmethod* nm) { | |
| 955 assert(nm != NULL, "sanity"); | |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
956 _code_roots.remove(nm); |
| 12080 | 957 // Check that there were no duplicates |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
958 guarantee(!_code_roots.contains(nm), "duplicate entry found"); |
| 12080 | 959 } |
| 960 | |
| 961 class NMethodMigrationOopClosure : public OopClosure { | |
| 962 G1CollectedHeap* _g1h; | |
| 963 HeapRegion* _from; | |
| 964 nmethod* _nm; | |
| 965 | |
| 966 uint _num_self_forwarded; | |
| 967 | |
| 968 template <class T> void do_oop_work(T* p) { | |
| 969 T heap_oop = oopDesc::load_heap_oop(p); | |
| 970 if (!oopDesc::is_null(heap_oop)) { | |
| 971 oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); | |
| 972 if (_from->is_in(obj)) { | |
| 973 // Reference still points into the source region. | |
| 974 // Since roots are immediately evacuated this means that | |
| 975 // we must have self forwarded the object | |
| 976 assert(obj->is_forwarded(), | |
| 977 err_msg("code roots should be immediately evacuated. " | |
| 978 "Ref: "PTR_FORMAT", " | |
| 979 "Obj: "PTR_FORMAT", " | |
| 980 "Region: "HR_FORMAT, | |
| 981 p, (void*) obj, HR_FORMAT_PARAMS(_from))); | |
| 982 assert(obj->forwardee() == obj, | |
| 983 err_msg("not self forwarded? obj = "PTR_FORMAT, (void*)obj)); | |
| 984 | |
| 985 // The object has been self forwarded. | |
| 986 // Note, if we're during an initial mark pause, there is | |
| 987 // no need to explicitly mark object. It will be marked | |
| 988 // during the regular evacuation failure handling code. | |
| 989 _num_self_forwarded++; | |
| 990 } else { | |
| 991 // The reference points into a promotion or to-space region | |
| 992 HeapRegion* to = _g1h->heap_region_containing(obj); | |
| 993 to->rem_set()->add_strong_code_root(_nm); | |
| 994 } | |
| 995 } | |
| 996 } | |
| 997 | |
| 998 public: | |
| 999 NMethodMigrationOopClosure(G1CollectedHeap* g1h, HeapRegion* from, nmethod* nm): | |
| 1000 _g1h(g1h), _from(from), _nm(nm), _num_self_forwarded(0) {} | |
| 1001 | |
| 1002 void do_oop(narrowOop* p) { do_oop_work(p); } | |
| 1003 void do_oop(oop* p) { do_oop_work(p); } | |
| 1004 | |
| 1005 uint retain() { return _num_self_forwarded > 0; } | |
| 1006 }; | |
| 1007 | |
| 1008 void HeapRegionRemSet::migrate_strong_code_roots() { | |
| 1009 assert(hr()->in_collection_set(), "only collection set regions"); | |
|
13062
28674af341ac
8027756: assert(!hr->isHumongous()) failed: code root in humongous region?
tschatzl
parents:
12316
diff
changeset
|
1010 assert(!hr()->isHumongous(), |
|
28674af341ac
8027756: assert(!hr->isHumongous()) failed: code root in humongous region?
tschatzl
parents:
12316
diff
changeset
|
1011 err_msg("humongous region "HR_FORMAT" should not have been added to the collection set", |
|
28674af341ac
8027756: assert(!hr->isHumongous()) failed: code root in humongous region?
tschatzl
parents:
12316
diff
changeset
|
1012 HR_FORMAT_PARAMS(hr()))); |
| 12080 | 1013 |
| 1014 ResourceMark rm; | |
| 1015 | |
| 1016 // List of code blobs to retain for this region | |
| 1017 GrowableArray<nmethod*> to_be_retained(10); | |
| 1018 G1CollectedHeap* g1h = G1CollectedHeap::heap(); | |
| 1019 | |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
1020 while (!_code_roots.is_empty()) { |
|
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
1021 nmethod *nm = _code_roots.pop(); |
| 12080 | 1022 if (nm != NULL) { |
| 1023 NMethodMigrationOopClosure oop_cl(g1h, hr(), nm); | |
| 1024 nm->oops_do(&oop_cl); | |
| 1025 if (oop_cl.retain()) { | |
| 1026 to_be_retained.push(nm); | |
| 1027 } | |
| 1028 } | |
| 1029 } | |
| 1030 | |
| 1031 // Now push any code roots we need to retain | |
| 1032 assert(to_be_retained.is_empty() || hr()->evacuation_failed(), | |
| 1033 "Retained nmethod list must be empty or " | |
| 1034 "evacuation of this region failed"); | |
| 1035 | |
| 1036 while (to_be_retained.is_nonempty()) { | |
| 1037 nmethod* nm = to_be_retained.pop(); | |
| 1038 assert(nm != NULL, "sanity"); | |
| 1039 add_strong_code_root(nm); | |
| 1040 } | |
| 1041 } | |
| 1042 | |
| 1043 void HeapRegionRemSet::strong_code_roots_do(CodeBlobClosure* blk) const { | |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
1044 _code_roots.nmethods_do(blk); |
| 12080 | 1045 } |
| 1046 | |
| 1047 size_t HeapRegionRemSet::strong_code_roots_mem_size() { | |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
1048 return _code_roots.mem_size(); |
| 12080 | 1049 } |
| 1050 | |
| 342 | 1051 //-------------------- Iteration -------------------- |
| 1052 | |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
1053 HeapRegionRemSetIterator:: HeapRegionRemSetIterator(HeapRegionRemSet* hrrs) : |
|
10182
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1054 _hrrs(hrrs), |
| 342 | 1055 _g1h(G1CollectedHeap::heap()), |
|
10182
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1056 _coarse_map(&hrrs->_other_regions._coarse_map), |
|
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1057 _fine_grain_regions(hrrs->_other_regions._fine_grain_regions), |
|
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1058 _bosa(hrrs->bosa()), |
|
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1059 _is(Sparse), |
| 342 | 1060 // Set these values so that we increment to the first region. |
|
10182
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1061 _coarse_cur_region_index(-1), |
|
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1062 _coarse_cur_region_cur_card(HeapRegion::CardsPerRegion-1), |
|
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1063 _cur_region_cur_card(0), |
|
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1064 _fine_array_index(-1), |
|
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1065 _fine_cur_prt(NULL), |
|
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1066 _n_yielded_coarse(0), |
|
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1067 _n_yielded_fine(0), |
|
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1068 _n_yielded_sparse(0), |
|
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1069 _sparse_iter(&hrrs->_other_regions._sparse_table) {} |
| 342 | 1070 |
| 1071 bool HeapRegionRemSetIterator::coarse_has_next(size_t& card_index) { | |
| 1072 if (_hrrs->_other_regions._n_coarse_entries == 0) return false; | |
| 1073 // Go to the next card. | |
| 1074 _coarse_cur_region_cur_card++; | |
| 1075 // Was the last the last card in the current region? | |
|
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
1076 if (_coarse_cur_region_cur_card == HeapRegion::CardsPerRegion) { |
| 342 | 1077 // Yes: find the next region. This may leave _coarse_cur_region_index |
| 1078 // Set to the last index, in which case there are no more coarse | |
| 1079 // regions. | |
| 1080 _coarse_cur_region_index = | |
| 1081 (int) _coarse_map->get_next_one_offset(_coarse_cur_region_index + 1); | |
| 1082 if ((size_t)_coarse_cur_region_index < _coarse_map->size()) { | |
| 1083 _coarse_cur_region_cur_card = 0; | |
| 1084 HeapWord* r_bot = | |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
1085 _g1h->region_at((uint) _coarse_cur_region_index)->bottom(); |
| 342 | 1086 _cur_region_card_offset = _bosa->index_for(r_bot); |
| 1087 } else { | |
| 1088 return false; | |
| 1089 } | |
| 1090 } | |
| 1091 // If we didn't return false above, then we can yield a card. | |
| 1092 card_index = _cur_region_card_offset + _coarse_cur_region_cur_card; | |
| 1093 return true; | |
| 1094 } | |
| 1095 | |
| 1096 void HeapRegionRemSetIterator::fine_find_next_non_null_prt() { | |
| 1097 // Otherwise, find the next bucket list in the array. | |
| 1098 _fine_array_index++; | |
| 1099 while (_fine_array_index < (int) OtherRegionsTable::_max_fine_entries) { | |
| 1100 _fine_cur_prt = _fine_grain_regions[_fine_array_index]; | |
| 1101 if (_fine_cur_prt != NULL) return; | |
| 1102 else _fine_array_index++; | |
| 1103 } | |
| 1104 assert(_fine_cur_prt == NULL, "Loop post"); | |
| 1105 } | |
| 1106 | |
| 1107 bool HeapRegionRemSetIterator::fine_has_next(size_t& card_index) { | |
| 1108 if (fine_has_next()) { | |
| 1109 _cur_region_cur_card = | |
| 1110 _fine_cur_prt->_bm.get_next_one_offset(_cur_region_cur_card + 1); | |
| 1111 } | |
| 1112 while (!fine_has_next()) { | |
|
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
1113 if (_cur_region_cur_card == (size_t) HeapRegion::CardsPerRegion) { |
| 342 | 1114 _cur_region_cur_card = 0; |
| 6253 | 1115 _fine_cur_prt = _fine_cur_prt->collision_list_next(); |
| 342 | 1116 } |
| 1117 if (_fine_cur_prt == NULL) { | |
| 1118 fine_find_next_non_null_prt(); | |
| 1119 if (_fine_cur_prt == NULL) return false; | |
| 1120 } | |
| 1121 assert(_fine_cur_prt != NULL && _cur_region_cur_card == 0, | |
| 1122 "inv."); | |
| 1123 HeapWord* r_bot = | |
| 1124 _fine_cur_prt->hr()->bottom(); | |
| 1125 _cur_region_card_offset = _bosa->index_for(r_bot); | |
| 1126 _cur_region_cur_card = _fine_cur_prt->_bm.get_next_one_offset(0); | |
| 1127 } | |
| 1128 assert(fine_has_next(), "Or else we exited the loop via the return."); | |
| 1129 card_index = _cur_region_card_offset + _cur_region_cur_card; | |
| 1130 return true; | |
| 1131 } | |
| 1132 | |
| 1133 bool HeapRegionRemSetIterator::fine_has_next() { | |
| 1134 return | |
| 1135 _fine_cur_prt != NULL && | |
|
3986
65a8ff39a6da
7095194: G1: HeapRegion::GrainBytes, GrainWords, and CardsPerRegion should be size_t
johnc
parents:
3830
diff
changeset
|
1136 _cur_region_cur_card < HeapRegion::CardsPerRegion; |
| 342 | 1137 } |
| 1138 | |
| 1139 bool HeapRegionRemSetIterator::has_next(size_t& card_index) { | |
| 1140 switch (_is) { | |
| 1141 case Sparse: | |
| 1142 if (_sparse_iter.has_next(card_index)) { | |
| 1143 _n_yielded_sparse++; | |
| 1144 return true; | |
| 1145 } | |
| 1146 // Otherwise, deliberate fall-through | |
| 1147 _is = Fine; | |
| 1148 case Fine: | |
| 1149 if (fine_has_next(card_index)) { | |
| 1150 _n_yielded_fine++; | |
| 1151 return true; | |
| 1152 } | |
| 1153 // Otherwise, deliberate fall-through | |
| 1154 _is = Coarse; | |
| 1155 case Coarse: | |
| 1156 if (coarse_has_next(card_index)) { | |
| 1157 _n_yielded_coarse++; | |
| 1158 return true; | |
| 1159 } | |
| 1160 // Otherwise... | |
| 1161 break; | |
| 1162 } | |
| 1163 assert(ParallelGCThreads > 1 || | |
| 1164 n_yielded() == _hrrs->occupied(), | |
| 1165 "Should have yielded all the cards in the rem set " | |
| 1166 "(in the non-par case)."); | |
| 1167 return false; | |
| 1168 } | |
| 1169 | |
| 1170 | |
| 1171 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1172 OopOrNarrowOopStar* HeapRegionRemSet::_recorded_oops = NULL; |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1173 HeapWord** HeapRegionRemSet::_recorded_cards = NULL; |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1174 HeapRegion** HeapRegionRemSet::_recorded_regions = NULL; |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1175 int HeapRegionRemSet::_n_recorded = 0; |
| 342 | 1176 |
| 1177 HeapRegionRemSet::Event* HeapRegionRemSet::_recorded_events = NULL; | |
| 1178 int* HeapRegionRemSet::_recorded_event_index = NULL; | |
| 1179 int HeapRegionRemSet::_n_recorded_events = 0; | |
| 1180 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1181 void HeapRegionRemSet::record(HeapRegion* hr, OopOrNarrowOopStar f) { |
| 342 | 1182 if (_recorded_oops == NULL) { |
| 1183 assert(_n_recorded == 0 | |
| 1184 && _recorded_cards == NULL | |
| 1185 && _recorded_regions == NULL, | |
| 1186 "Inv"); | |
| 6197 | 1187 _recorded_oops = NEW_C_HEAP_ARRAY(OopOrNarrowOopStar, MaxRecorded, mtGC); |
| 1188 _recorded_cards = NEW_C_HEAP_ARRAY(HeapWord*, MaxRecorded, mtGC); | |
| 1189 _recorded_regions = NEW_C_HEAP_ARRAY(HeapRegion*, MaxRecorded, mtGC); | |
| 342 | 1190 } |
| 1191 if (_n_recorded == MaxRecorded) { | |
| 1192 gclog_or_tty->print_cr("Filled up 'recorded' (%d).", MaxRecorded); | |
| 1193 } else { | |
| 1194 _recorded_cards[_n_recorded] = | |
| 1195 (HeapWord*)align_size_down(uintptr_t(f), | |
| 1196 CardTableModRefBS::card_size); | |
| 1197 _recorded_oops[_n_recorded] = f; | |
| 1198 _recorded_regions[_n_recorded] = hr; | |
| 1199 _n_recorded++; | |
| 1200 } | |
| 1201 } | |
| 1202 | |
| 1203 void HeapRegionRemSet::record_event(Event evnt) { | |
| 1204 if (!G1RecordHRRSEvents) return; | |
| 1205 | |
| 1206 if (_recorded_events == NULL) { | |
| 1207 assert(_n_recorded_events == 0 | |
| 1208 && _recorded_event_index == NULL, | |
| 1209 "Inv"); | |
| 6197 | 1210 _recorded_events = NEW_C_HEAP_ARRAY(Event, MaxRecordedEvents, mtGC); |
| 1211 _recorded_event_index = NEW_C_HEAP_ARRAY(int, MaxRecordedEvents, mtGC); | |
| 342 | 1212 } |
| 1213 if (_n_recorded_events == MaxRecordedEvents) { | |
| 1214 gclog_or_tty->print_cr("Filled up 'recorded_events' (%d).", MaxRecordedEvents); | |
| 1215 } else { | |
| 1216 _recorded_events[_n_recorded_events] = evnt; | |
| 1217 _recorded_event_index[_n_recorded_events] = _n_recorded; | |
| 1218 _n_recorded_events++; | |
| 1219 } | |
| 1220 } | |
| 1221 | |
| 1222 void HeapRegionRemSet::print_event(outputStream* str, Event evnt) { | |
| 1223 switch (evnt) { | |
| 1224 case Event_EvacStart: | |
| 1225 str->print("Evac Start"); | |
| 1226 break; | |
| 1227 case Event_EvacEnd: | |
| 1228 str->print("Evac End"); | |
| 1229 break; | |
| 1230 case Event_RSUpdateEnd: | |
| 1231 str->print("RS Update End"); | |
| 1232 break; | |
| 1233 } | |
| 1234 } | |
| 1235 | |
| 1236 void HeapRegionRemSet::print_recorded() { | |
| 1237 int cur_evnt = 0; | |
| 1238 Event cur_evnt_kind; | |
| 1239 int cur_evnt_ind = 0; | |
| 1240 if (_n_recorded_events > 0) { | |
| 1241 cur_evnt_kind = _recorded_events[cur_evnt]; | |
| 1242 cur_evnt_ind = _recorded_event_index[cur_evnt]; | |
| 1243 } | |
| 1244 | |
| 1245 for (int i = 0; i < _n_recorded; i++) { | |
| 1246 while (cur_evnt < _n_recorded_events && i == cur_evnt_ind) { | |
| 1247 gclog_or_tty->print("Event: "); | |
| 1248 print_event(gclog_or_tty, cur_evnt_kind); | |
|
17937
78bbf4d43a14
8037816: Fix for 8036122 breaks build with Xcode5/clang
drchase
parents:
17844
diff
changeset
|
1249 gclog_or_tty->cr(); |
| 342 | 1250 cur_evnt++; |
| 1251 if (cur_evnt < MaxRecordedEvents) { | |
| 1252 cur_evnt_kind = _recorded_events[cur_evnt]; | |
| 1253 cur_evnt_ind = _recorded_event_index[cur_evnt]; | |
| 1254 } | |
| 1255 } | |
| 1256 gclog_or_tty->print("Added card " PTR_FORMAT " to region [" PTR_FORMAT "...]" | |
| 1257 " for ref " PTR_FORMAT ".\n", | |
| 1258 _recorded_cards[i], _recorded_regions[i]->bottom(), | |
| 1259 _recorded_oops[i]); | |
| 1260 } | |
| 1261 } | |
| 1262 | |
| 2173 | 1263 void HeapRegionRemSet::reset_for_cleanup_tasks() { |
| 1264 SparsePRT::reset_for_cleanup_tasks(); | |
| 1265 } | |
| 1266 | |
| 1267 void HeapRegionRemSet::do_cleanup_work(HRRSCleanupTask* hrrs_cleanup_task) { | |
| 1268 _other_regions.do_cleanup_work(hrrs_cleanup_task); | |
| 1269 } | |
| 1270 | |
| 1271 void | |
| 1272 HeapRegionRemSet::finish_cleanup_task(HRRSCleanupTask* hrrs_cleanup_task) { | |
| 1273 SparsePRT::finish_cleanup_task(hrrs_cleanup_task); | |
| 1274 } | |
| 1275 | |
| 342 | 1276 #ifndef PRODUCT |
|
10333
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
1277 void PerRegionTable::test_fl_mem_size() { |
|
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
1278 PerRegionTable* dummy = alloc(NULL); |
|
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
1279 free(dummy); |
|
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
1280 guarantee(dummy->mem_size() == fl_mem_size(), "fl_mem_size() does not return the correct element size"); |
|
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
1281 // try to reset the state |
|
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
1282 _free_list = NULL; |
|
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
1283 delete dummy; |
|
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
1284 } |
|
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
1285 |
|
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
1286 void HeapRegionRemSet::test_prt() { |
|
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
1287 PerRegionTable::test_fl_mem_size(); |
|
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
1288 } |
|
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
1289 |
| 342 | 1290 void HeapRegionRemSet::test() { |
| 1291 os::sleep(Thread::current(), (jlong)5000, false); | |
| 1292 G1CollectedHeap* g1h = G1CollectedHeap::heap(); | |
| 1293 | |
|
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1294 // Run with "-XX:G1LogRSetRegionEntries=2", so that 1 and 5 end up in same |
| 342 | 1295 // hash bucket. |
| 1296 HeapRegion* hr0 = g1h->region_at(0); | |
| 1297 HeapRegion* hr1 = g1h->region_at(1); | |
| 1298 HeapRegion* hr2 = g1h->region_at(5); | |
| 1299 HeapRegion* hr3 = g1h->region_at(6); | |
| 1300 HeapRegion* hr4 = g1h->region_at(7); | |
| 1301 HeapRegion* hr5 = g1h->region_at(8); | |
| 1302 | |
| 1303 HeapWord* hr1_start = hr1->bottom(); | |
| 1304 HeapWord* hr1_mid = hr1_start + HeapRegion::GrainWords/2; | |
| 1305 HeapWord* hr1_last = hr1->end() - 1; | |
| 1306 | |
| 1307 HeapWord* hr2_start = hr2->bottom(); | |
| 1308 HeapWord* hr2_mid = hr2_start + HeapRegion::GrainWords/2; | |
| 1309 HeapWord* hr2_last = hr2->end() - 1; | |
| 1310 | |
| 1311 HeapWord* hr3_start = hr3->bottom(); | |
| 1312 HeapWord* hr3_mid = hr3_start + HeapRegion::GrainWords/2; | |
| 1313 HeapWord* hr3_last = hr3->end() - 1; | |
| 1314 | |
| 1315 HeapRegionRemSet* hrrs = hr0->rem_set(); | |
| 1316 | |
| 1317 // Make three references from region 0x101... | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1318 hrrs->add_reference((OopOrNarrowOopStar)hr1_start); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1319 hrrs->add_reference((OopOrNarrowOopStar)hr1_mid); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1320 hrrs->add_reference((OopOrNarrowOopStar)hr1_last); |
| 342 | 1321 |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1322 hrrs->add_reference((OopOrNarrowOopStar)hr2_start); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1323 hrrs->add_reference((OopOrNarrowOopStar)hr2_mid); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1324 hrrs->add_reference((OopOrNarrowOopStar)hr2_last); |
| 342 | 1325 |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1326 hrrs->add_reference((OopOrNarrowOopStar)hr3_start); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1327 hrrs->add_reference((OopOrNarrowOopStar)hr3_mid); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1328 hrrs->add_reference((OopOrNarrowOopStar)hr3_last); |
| 342 | 1329 |
| 1330 // Now cause a coarsening. | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1331 hrrs->add_reference((OopOrNarrowOopStar)hr4->bottom()); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1332 hrrs->add_reference((OopOrNarrowOopStar)hr5->bottom()); |
| 342 | 1333 |
| 1334 // Now, does iteration yield these three? | |
|
10182
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1335 HeapRegionRemSetIterator iter(hrrs); |
| 342 | 1336 size_t sum = 0; |
| 1337 size_t card_index; | |
| 1338 while (iter.has_next(card_index)) { | |
| 1339 HeapWord* card_start = | |
| 1340 G1CollectedHeap::heap()->bot_shared()->address_for_index(card_index); | |
| 1341 gclog_or_tty->print_cr(" Card " PTR_FORMAT ".", card_start); | |
| 1342 sum++; | |
| 1343 } | |
| 1344 guarantee(sum == 11 - 3 + 2048, "Failure"); | |
| 1345 guarantee(sum == hrrs->occupied(), "Failure"); | |
| 1346 } | |
| 1347 #endif |