Mercurial > hg > graal-compiler
annotate src/share/vm/gc_implementation/g1/heapRegionRemSet.cpp @ 17833:bfdf528be8e8
8038498: Fix includes and C inlining after 8035330
Summary: Change 8035330: Remove G1ParScanPartialArrayClosure and G1ParScanHeapEvacClosure broke the debug build on AIX. The method do_oop_partial_array() is added in a header, but requires the inline function par_write_ref() through several inlined calls. In some cpp files, like arguments.cpp, par_write_ref() is not defined as the corresponding inline header and is not included. The AIX debug VM does not start because of the missing symbol. This change solves this by cleaning up include dependencies.
Reviewed-by: tschatzl, stefank
| author | tschatzl |
|---|---|
| date | Fri, 04 Apr 2014 10:43:56 +0200 |
| parents | ae7336d6337e |
| children | 8847586c9037 |
| rev | line source |
|---|---|
| 342 | 1 /* |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
2 * Copyright (c) 2001, 2014, Oracle and/or its affiliates. All rights reserved. |
| 342 | 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| 4 * | |
| 5 * This code is free software; you can redistribute it and/or modify it | |
| 6 * under the terms of the GNU General Public License version 2 only, as | |
| 7 * published by the Free Software Foundation. | |
| 8 * | |
| 9 * This code is distributed in the hope that it will be useful, but WITHOUT | |
| 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
| 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
| 12 * version 2 for more details (a copy is included in the LICENSE file that | |
| 13 * accompanied this code). | |
| 14 * | |
| 15 * You should have received a copy of the GNU General Public License version | |
| 16 * 2 along with this work; if not, write to the Free Software Foundation, | |
| 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |
| 18 * | |
|
1552
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1390
diff
changeset
|
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
|
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1390
diff
changeset
|
20 * or visit www.oracle.com if you need additional information or have any |
|
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1390
diff
changeset
|
21 * questions. |
| 342 | 22 * |
| 23 */ | |
| 24 | |
| 1972 | 25 #include "precompiled.hpp" |
| 26 #include "gc_implementation/g1/concurrentG1Refine.hpp" | |
| 27 #include "gc_implementation/g1/g1BlockOffsetTable.inline.hpp" | |
| 28 #include "gc_implementation/g1/g1CollectedHeap.inline.hpp" | |
| 29 #include "gc_implementation/g1/heapRegionRemSet.hpp" | |
| 30 #include "gc_implementation/g1/heapRegionSeq.inline.hpp" | |
| 31 #include "memory/allocation.hpp" | |
|
17754
d7070f371770
8035815: Cache-align and pad the from card cache
tschatzl
parents:
17753
diff
changeset
|
32 #include "memory/padded.inline.hpp" |
| 1972 | 33 #include "memory/space.inline.hpp" |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
34 #include "oops/oop.inline.hpp" |
| 1972 | 35 #include "utilities/bitMap.inline.hpp" |
| 36 #include "utilities/globalDefinitions.hpp" | |
| 12080 | 37 #include "utilities/growableArray.hpp" |
| 342 | 38 |
| 6197 | 39 class PerRegionTable: public CHeapObj<mtGC> { |
| 342 | 40 friend class OtherRegionsTable; |
| 41 friend class HeapRegionRemSetIterator; | |
| 42 | |
| 43 HeapRegion* _hr; | |
| 44 BitMap _bm; | |
| 45 jint _occupied; | |
| 46 | |
| 6253 | 47 // next pointer for free/allocated 'all' list |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
48 PerRegionTable* _next; |
| 342 | 49 |
| 6253 | 50 // prev pointer for the allocated 'all' list |
| 51 PerRegionTable* _prev; | |
| 342 | 52 |
| 6253 | 53 // next pointer in collision list |
| 54 PerRegionTable * _collision_list_next; | |
| 342 | 55 |
| 6253 | 56 // Global free list of PRTs |
| 57 static PerRegionTable* _free_list; | |
| 342 | 58 |
| 59 protected: | |
| 60 // We need access in order to union things into the base table. | |
| 61 BitMap* bm() { return &_bm; } | |
| 62 | |
| 63 void recount_occupied() { | |
| 64 _occupied = (jint) bm()->count_one_bits(); | |
| 65 } | |
| 66 | |
| 67 PerRegionTable(HeapRegion* hr) : | |
| 68 _hr(hr), | |
| 69 _occupied(0), | |
| 6253 | 70 _bm(HeapRegion::CardsPerRegion, false /* in-resource-area */), |
| 71 _collision_list_next(NULL), _next(NULL), _prev(NULL) | |
| 342 | 72 {} |
| 73 | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
74 void add_card_work(CardIdx_t from_card, bool par) { |
| 342 | 75 if (!_bm.at(from_card)) { |
| 76 if (par) { | |
| 77 if (_bm.par_at_put(from_card, 1)) { | |
| 78 Atomic::inc(&_occupied); | |
| 79 } | |
| 80 } else { | |
| 81 _bm.at_put(from_card, 1); | |
| 82 _occupied++; | |
| 83 } | |
| 84 } | |
| 85 } | |
| 86 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
87 void add_reference_work(OopOrNarrowOopStar from, bool par) { |
| 342 | 88 // Must make this robust in case "from" is not in "_hr", because of |
| 89 // concurrency. | |
| 90 | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
91 if (G1TraceHeapRegionRememberedSet) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
92 gclog_or_tty->print_cr(" PRT::Add_reference_work(" PTR_FORMAT "->" PTR_FORMAT").", |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
93 from, |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
94 UseCompressedOops |
|
12316
190899198332
7195622: CheckUnhandledOops has limited usefulness now
hseigel
parents:
12080
diff
changeset
|
95 ? (void *)oopDesc::load_decode_heap_oop((narrowOop*)from) |
|
190899198332
7195622: CheckUnhandledOops has limited usefulness now
hseigel
parents:
12080
diff
changeset
|
96 : (void *)oopDesc::load_decode_heap_oop((oop*)from)); |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
97 } |
| 342 | 98 |
| 99 HeapRegion* loc_hr = hr(); | |
| 100 // If the test below fails, then this table was reused concurrently | |
| 101 // with this operation. This is OK, since the old table was coarsened, | |
| 102 // and adding a bit to the new table is never incorrect. | |
|
4020
5e5d4821bf07
7097516: G1: assert(0<= from_card && from_card<HeapRegion::CardsPerRegion) failed: Must be in range.
brutisso
parents:
3986
diff
changeset
|
103 // If the table used to belong to a continues humongous region and is |
|
5e5d4821bf07
7097516: G1: assert(0<= from_card && from_card<HeapRegion::CardsPerRegion) failed: Must be in range.
brutisso
parents:
3986
diff
changeset
|
104 // now reused for the corresponding start humongous region, we need to |
|
5e5d4821bf07
7097516: G1: assert(0<= from_card && from_card<HeapRegion::CardsPerRegion) failed: Must be in range.
brutisso
parents:
3986
diff
changeset
|
105 // make sure that we detect this. Thus, we call is_in_reserved_raw() |
|
5e5d4821bf07
7097516: G1: assert(0<= from_card && from_card<HeapRegion::CardsPerRegion) failed: Must be in range.
brutisso
parents:
3986
diff
changeset
|
106 // instead of just is_in_reserved() here. |
|
5e5d4821bf07
7097516: G1: assert(0<= from_card && from_card<HeapRegion::CardsPerRegion) failed: Must be in range.
brutisso
parents:
3986
diff
changeset
|
107 if (loc_hr->is_in_reserved_raw(from)) { |
| 342 | 108 size_t hw_offset = pointer_delta((HeapWord*)from, loc_hr->bottom()); |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
109 CardIdx_t from_card = (CardIdx_t) |
|
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
110 hw_offset >> (CardTableModRefBS::card_shift - LogHeapWordSize); |
| 342 | 111 |
|
3986
65a8ff39a6da
7095194: G1: HeapRegion::GrainBytes, GrainWords, and CardsPerRegion should be size_t
johnc
parents:
3830
diff
changeset
|
112 assert(0 <= from_card && (size_t)from_card < HeapRegion::CardsPerRegion, |
|
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
113 "Must be in range."); |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
114 add_card_work(from_card, par); |
| 342 | 115 } |
| 116 } | |
| 117 | |
| 118 public: | |
| 119 | |
| 120 HeapRegion* hr() const { return _hr; } | |
| 121 | |
| 122 jint occupied() const { | |
| 123 // Overkill, but if we ever need it... | |
| 124 // guarantee(_occupied == _bm.count_one_bits(), "Check"); | |
| 125 return _occupied; | |
| 126 } | |
| 127 | |
| 6253 | 128 void init(HeapRegion* hr, bool clear_links_to_all_list) { |
| 129 if (clear_links_to_all_list) { | |
| 130 set_next(NULL); | |
| 131 set_prev(NULL); | |
| 132 } | |
| 342 | 133 _hr = hr; |
| 6253 | 134 _collision_list_next = NULL; |
| 342 | 135 _occupied = 0; |
| 136 _bm.clear(); | |
| 137 } | |
| 138 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
139 void add_reference(OopOrNarrowOopStar from) { |
| 342 | 140 add_reference_work(from, /*parallel*/ true); |
| 141 } | |
| 142 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
143 void seq_add_reference(OopOrNarrowOopStar from) { |
| 342 | 144 add_reference_work(from, /*parallel*/ false); |
| 145 } | |
| 146 | |
| 147 void scrub(CardTableModRefBS* ctbs, BitMap* card_bm) { | |
| 148 HeapWord* hr_bot = hr()->bottom(); | |
|
489
2494ab195856
6653214: MemoryPoolMXBean.setUsageThreshold() does not support large heap sizes.
swamyv
parents:
342
diff
changeset
|
149 size_t hr_first_card_index = ctbs->index_for(hr_bot); |
| 342 | 150 bm()->set_intersection_at_offset(*card_bm, hr_first_card_index); |
| 151 recount_occupied(); | |
| 152 } | |
| 153 | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
154 void add_card(CardIdx_t from_card_index) { |
| 342 | 155 add_card_work(from_card_index, /*parallel*/ true); |
| 156 } | |
| 157 | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
158 void seq_add_card(CardIdx_t from_card_index) { |
| 342 | 159 add_card_work(from_card_index, /*parallel*/ false); |
| 160 } | |
| 161 | |
| 162 // (Destructively) union the bitmap of the current table into the given | |
| 163 // bitmap (which is assumed to be of the same size.) | |
| 164 void union_bitmap_into(BitMap* bm) { | |
| 165 bm->set_union(_bm); | |
| 166 } | |
| 167 | |
| 168 // Mem size in bytes. | |
| 169 size_t mem_size() const { | |
| 170 return sizeof(this) + _bm.size_in_words() * HeapWordSize; | |
| 171 } | |
| 172 | |
| 173 // Requires "from" to be in "hr()". | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
174 bool contains_reference(OopOrNarrowOopStar from) const { |
| 342 | 175 assert(hr()->is_in_reserved(from), "Precondition."); |
| 176 size_t card_ind = pointer_delta(from, hr()->bottom(), | |
| 177 CardTableModRefBS::card_size); | |
| 178 return _bm.at(card_ind); | |
| 179 } | |
| 180 | |
| 6253 | 181 // Bulk-free the PRTs from prt to last, assumes that they are |
| 182 // linked together using their _next field. | |
| 183 static void bulk_free(PerRegionTable* prt, PerRegionTable* last) { | |
| 342 | 184 while (true) { |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
185 PerRegionTable* fl = _free_list; |
| 6253 | 186 last->set_next(fl); |
| 187 PerRegionTable* res = (PerRegionTable*) Atomic::cmpxchg_ptr(prt, &_free_list, fl); | |
| 188 if (res == fl) { | |
| 189 return; | |
| 190 } | |
| 342 | 191 } |
| 192 ShouldNotReachHere(); | |
| 193 } | |
| 194 | |
| 6253 | 195 static void free(PerRegionTable* prt) { |
| 196 bulk_free(prt, prt); | |
| 197 } | |
| 198 | |
| 199 // Returns an initialized PerRegionTable instance. | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
200 static PerRegionTable* alloc(HeapRegion* hr) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
201 PerRegionTable* fl = _free_list; |
| 342 | 202 while (fl != NULL) { |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
203 PerRegionTable* nxt = fl->next(); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
204 PerRegionTable* res = |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
205 (PerRegionTable*) |
| 342 | 206 Atomic::cmpxchg_ptr(nxt, &_free_list, fl); |
| 207 if (res == fl) { | |
| 6253 | 208 fl->init(hr, true); |
| 342 | 209 return fl; |
| 210 } else { | |
| 211 fl = _free_list; | |
| 212 } | |
| 213 } | |
| 214 assert(fl == NULL, "Loop condition."); | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
215 return new PerRegionTable(hr); |
| 342 | 216 } |
| 217 | |
| 6253 | 218 PerRegionTable* next() const { return _next; } |
| 219 void set_next(PerRegionTable* next) { _next = next; } | |
| 220 PerRegionTable* prev() const { return _prev; } | |
| 221 void set_prev(PerRegionTable* prev) { _prev = prev; } | |
| 222 | |
| 223 // Accessor and Modification routines for the pointer for the | |
| 224 // singly linked collision list that links the PRTs within the | |
| 225 // OtherRegionsTable::_fine_grain_regions hash table. | |
| 226 // | |
| 227 // It might be useful to also make the collision list doubly linked | |
| 228 // to avoid iteration over the collisions list during scrubbing/deletion. | |
| 229 // OTOH there might not be many collisions. | |
| 230 | |
| 231 PerRegionTable* collision_list_next() const { | |
| 232 return _collision_list_next; | |
| 233 } | |
| 234 | |
| 235 void set_collision_list_next(PerRegionTable* next) { | |
| 236 _collision_list_next = next; | |
| 237 } | |
| 238 | |
| 239 PerRegionTable** collision_list_next_addr() { | |
| 240 return &_collision_list_next; | |
| 241 } | |
| 242 | |
| 342 | 243 static size_t fl_mem_size() { |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
244 PerRegionTable* cur = _free_list; |
| 342 | 245 size_t res = 0; |
| 246 while (cur != NULL) { | |
|
10333
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
247 res += cur->mem_size(); |
| 342 | 248 cur = cur->next(); |
| 249 } | |
| 250 return res; | |
| 251 } | |
|
10333
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
252 |
|
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
253 static void test_fl_mem_size(); |
| 342 | 254 }; |
| 255 | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
256 PerRegionTable* PerRegionTable::_free_list = NULL; |
| 342 | 257 |
| 258 size_t OtherRegionsTable::_max_fine_entries = 0; | |
| 259 size_t OtherRegionsTable::_mod_max_fine_entries_mask = 0; | |
| 260 size_t OtherRegionsTable::_fine_eviction_stride = 0; | |
| 261 size_t OtherRegionsTable::_fine_eviction_sample_size = 0; | |
| 262 | |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
263 OtherRegionsTable::OtherRegionsTable(HeapRegion* hr, Mutex* m) : |
| 342 | 264 _g1h(G1CollectedHeap::heap()), |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
265 _hr(hr), _m(m), |
| 342 | 266 _coarse_map(G1CollectedHeap::heap()->max_regions(), |
| 267 false /* in-resource-area */), | |
| 268 _fine_grain_regions(NULL), | |
| 6253 | 269 _first_all_fine_prts(NULL), _last_all_fine_prts(NULL), |
| 342 | 270 _n_fine_entries(0), _n_coarse_entries(0), |
| 271 _fine_eviction_start(0), | |
| 272 _sparse_table(hr) | |
| 273 { | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
274 typedef PerRegionTable* PerRegionTablePtr; |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
275 |
| 342 | 276 if (_max_fine_entries == 0) { |
| 277 assert(_mod_max_fine_entries_mask == 0, "Both or none."); | |
|
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
278 size_t max_entries_log = (size_t)log2_long((jlong)G1RSetRegionEntries); |
|
6750
859cd1a76f8a
7197906: BlockOffsetArray::power_to_cards_back() needs to handle > 32 bit shifts
brutisso
parents:
6253
diff
changeset
|
279 _max_fine_entries = (size_t)1 << max_entries_log; |
| 342 | 280 _mod_max_fine_entries_mask = _max_fine_entries - 1; |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
281 |
| 342 | 282 assert(_fine_eviction_sample_size == 0 |
| 283 && _fine_eviction_stride == 0, "All init at same time."); | |
|
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
284 _fine_eviction_sample_size = MAX2((size_t)4, max_entries_log); |
| 342 | 285 _fine_eviction_stride = _max_fine_entries / _fine_eviction_sample_size; |
| 286 } | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
287 |
|
10271
f9be75d21404
8012902: remove use of global operator new - take 2
minqi
parents:
10193
diff
changeset
|
288 _fine_grain_regions = NEW_C_HEAP_ARRAY3(PerRegionTablePtr, _max_fine_entries, |
|
f9be75d21404
8012902: remove use of global operator new - take 2
minqi
parents:
10193
diff
changeset
|
289 mtGC, 0, AllocFailStrategy::RETURN_NULL); |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
290 |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
291 if (_fine_grain_regions == NULL) { |
|
10161
746b070f5022
8011661: Insufficient memory message says "malloc" when sometimes it should say "mmap"
ccheung
parents:
10135
diff
changeset
|
292 vm_exit_out_of_memory(sizeof(void*)*_max_fine_entries, OOM_MALLOC_ERROR, |
| 342 | 293 "Failed to allocate _fine_grain_entries."); |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
294 } |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
295 |
| 342 | 296 for (size_t i = 0; i < _max_fine_entries; i++) { |
| 297 _fine_grain_regions[i] = NULL; | |
| 298 } | |
| 299 } | |
| 300 | |
| 6253 | 301 void OtherRegionsTable::link_to_all(PerRegionTable* prt) { |
| 302 // We always append to the beginning of the list for convenience; | |
| 303 // the order of entries in this list does not matter. | |
| 304 if (_first_all_fine_prts != NULL) { | |
| 305 assert(_first_all_fine_prts->prev() == NULL, "invariant"); | |
| 306 _first_all_fine_prts->set_prev(prt); | |
| 307 prt->set_next(_first_all_fine_prts); | |
| 308 } else { | |
| 309 // this is the first element we insert. Adjust the "last" pointer | |
| 310 _last_all_fine_prts = prt; | |
| 311 assert(prt->next() == NULL, "just checking"); | |
| 312 } | |
| 313 // the new element is always the first element without a predecessor | |
| 314 prt->set_prev(NULL); | |
| 315 _first_all_fine_prts = prt; | |
| 316 | |
| 317 assert(prt->prev() == NULL, "just checking"); | |
| 318 assert(_first_all_fine_prts == prt, "just checking"); | |
| 319 assert((_first_all_fine_prts == NULL && _last_all_fine_prts == NULL) || | |
| 320 (_first_all_fine_prts != NULL && _last_all_fine_prts != NULL), | |
| 321 "just checking"); | |
| 322 assert(_last_all_fine_prts == NULL || _last_all_fine_prts->next() == NULL, | |
| 323 "just checking"); | |
| 324 assert(_first_all_fine_prts == NULL || _first_all_fine_prts->prev() == NULL, | |
| 325 "just checking"); | |
| 326 } | |
| 327 | |
| 328 void OtherRegionsTable::unlink_from_all(PerRegionTable* prt) { | |
| 329 if (prt->prev() != NULL) { | |
| 330 assert(_first_all_fine_prts != prt, "just checking"); | |
| 331 prt->prev()->set_next(prt->next()); | |
| 332 // removing the last element in the list? | |
| 333 if (_last_all_fine_prts == prt) { | |
| 334 _last_all_fine_prts = prt->prev(); | |
| 335 } | |
| 336 } else { | |
| 337 assert(_first_all_fine_prts == prt, "just checking"); | |
| 338 _first_all_fine_prts = prt->next(); | |
| 339 // list is empty now? | |
| 340 if (_first_all_fine_prts == NULL) { | |
| 341 _last_all_fine_prts = NULL; | |
| 342 } | |
| 343 } | |
| 344 | |
| 345 if (prt->next() != NULL) { | |
| 346 prt->next()->set_prev(prt->prev()); | |
| 347 } | |
| 348 | |
| 349 prt->set_next(NULL); | |
| 350 prt->set_prev(NULL); | |
| 351 | |
| 352 assert((_first_all_fine_prts == NULL && _last_all_fine_prts == NULL) || | |
| 353 (_first_all_fine_prts != NULL && _last_all_fine_prts != NULL), | |
| 354 "just checking"); | |
| 355 assert(_last_all_fine_prts == NULL || _last_all_fine_prts->next() == NULL, | |
| 356 "just checking"); | |
| 357 assert(_first_all_fine_prts == NULL || _first_all_fine_prts->prev() == NULL, | |
| 358 "just checking"); | |
| 359 } | |
| 360 | |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
361 int** FromCardCache::_cache = NULL; |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
362 uint FromCardCache::_max_regions = 0; |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
363 size_t FromCardCache::_static_mem_size = 0; |
| 342 | 364 |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
365 void FromCardCache::initialize(uint n_par_rs, uint max_num_regions) { |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
366 guarantee(_cache == NULL, "Should not call this multiple times"); |
| 342 | 367 |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
368 _max_regions = max_num_regions; |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
369 _cache = Padded2DArray<int, mtGC>::create_unfreeable(n_par_rs, |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
370 _max_regions, |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
371 &_static_mem_size); |
|
17754
d7070f371770
8035815: Cache-align and pad the from card cache
tschatzl
parents:
17753
diff
changeset
|
372 |
|
d7070f371770
8035815: Cache-align and pad the from card cache
tschatzl
parents:
17753
diff
changeset
|
373 for (uint i = 0; i < n_par_rs; i++) { |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
374 for (uint j = 0; j < _max_regions; j++) { |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
375 set(i, j, InvalidCard); |
| 342 | 376 } |
| 377 } | |
| 378 } | |
| 379 | |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
380 void FromCardCache::shrink(uint new_num_regions) { |
|
17754
d7070f371770
8035815: Cache-align and pad the from card cache
tschatzl
parents:
17753
diff
changeset
|
381 for (uint i = 0; i < HeapRegionRemSet::num_par_rem_sets(); i++) { |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
382 assert(new_num_regions <= _max_regions, "Must be within max."); |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
383 for (uint j = new_num_regions; j < _max_regions; j++) { |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
384 set(i, j, InvalidCard); |
| 342 | 385 } |
| 386 } | |
| 387 } | |
| 388 | |
| 389 #ifndef PRODUCT | |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
390 void FromCardCache::print(outputStream* out) { |
|
17754
d7070f371770
8035815: Cache-align and pad the from card cache
tschatzl
parents:
17753
diff
changeset
|
391 for (uint i = 0; i < HeapRegionRemSet::num_par_rem_sets(); i++) { |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
392 for (uint j = 0; j < _max_regions; j++) { |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
393 out->print_cr("_from_card_cache["UINT32_FORMAT"]["UINT32_FORMAT"] = "INT32_FORMAT".", |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
394 i, j, at(i, j)); |
| 342 | 395 } |
| 396 } | |
| 397 } | |
| 398 #endif | |
| 399 | |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
400 void FromCardCache::clear(uint region_idx) { |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
401 uint num_par_remsets = HeapRegionRemSet::num_par_rem_sets(); |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
402 for (uint i = 0; i < num_par_remsets; i++) { |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
403 set(i, region_idx, InvalidCard); |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
404 } |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
405 } |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
406 |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
407 void OtherRegionsTable::init_from_card_cache(uint max_regions) { |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
408 FromCardCache::initialize(HeapRegionRemSet::num_par_rem_sets(), max_regions); |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
409 } |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
410 |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
411 void OtherRegionsTable::shrink_from_card_cache(uint new_num_regions) { |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
412 FromCardCache::shrink(new_num_regions); |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
413 } |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
414 |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
415 void OtherRegionsTable::print_from_card_cache() { |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
416 FromCardCache::print(); |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
417 } |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
418 |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
419 void OtherRegionsTable::add_reference(OopOrNarrowOopStar from, int tid) { |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
420 uint cur_hrs_ind = hr()->hrs_index(); |
| 342 | 421 |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
422 if (G1TraceHeapRegionRememberedSet) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
423 gclog_or_tty->print_cr("ORT::add_reference_work(" PTR_FORMAT "->" PTR_FORMAT ").", |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
424 from, |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
425 UseCompressedOops |
|
12316
190899198332
7195622: CheckUnhandledOops has limited usefulness now
hseigel
parents:
12080
diff
changeset
|
426 ? (void *)oopDesc::load_decode_heap_oop((narrowOop*)from) |
|
190899198332
7195622: CheckUnhandledOops has limited usefulness now
hseigel
parents:
12080
diff
changeset
|
427 : (void *)oopDesc::load_decode_heap_oop((oop*)from)); |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
428 } |
| 342 | 429 |
| 430 int from_card = (int)(uintptr_t(from) >> CardTableModRefBS::card_shift); | |
| 431 | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
432 if (G1TraceHeapRegionRememberedSet) { |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
433 gclog_or_tty->print_cr("Table for [" PTR_FORMAT "...): card %d (cache = "INT32_FORMAT")", |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
434 hr()->bottom(), from_card, |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
435 FromCardCache::at((uint)tid, cur_hrs_ind)); |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
436 } |
| 342 | 437 |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
438 if (FromCardCache::contains_or_replace((uint)tid, cur_hrs_ind, from_card)) { |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
439 if (G1TraceHeapRegionRememberedSet) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
440 gclog_or_tty->print_cr(" from-card cache hit."); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
441 } |
| 342 | 442 assert(contains_reference(from), "We just added it!"); |
| 443 return; | |
| 444 } | |
| 445 | |
| 446 // Note that this may be a continued H region. | |
| 447 HeapRegion* from_hr = _g1h->heap_region_containing_raw(from); | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
448 RegionIdx_t from_hrs_ind = (RegionIdx_t) from_hr->hrs_index(); |
| 342 | 449 |
| 450 // If the region is already coarsened, return. | |
| 451 if (_coarse_map.at(from_hrs_ind)) { | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
452 if (G1TraceHeapRegionRememberedSet) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
453 gclog_or_tty->print_cr(" coarse map hit."); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
454 } |
| 342 | 455 assert(contains_reference(from), "We just added it!"); |
| 456 return; | |
| 457 } | |
| 458 | |
| 459 // Otherwise find a per-region table to add it to. | |
| 460 size_t ind = from_hrs_ind & _mod_max_fine_entries_mask; | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
461 PerRegionTable* prt = find_region_table(ind, from_hr); |
| 342 | 462 if (prt == NULL) { |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
463 MutexLockerEx x(_m, Mutex::_no_safepoint_check_flag); |
| 342 | 464 // Confirm that it's really not there... |
| 465 prt = find_region_table(ind, from_hr); | |
| 466 if (prt == NULL) { | |
| 467 | |
| 468 uintptr_t from_hr_bot_card_index = | |
| 469 uintptr_t(from_hr->bottom()) | |
| 470 >> CardTableModRefBS::card_shift; | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
471 CardIdx_t card_index = from_card - from_hr_bot_card_index; |
|
3986
65a8ff39a6da
7095194: G1: HeapRegion::GrainBytes, GrainWords, and CardsPerRegion should be size_t
johnc
parents:
3830
diff
changeset
|
472 assert(0 <= card_index && (size_t)card_index < HeapRegion::CardsPerRegion, |
| 342 | 473 "Must be in range."); |
| 474 if (G1HRRSUseSparseTable && | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
475 _sparse_table.add_card(from_hrs_ind, card_index)) { |
| 342 | 476 if (G1RecordHRRSOops) { |
| 477 HeapRegionRemSet::record(hr(), from); | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
478 if (G1TraceHeapRegionRememberedSet) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
479 gclog_or_tty->print(" Added card " PTR_FORMAT " to region " |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
480 "[" PTR_FORMAT "...) for ref " PTR_FORMAT ".\n", |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
481 align_size_down(uintptr_t(from), |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
482 CardTableModRefBS::card_size), |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
483 hr()->bottom(), from); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
484 } |
| 342 | 485 } |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
486 if (G1TraceHeapRegionRememberedSet) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
487 gclog_or_tty->print_cr(" added card to sparse table."); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
488 } |
| 342 | 489 assert(contains_reference_locked(from), "We just added it!"); |
| 490 return; | |
| 491 } else { | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
492 if (G1TraceHeapRegionRememberedSet) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
493 gclog_or_tty->print_cr(" [tid %d] sparse table entry " |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
494 "overflow(f: %d, t: %d)", |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
495 tid, from_hrs_ind, cur_hrs_ind); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
496 } |
| 342 | 497 } |
| 498 | |
| 499 if (_n_fine_entries == _max_fine_entries) { | |
| 500 prt = delete_region_table(); | |
| 6253 | 501 // There is no need to clear the links to the 'all' list here: |
| 502 // prt will be reused immediately, i.e. remain in the 'all' list. | |
| 503 prt->init(from_hr, false /* clear_links_to_all_list */); | |
| 342 | 504 } else { |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
505 prt = PerRegionTable::alloc(from_hr); |
| 6253 | 506 link_to_all(prt); |
| 342 | 507 } |
| 508 | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
509 PerRegionTable* first_prt = _fine_grain_regions[ind]; |
| 6253 | 510 prt->set_collision_list_next(first_prt); |
| 342 | 511 _fine_grain_regions[ind] = prt; |
| 512 _n_fine_entries++; | |
| 513 | |
| 514 if (G1HRRSUseSparseTable) { | |
|
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
515 // Transfer from sparse to fine-grain. |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
516 SparsePRTEntry *sprt_entry = _sparse_table.get_entry(from_hrs_ind); |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
517 assert(sprt_entry != NULL, "There should have been an entry"); |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
518 for (int i = 0; i < SparsePRTEntry::cards_num(); i++) { |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
519 CardIdx_t c = sprt_entry->card(i); |
| 342 | 520 if (c != SparsePRTEntry::NullEntry) { |
| 521 prt->add_card(c); | |
| 522 } | |
| 523 } | |
| 524 // Now we can delete the sparse entry. | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
525 bool res = _sparse_table.delete_entry(from_hrs_ind); |
| 342 | 526 assert(res, "It should have been there."); |
| 527 } | |
| 528 } | |
| 529 assert(prt != NULL && prt->hr() == from_hr, "consequence"); | |
| 530 } | |
| 531 // Note that we can't assert "prt->hr() == from_hr", because of the | |
| 532 // possibility of concurrent reuse. But see head comment of | |
| 533 // OtherRegionsTable for why this is OK. | |
| 534 assert(prt != NULL, "Inv"); | |
| 535 | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
536 prt->add_reference(from); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
537 |
| 342 | 538 if (G1RecordHRRSOops) { |
| 539 HeapRegionRemSet::record(hr(), from); | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
540 if (G1TraceHeapRegionRememberedSet) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
541 gclog_or_tty->print("Added card " PTR_FORMAT " to region " |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
542 "[" PTR_FORMAT "...) for ref " PTR_FORMAT ".\n", |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
543 align_size_down(uintptr_t(from), |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
544 CardTableModRefBS::card_size), |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
545 hr()->bottom(), from); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
546 } |
| 342 | 547 } |
| 548 assert(contains_reference(from), "We just added it!"); | |
| 549 } | |
| 550 | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
551 PerRegionTable* |
| 342 | 552 OtherRegionsTable::find_region_table(size_t ind, HeapRegion* hr) const { |
| 553 assert(0 <= ind && ind < _max_fine_entries, "Preconditions."); | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
554 PerRegionTable* prt = _fine_grain_regions[ind]; |
| 342 | 555 while (prt != NULL && prt->hr() != hr) { |
| 6253 | 556 prt = prt->collision_list_next(); |
| 342 | 557 } |
| 558 // Loop postcondition is the method postcondition. | |
| 559 return prt; | |
| 560 } | |
| 561 | |
| 562 jint OtherRegionsTable::_n_coarsenings = 0; | |
| 563 | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
564 PerRegionTable* OtherRegionsTable::delete_region_table() { |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
565 assert(_m->owned_by_self(), "Precondition"); |
| 342 | 566 assert(_n_fine_entries == _max_fine_entries, "Precondition"); |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
567 PerRegionTable* max = NULL; |
| 342 | 568 jint max_occ = 0; |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
569 PerRegionTable** max_prev; |
| 342 | 570 size_t max_ind; |
| 571 | |
| 572 size_t i = _fine_eviction_start; | |
| 573 for (size_t k = 0; k < _fine_eviction_sample_size; k++) { | |
| 574 size_t ii = i; | |
| 575 // Make sure we get a non-NULL sample. | |
| 576 while (_fine_grain_regions[ii] == NULL) { | |
| 577 ii++; | |
| 578 if (ii == _max_fine_entries) ii = 0; | |
| 579 guarantee(ii != i, "We must find one."); | |
| 580 } | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
581 PerRegionTable** prev = &_fine_grain_regions[ii]; |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
582 PerRegionTable* cur = *prev; |
| 342 | 583 while (cur != NULL) { |
| 584 jint cur_occ = cur->occupied(); | |
| 585 if (max == NULL || cur_occ > max_occ) { | |
| 586 max = cur; | |
| 587 max_prev = prev; | |
| 588 max_ind = i; | |
| 589 max_occ = cur_occ; | |
| 590 } | |
| 6253 | 591 prev = cur->collision_list_next_addr(); |
| 592 cur = cur->collision_list_next(); | |
| 342 | 593 } |
| 594 i = i + _fine_eviction_stride; | |
| 595 if (i >= _n_fine_entries) i = i - _n_fine_entries; | |
| 596 } | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
597 |
| 342 | 598 _fine_eviction_start++; |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
599 |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
600 if (_fine_eviction_start >= _n_fine_entries) { |
| 342 | 601 _fine_eviction_start -= _n_fine_entries; |
| 602 } | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
603 |
| 342 | 604 guarantee(max != NULL, "Since _n_fine_entries > 0"); |
| 605 | |
| 606 // Set the corresponding coarse bit. | |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
607 size_t max_hrs_index = (size_t) max->hr()->hrs_index(); |
| 342 | 608 if (!_coarse_map.at(max_hrs_index)) { |
| 609 _coarse_map.at_put(max_hrs_index, true); | |
| 610 _n_coarse_entries++; | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
611 if (G1TraceHeapRegionRememberedSet) { |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
612 gclog_or_tty->print("Coarsened entry in region [" PTR_FORMAT "...] " |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
613 "for region [" PTR_FORMAT "...] (%d coarse entries).\n", |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
614 hr()->bottom(), |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
615 max->hr()->bottom(), |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
616 _n_coarse_entries); |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
617 } |
| 342 | 618 } |
| 619 | |
| 620 // Unsplice. | |
| 6253 | 621 *max_prev = max->collision_list_next(); |
| 342 | 622 Atomic::inc(&_n_coarsenings); |
| 623 _n_fine_entries--; | |
| 624 return max; | |
| 625 } | |
| 626 | |
| 627 | |
| 628 // At present, this must be called stop-world single-threaded. | |
| 629 void OtherRegionsTable::scrub(CardTableModRefBS* ctbs, | |
| 630 BitMap* region_bm, BitMap* card_bm) { | |
| 631 // First eliminated garbage regions from the coarse map. | |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
632 if (G1RSScrubVerbose) { |
|
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
633 gclog_or_tty->print_cr("Scrubbing region %u:", hr()->hrs_index()); |
|
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
634 } |
| 342 | 635 |
| 636 assert(_coarse_map.size() == region_bm->size(), "Precondition"); | |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
637 if (G1RSScrubVerbose) { |
|
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
638 gclog_or_tty->print(" Coarse map: before = "SIZE_FORMAT"...", |
|
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
639 _n_coarse_entries); |
|
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
640 } |
| 342 | 641 _coarse_map.set_intersection(*region_bm); |
| 642 _n_coarse_entries = _coarse_map.count_one_bits(); | |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
643 if (G1RSScrubVerbose) { |
|
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
644 gclog_or_tty->print_cr(" after = "SIZE_FORMAT".", _n_coarse_entries); |
|
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
645 } |
| 342 | 646 |
| 647 // Now do the fine-grained maps. | |
| 648 for (size_t i = 0; i < _max_fine_entries; i++) { | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
649 PerRegionTable* cur = _fine_grain_regions[i]; |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
650 PerRegionTable** prev = &_fine_grain_regions[i]; |
| 342 | 651 while (cur != NULL) { |
| 6253 | 652 PerRegionTable* nxt = cur->collision_list_next(); |
| 342 | 653 // If the entire region is dead, eliminate. |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
654 if (G1RSScrubVerbose) { |
|
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
655 gclog_or_tty->print_cr(" For other region %u:", |
| 3766 | 656 cur->hr()->hrs_index()); |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
657 } |
|
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
658 if (!region_bm->at((size_t) cur->hr()->hrs_index())) { |
| 342 | 659 *prev = nxt; |
| 6253 | 660 cur->set_collision_list_next(NULL); |
| 342 | 661 _n_fine_entries--; |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
662 if (G1RSScrubVerbose) { |
| 342 | 663 gclog_or_tty->print_cr(" deleted via region map."); |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
664 } |
| 6253 | 665 unlink_from_all(cur); |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
666 PerRegionTable::free(cur); |
| 342 | 667 } else { |
| 668 // Do fine-grain elimination. | |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
669 if (G1RSScrubVerbose) { |
| 342 | 670 gclog_or_tty->print(" occ: before = %4d.", cur->occupied()); |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
671 } |
| 342 | 672 cur->scrub(ctbs, card_bm); |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
673 if (G1RSScrubVerbose) { |
| 342 | 674 gclog_or_tty->print_cr(" after = %4d.", cur->occupied()); |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
675 } |
| 342 | 676 // Did that empty the table completely? |
| 677 if (cur->occupied() == 0) { | |
| 678 *prev = nxt; | |
| 6253 | 679 cur->set_collision_list_next(NULL); |
| 342 | 680 _n_fine_entries--; |
| 6253 | 681 unlink_from_all(cur); |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
682 PerRegionTable::free(cur); |
| 342 | 683 } else { |
| 6253 | 684 prev = cur->collision_list_next_addr(); |
| 342 | 685 } |
| 686 } | |
| 687 cur = nxt; | |
| 688 } | |
| 689 } | |
| 690 // Since we may have deleted a from_card_cache entry from the RS, clear | |
| 691 // the FCC. | |
| 692 clear_fcc(); | |
| 693 } | |
| 694 | |
| 695 | |
| 696 size_t OtherRegionsTable::occupied() const { | |
| 697 size_t sum = occ_fine(); | |
| 698 sum += occ_sparse(); | |
| 699 sum += occ_coarse(); | |
| 700 return sum; | |
| 701 } | |
| 702 | |
| 703 size_t OtherRegionsTable::occ_fine() const { | |
| 704 size_t sum = 0; | |
| 6253 | 705 |
| 706 size_t num = 0; | |
| 707 PerRegionTable * cur = _first_all_fine_prts; | |
| 708 while (cur != NULL) { | |
| 709 sum += cur->occupied(); | |
| 710 cur = cur->next(); | |
| 711 num++; | |
| 342 | 712 } |
| 6253 | 713 guarantee(num == _n_fine_entries, "just checking"); |
| 342 | 714 return sum; |
| 715 } | |
| 716 | |
| 717 size_t OtherRegionsTable::occ_coarse() const { | |
|
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
718 return (_n_coarse_entries * HeapRegion::CardsPerRegion); |
| 342 | 719 } |
| 720 | |
| 721 size_t OtherRegionsTable::occ_sparse() const { | |
| 722 return _sparse_table.occupied(); | |
| 723 } | |
| 724 | |
| 725 size_t OtherRegionsTable::mem_size() const { | |
| 726 size_t sum = 0; | |
|
10290
05a17f270c7e
8014240: G1: Add remembered set size information to output of G1PrintRegionLivenessInfo
tschatzl
parents:
10193
diff
changeset
|
727 // all PRTs are of the same size so it is sufficient to query only one of them. |
|
05a17f270c7e
8014240: G1: Add remembered set size information to output of G1PrintRegionLivenessInfo
tschatzl
parents:
10193
diff
changeset
|
728 if (_first_all_fine_prts != NULL) { |
|
05a17f270c7e
8014240: G1: Add remembered set size information to output of G1PrintRegionLivenessInfo
tschatzl
parents:
10193
diff
changeset
|
729 assert(_last_all_fine_prts != NULL && |
|
05a17f270c7e
8014240: G1: Add remembered set size information to output of G1PrintRegionLivenessInfo
tschatzl
parents:
10193
diff
changeset
|
730 _first_all_fine_prts->mem_size() == _last_all_fine_prts->mem_size(), "check that mem_size() is constant"); |
|
05a17f270c7e
8014240: G1: Add remembered set size information to output of G1PrintRegionLivenessInfo
tschatzl
parents:
10193
diff
changeset
|
731 sum += _first_all_fine_prts->mem_size() * _n_fine_entries; |
| 342 | 732 } |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
733 sum += (sizeof(PerRegionTable*) * _max_fine_entries); |
| 342 | 734 sum += (_coarse_map.size_in_words() * HeapWordSize); |
| 735 sum += (_sparse_table.mem_size()); | |
| 736 sum += sizeof(*this) - sizeof(_sparse_table); // Avoid double counting above. | |
| 737 return sum; | |
| 738 } | |
| 739 | |
| 740 size_t OtherRegionsTable::static_mem_size() { | |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
741 return FromCardCache::static_mem_size(); |
| 342 | 742 } |
| 743 | |
| 744 size_t OtherRegionsTable::fl_mem_size() { | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
745 return PerRegionTable::fl_mem_size(); |
| 342 | 746 } |
| 747 | |
| 748 void OtherRegionsTable::clear_fcc() { | |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
749 FromCardCache::clear(hr()->hrs_index()); |
| 342 | 750 } |
| 751 | |
| 752 void OtherRegionsTable::clear() { | |
| 6253 | 753 // if there are no entries, skip this step |
| 754 if (_first_all_fine_prts != NULL) { | |
| 755 guarantee(_first_all_fine_prts != NULL && _last_all_fine_prts != NULL, "just checking"); | |
| 756 PerRegionTable::bulk_free(_first_all_fine_prts, _last_all_fine_prts); | |
| 757 memset(_fine_grain_regions, 0, _max_fine_entries * sizeof(_fine_grain_regions[0])); | |
| 758 } else { | |
| 759 guarantee(_first_all_fine_prts == NULL && _last_all_fine_prts == NULL, "just checking"); | |
| 342 | 760 } |
| 6253 | 761 |
| 762 _first_all_fine_prts = _last_all_fine_prts = NULL; | |
| 342 | 763 _sparse_table.clear(); |
| 764 _coarse_map.clear(); | |
| 765 _n_fine_entries = 0; | |
| 766 _n_coarse_entries = 0; | |
| 767 | |
| 768 clear_fcc(); | |
| 769 } | |
| 770 | |
| 771 void OtherRegionsTable::clear_incoming_entry(HeapRegion* from_hr) { | |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
772 MutexLockerEx x(_m, Mutex::_no_safepoint_check_flag); |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
773 size_t hrs_ind = (size_t) from_hr->hrs_index(); |
| 342 | 774 size_t ind = hrs_ind & _mod_max_fine_entries_mask; |
| 775 if (del_single_region_table(ind, from_hr)) { | |
| 776 assert(!_coarse_map.at(hrs_ind), "Inv"); | |
| 777 } else { | |
| 778 _coarse_map.par_at_put(hrs_ind, 0); | |
| 779 } | |
| 780 // Check to see if any of the fcc entries come from here. | |
|
17754
d7070f371770
8035815: Cache-align and pad the from card cache
tschatzl
parents:
17753
diff
changeset
|
781 uint hr_ind = hr()->hrs_index(); |
|
d7070f371770
8035815: Cache-align and pad the from card cache
tschatzl
parents:
17753
diff
changeset
|
782 for (uint tid = 0; tid < HeapRegionRemSet::num_par_rem_sets(); tid++) { |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
783 int fcc_ent = FromCardCache::at(tid, hr_ind); |
|
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
784 if (fcc_ent != FromCardCache::InvalidCard) { |
| 342 | 785 HeapWord* card_addr = (HeapWord*) |
| 786 (uintptr_t(fcc_ent) << CardTableModRefBS::card_shift); | |
| 787 if (hr()->is_in_reserved(card_addr)) { | |
| 788 // Clear the from card cache. | |
|
17758
ae7336d6337e
8034868: Extract G1 From Card Cache into separate class
tschatzl
parents:
17755
diff
changeset
|
789 FromCardCache::set(tid, hr_ind, FromCardCache::InvalidCard); |
| 342 | 790 } |
| 791 } | |
| 792 } | |
| 793 } | |
| 794 | |
| 795 bool OtherRegionsTable::del_single_region_table(size_t ind, | |
| 796 HeapRegion* hr) { | |
| 797 assert(0 <= ind && ind < _max_fine_entries, "Preconditions."); | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
798 PerRegionTable** prev_addr = &_fine_grain_regions[ind]; |
|
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
799 PerRegionTable* prt = *prev_addr; |
| 342 | 800 while (prt != NULL && prt->hr() != hr) { |
| 6253 | 801 prev_addr = prt->collision_list_next_addr(); |
| 802 prt = prt->collision_list_next(); | |
| 342 | 803 } |
| 804 if (prt != NULL) { | |
| 805 assert(prt->hr() == hr, "Loop postcondition."); | |
| 6253 | 806 *prev_addr = prt->collision_list_next(); |
| 807 unlink_from_all(prt); | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
808 PerRegionTable::free(prt); |
| 342 | 809 _n_fine_entries--; |
| 810 return true; | |
| 811 } else { | |
| 812 return false; | |
| 813 } | |
| 814 } | |
| 815 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
816 bool OtherRegionsTable::contains_reference(OopOrNarrowOopStar from) const { |
| 342 | 817 // Cast away const in this case. |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
818 MutexLockerEx x((Mutex*)_m, Mutex::_no_safepoint_check_flag); |
| 342 | 819 return contains_reference_locked(from); |
| 820 } | |
| 821 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
822 bool OtherRegionsTable::contains_reference_locked(OopOrNarrowOopStar from) const { |
| 342 | 823 HeapRegion* hr = _g1h->heap_region_containing_raw(from); |
| 824 if (hr == NULL) return false; | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
825 RegionIdx_t hr_ind = (RegionIdx_t) hr->hrs_index(); |
| 342 | 826 // Is this region in the coarse map? |
| 827 if (_coarse_map.at(hr_ind)) return true; | |
| 828 | |
|
6188
7994a5a35fcf
6921087: G1: remove per-GC-thread expansion tables from the fine-grain remembered sets
johnc
parents:
6107
diff
changeset
|
829 PerRegionTable* prt = find_region_table(hr_ind & _mod_max_fine_entries_mask, |
| 342 | 830 hr); |
| 831 if (prt != NULL) { | |
| 832 return prt->contains_reference(from); | |
| 833 | |
| 834 } else { | |
| 835 uintptr_t from_card = | |
| 836 (uintptr_t(from) >> CardTableModRefBS::card_shift); | |
| 837 uintptr_t hr_bot_card_index = | |
| 838 uintptr_t(hr->bottom()) >> CardTableModRefBS::card_shift; | |
| 839 assert(from_card >= hr_bot_card_index, "Inv"); | |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
840 CardIdx_t card_index = from_card - hr_bot_card_index; |
|
3986
65a8ff39a6da
7095194: G1: HeapRegion::GrainBytes, GrainWords, and CardsPerRegion should be size_t
johnc
parents:
3830
diff
changeset
|
841 assert(0 <= card_index && (size_t)card_index < HeapRegion::CardsPerRegion, |
|
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
842 "Must be in range."); |
|
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
795
diff
changeset
|
843 return _sparse_table.contains_card(hr_ind, card_index); |
| 342 | 844 } |
| 845 } | |
| 846 | |
| 2173 | 847 void |
| 848 OtherRegionsTable::do_cleanup_work(HRRSCleanupTask* hrrs_cleanup_task) { | |
| 849 _sparse_table.do_cleanup_work(hrrs_cleanup_task); | |
| 850 } | |
| 851 | |
|
795
215f81b4d9b3
6841831: G1: assert(contains_reference(from),"We just added it!") fires
iveresov
parents:
794
diff
changeset
|
852 // Determines how many threads can add records to an rset in parallel. |
|
215f81b4d9b3
6841831: G1: assert(contains_reference(from),"We just added it!") fires
iveresov
parents:
794
diff
changeset
|
853 // This can be done by either mutator threads together with the |
|
215f81b4d9b3
6841831: G1: assert(contains_reference(from),"We just added it!") fires
iveresov
parents:
794
diff
changeset
|
854 // concurrent refinement threads or GC threads. |
|
17754
d7070f371770
8035815: Cache-align and pad the from card cache
tschatzl
parents:
17753
diff
changeset
|
855 uint HeapRegionRemSet::num_par_rem_sets() { |
|
d7070f371770
8035815: Cache-align and pad the from card cache
tschatzl
parents:
17753
diff
changeset
|
856 return (uint)MAX2(DirtyCardQueueSet::num_par_ids() + ConcurrentG1Refine::thread_num(), ParallelGCThreads); |
| 342 | 857 } |
| 858 | |
| 859 HeapRegionRemSet::HeapRegionRemSet(G1BlockOffsetSharedArray* bosa, | |
| 860 HeapRegion* hr) | |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
861 : _bosa(bosa), |
|
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
862 _m(Mutex::leaf, FormatBuffer<128>("HeapRegionRemSet lock #"UINT32_FORMAT, hr->hrs_index()), true), |
|
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
863 _code_roots(), _other_regions(hr, &_m) { |
|
3777
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
864 reset_for_par_iteration(); |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
865 } |
| 342 | 866 |
|
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
867 void HeapRegionRemSet::setup_remset_size() { |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
868 // Setup sparse and fine-grain tables sizes. |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
869 // table_size = base * (log(region_size / 1M) + 1) |
|
6107
5c8bd7c16119
7171936: LOG_G incorrectly defined in globalDefinitions.hpp
brutisso
parents:
6010
diff
changeset
|
870 const int LOG_M = 20; |
|
5c8bd7c16119
7171936: LOG_G incorrectly defined in globalDefinitions.hpp
brutisso
parents:
6010
diff
changeset
|
871 int region_size_log_mb = MAX2(HeapRegion::LogOfHRGrainBytes - LOG_M, 0); |
|
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
872 if (FLAG_IS_DEFAULT(G1RSetSparseRegionEntries)) { |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
873 G1RSetSparseRegionEntries = G1RSetSparseRegionEntriesBase * (region_size_log_mb + 1); |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
874 } |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
875 if (FLAG_IS_DEFAULT(G1RSetRegionEntries)) { |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
876 G1RSetRegionEntries = G1RSetRegionEntriesBase * (region_size_log_mb + 1); |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
877 } |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
878 guarantee(G1RSetSparseRegionEntries > 0 && G1RSetRegionEntries > 0 , "Sanity"); |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
879 } |
|
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
880 |
| 342 | 881 bool HeapRegionRemSet::claim_iter() { |
| 882 if (_iter_state != Unclaimed) return false; | |
| 883 jint res = Atomic::cmpxchg(Claimed, (jint*)(&_iter_state), Unclaimed); | |
| 884 return (res == Unclaimed); | |
| 885 } | |
| 886 | |
| 887 void HeapRegionRemSet::set_iter_complete() { | |
| 888 _iter_state = Complete; | |
| 889 } | |
| 890 | |
| 891 bool HeapRegionRemSet::iter_is_complete() { | |
| 892 return _iter_state == Complete; | |
| 893 } | |
| 894 | |
| 895 #ifndef PRODUCT | |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
896 void HeapRegionRemSet::print() { |
|
10182
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
897 HeapRegionRemSetIterator iter(this); |
| 342 | 898 size_t card_index; |
| 899 while (iter.has_next(card_index)) { | |
| 900 HeapWord* card_start = | |
| 901 G1CollectedHeap::heap()->bot_shared()->address_for_index(card_index); | |
|
3777
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
902 gclog_or_tty->print_cr(" Card " PTR_FORMAT, card_start); |
| 342 | 903 } |
| 904 if (iter.n_yielded() != occupied()) { | |
| 905 gclog_or_tty->print_cr("Yielded disagrees with occupied:"); | |
| 906 gclog_or_tty->print_cr(" %6d yielded (%6d coarse, %6d fine).", | |
| 907 iter.n_yielded(), | |
| 908 iter.n_yielded_coarse(), iter.n_yielded_fine()); | |
| 909 gclog_or_tty->print_cr(" %6d occ (%6d coarse, %6d fine).", | |
| 910 occupied(), occ_coarse(), occ_fine()); | |
| 911 } | |
| 912 guarantee(iter.n_yielded() == occupied(), | |
| 913 "We should have yielded all the represented cards."); | |
| 914 } | |
| 915 #endif | |
| 916 | |
| 917 void HeapRegionRemSet::cleanup() { | |
| 918 SparsePRT::cleanup_all(); | |
| 919 } | |
| 920 | |
| 921 void HeapRegionRemSet::clear() { | |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
922 MutexLockerEx x(&_m, Mutex::_no_safepoint_check_flag); |
|
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
923 clear_locked(); |
|
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
924 } |
| 12080 | 925 |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
926 void HeapRegionRemSet::clear_locked() { |
|
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
927 _code_roots.clear(); |
| 342 | 928 _other_regions.clear(); |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
929 assert(occupied_locked() == 0, "Should be clear."); |
|
3777
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
930 reset_for_par_iteration(); |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
931 } |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
932 |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
933 void HeapRegionRemSet::reset_for_par_iteration() { |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
934 _iter_state = Unclaimed; |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
935 _iter_claimed = 0; |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
936 // It's good to check this to make sure that the two methods are in sync. |
|
e8b0b0392037
7046182: G1: remove unnecessary iterations over the collection set
tonyp
parents:
3766
diff
changeset
|
937 assert(verify_ready_for_par_iteration(), "post-condition"); |
| 342 | 938 } |
| 939 | |
| 940 void HeapRegionRemSet::scrub(CardTableModRefBS* ctbs, | |
| 941 BitMap* region_bm, BitMap* card_bm) { | |
| 942 _other_regions.scrub(ctbs, region_bm, card_bm); | |
| 943 } | |
| 944 | |
| 12080 | 945 // Code roots support |
| 946 | |
| 947 void HeapRegionRemSet::add_strong_code_root(nmethod* nm) { | |
| 948 assert(nm != NULL, "sanity"); | |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
949 _code_roots.add(nm); |
| 12080 | 950 } |
| 951 | |
| 952 void HeapRegionRemSet::remove_strong_code_root(nmethod* nm) { | |
| 953 assert(nm != NULL, "sanity"); | |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
954 _code_roots.remove(nm); |
| 12080 | 955 // Check that there were no duplicates |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
956 guarantee(!_code_roots.contains(nm), "duplicate entry found"); |
| 12080 | 957 } |
| 958 | |
| 959 class NMethodMigrationOopClosure : public OopClosure { | |
| 960 G1CollectedHeap* _g1h; | |
| 961 HeapRegion* _from; | |
| 962 nmethod* _nm; | |
| 963 | |
| 964 uint _num_self_forwarded; | |
| 965 | |
| 966 template <class T> void do_oop_work(T* p) { | |
| 967 T heap_oop = oopDesc::load_heap_oop(p); | |
| 968 if (!oopDesc::is_null(heap_oop)) { | |
| 969 oop obj = oopDesc::decode_heap_oop_not_null(heap_oop); | |
| 970 if (_from->is_in(obj)) { | |
| 971 // Reference still points into the source region. | |
| 972 // Since roots are immediately evacuated this means that | |
| 973 // we must have self forwarded the object | |
| 974 assert(obj->is_forwarded(), | |
| 975 err_msg("code roots should be immediately evacuated. " | |
| 976 "Ref: "PTR_FORMAT", " | |
| 977 "Obj: "PTR_FORMAT", " | |
| 978 "Region: "HR_FORMAT, | |
| 979 p, (void*) obj, HR_FORMAT_PARAMS(_from))); | |
| 980 assert(obj->forwardee() == obj, | |
| 981 err_msg("not self forwarded? obj = "PTR_FORMAT, (void*)obj)); | |
| 982 | |
| 983 // The object has been self forwarded. | |
| 984 // Note, if we're during an initial mark pause, there is | |
| 985 // no need to explicitly mark object. It will be marked | |
| 986 // during the regular evacuation failure handling code. | |
| 987 _num_self_forwarded++; | |
| 988 } else { | |
| 989 // The reference points into a promotion or to-space region | |
| 990 HeapRegion* to = _g1h->heap_region_containing(obj); | |
| 991 to->rem_set()->add_strong_code_root(_nm); | |
| 992 } | |
| 993 } | |
| 994 } | |
| 995 | |
| 996 public: | |
| 997 NMethodMigrationOopClosure(G1CollectedHeap* g1h, HeapRegion* from, nmethod* nm): | |
| 998 _g1h(g1h), _from(from), _nm(nm), _num_self_forwarded(0) {} | |
| 999 | |
| 1000 void do_oop(narrowOop* p) { do_oop_work(p); } | |
| 1001 void do_oop(oop* p) { do_oop_work(p); } | |
| 1002 | |
| 1003 uint retain() { return _num_self_forwarded > 0; } | |
| 1004 }; | |
| 1005 | |
| 1006 void HeapRegionRemSet::migrate_strong_code_roots() { | |
| 1007 assert(hr()->in_collection_set(), "only collection set regions"); | |
|
13062
28674af341ac
8027756: assert(!hr->isHumongous()) failed: code root in humongous region?
tschatzl
parents:
12316
diff
changeset
|
1008 assert(!hr()->isHumongous(), |
|
28674af341ac
8027756: assert(!hr->isHumongous()) failed: code root in humongous region?
tschatzl
parents:
12316
diff
changeset
|
1009 err_msg("humongous region "HR_FORMAT" should not have been added to the collection set", |
|
28674af341ac
8027756: assert(!hr->isHumongous()) failed: code root in humongous region?
tschatzl
parents:
12316
diff
changeset
|
1010 HR_FORMAT_PARAMS(hr()))); |
| 12080 | 1011 |
| 1012 ResourceMark rm; | |
| 1013 | |
| 1014 // List of code blobs to retain for this region | |
| 1015 GrowableArray<nmethod*> to_be_retained(10); | |
| 1016 G1CollectedHeap* g1h = G1CollectedHeap::heap(); | |
| 1017 | |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
1018 while (!_code_roots.is_empty()) { |
|
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
1019 nmethod *nm = _code_roots.pop(); |
| 12080 | 1020 if (nm != NULL) { |
| 1021 NMethodMigrationOopClosure oop_cl(g1h, hr(), nm); | |
| 1022 nm->oops_do(&oop_cl); | |
| 1023 if (oop_cl.retain()) { | |
| 1024 to_be_retained.push(nm); | |
| 1025 } | |
| 1026 } | |
| 1027 } | |
| 1028 | |
| 1029 // Now push any code roots we need to retain | |
| 1030 assert(to_be_retained.is_empty() || hr()->evacuation_failed(), | |
| 1031 "Retained nmethod list must be empty or " | |
| 1032 "evacuation of this region failed"); | |
| 1033 | |
| 1034 while (to_be_retained.is_nonempty()) { | |
| 1035 nmethod* nm = to_be_retained.pop(); | |
| 1036 assert(nm != NULL, "sanity"); | |
| 1037 add_strong_code_root(nm); | |
| 1038 } | |
| 1039 } | |
| 1040 | |
| 1041 void HeapRegionRemSet::strong_code_roots_do(CodeBlobClosure* blk) const { | |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
1042 _code_roots.nmethods_do(blk); |
| 12080 | 1043 } |
| 1044 | |
| 1045 size_t HeapRegionRemSet::strong_code_roots_mem_size() { | |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
1046 return _code_roots.mem_size(); |
| 12080 | 1047 } |
| 1048 | |
| 342 | 1049 //-------------------- Iteration -------------------- |
| 1050 | |
|
17753
191174b49bec
8035406: Improve data structure for Code Cache remembered sets
tschatzl
parents:
13062
diff
changeset
|
1051 HeapRegionRemSetIterator:: HeapRegionRemSetIterator(HeapRegionRemSet* hrrs) : |
|
10182
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1052 _hrrs(hrrs), |
| 342 | 1053 _g1h(G1CollectedHeap::heap()), |
|
10182
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1054 _coarse_map(&hrrs->_other_regions._coarse_map), |
|
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1055 _fine_grain_regions(hrrs->_other_regions._fine_grain_regions), |
|
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1056 _bosa(hrrs->bosa()), |
|
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1057 _is(Sparse), |
| 342 | 1058 // Set these values so that we increment to the first region. |
|
10182
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1059 _coarse_cur_region_index(-1), |
|
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1060 _coarse_cur_region_cur_card(HeapRegion::CardsPerRegion-1), |
|
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1061 _cur_region_cur_card(0), |
|
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1062 _fine_array_index(-1), |
|
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1063 _fine_cur_prt(NULL), |
|
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1064 _n_yielded_coarse(0), |
|
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1065 _n_yielded_fine(0), |
|
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1066 _n_yielded_sparse(0), |
|
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1067 _sparse_iter(&hrrs->_other_regions._sparse_table) {} |
| 342 | 1068 |
| 1069 bool HeapRegionRemSetIterator::coarse_has_next(size_t& card_index) { | |
| 1070 if (_hrrs->_other_regions._n_coarse_entries == 0) return false; | |
| 1071 // Go to the next card. | |
| 1072 _coarse_cur_region_cur_card++; | |
| 1073 // Was the last the last card in the current region? | |
|
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
1074 if (_coarse_cur_region_cur_card == HeapRegion::CardsPerRegion) { |
| 342 | 1075 // Yes: find the next region. This may leave _coarse_cur_region_index |
| 1076 // Set to the last index, in which case there are no more coarse | |
| 1077 // regions. | |
| 1078 _coarse_cur_region_index = | |
| 1079 (int) _coarse_map->get_next_one_offset(_coarse_cur_region_index + 1); | |
| 1080 if ((size_t)_coarse_cur_region_index < _coarse_map->size()) { | |
| 1081 _coarse_cur_region_cur_card = 0; | |
| 1082 HeapWord* r_bot = | |
|
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
4020
diff
changeset
|
1083 _g1h->region_at((uint) _coarse_cur_region_index)->bottom(); |
| 342 | 1084 _cur_region_card_offset = _bosa->index_for(r_bot); |
| 1085 } else { | |
| 1086 return false; | |
| 1087 } | |
| 1088 } | |
| 1089 // If we didn't return false above, then we can yield a card. | |
| 1090 card_index = _cur_region_card_offset + _coarse_cur_region_cur_card; | |
| 1091 return true; | |
| 1092 } | |
| 1093 | |
| 1094 void HeapRegionRemSetIterator::fine_find_next_non_null_prt() { | |
| 1095 // Otherwise, find the next bucket list in the array. | |
| 1096 _fine_array_index++; | |
| 1097 while (_fine_array_index < (int) OtherRegionsTable::_max_fine_entries) { | |
| 1098 _fine_cur_prt = _fine_grain_regions[_fine_array_index]; | |
| 1099 if (_fine_cur_prt != NULL) return; | |
| 1100 else _fine_array_index++; | |
| 1101 } | |
| 1102 assert(_fine_cur_prt == NULL, "Loop post"); | |
| 1103 } | |
| 1104 | |
| 1105 bool HeapRegionRemSetIterator::fine_has_next(size_t& card_index) { | |
| 1106 if (fine_has_next()) { | |
| 1107 _cur_region_cur_card = | |
| 1108 _fine_cur_prt->_bm.get_next_one_offset(_cur_region_cur_card + 1); | |
| 1109 } | |
| 1110 while (!fine_has_next()) { | |
|
942
2c79770d1f6e
6819085: G1: use larger and/or user settable region size
tonyp
parents:
845
diff
changeset
|
1111 if (_cur_region_cur_card == (size_t) HeapRegion::CardsPerRegion) { |
| 342 | 1112 _cur_region_cur_card = 0; |
| 6253 | 1113 _fine_cur_prt = _fine_cur_prt->collision_list_next(); |
| 342 | 1114 } |
| 1115 if (_fine_cur_prt == NULL) { | |
| 1116 fine_find_next_non_null_prt(); | |
| 1117 if (_fine_cur_prt == NULL) return false; | |
| 1118 } | |
| 1119 assert(_fine_cur_prt != NULL && _cur_region_cur_card == 0, | |
| 1120 "inv."); | |
| 1121 HeapWord* r_bot = | |
| 1122 _fine_cur_prt->hr()->bottom(); | |
| 1123 _cur_region_card_offset = _bosa->index_for(r_bot); | |
| 1124 _cur_region_cur_card = _fine_cur_prt->_bm.get_next_one_offset(0); | |
| 1125 } | |
| 1126 assert(fine_has_next(), "Or else we exited the loop via the return."); | |
| 1127 card_index = _cur_region_card_offset + _cur_region_cur_card; | |
| 1128 return true; | |
| 1129 } | |
| 1130 | |
| 1131 bool HeapRegionRemSetIterator::fine_has_next() { | |
| 1132 return | |
| 1133 _fine_cur_prt != NULL && | |
|
3986
65a8ff39a6da
7095194: G1: HeapRegion::GrainBytes, GrainWords, and CardsPerRegion should be size_t
johnc
parents:
3830
diff
changeset
|
1134 _cur_region_cur_card < HeapRegion::CardsPerRegion; |
| 342 | 1135 } |
| 1136 | |
| 1137 bool HeapRegionRemSetIterator::has_next(size_t& card_index) { | |
| 1138 switch (_is) { | |
| 1139 case Sparse: | |
| 1140 if (_sparse_iter.has_next(card_index)) { | |
| 1141 _n_yielded_sparse++; | |
| 1142 return true; | |
| 1143 } | |
| 1144 // Otherwise, deliberate fall-through | |
| 1145 _is = Fine; | |
| 1146 case Fine: | |
| 1147 if (fine_has_next(card_index)) { | |
| 1148 _n_yielded_fine++; | |
| 1149 return true; | |
| 1150 } | |
| 1151 // Otherwise, deliberate fall-through | |
| 1152 _is = Coarse; | |
| 1153 case Coarse: | |
| 1154 if (coarse_has_next(card_index)) { | |
| 1155 _n_yielded_coarse++; | |
| 1156 return true; | |
| 1157 } | |
| 1158 // Otherwise... | |
| 1159 break; | |
| 1160 } | |
| 1161 assert(ParallelGCThreads > 1 || | |
| 1162 n_yielded() == _hrrs->occupied(), | |
| 1163 "Should have yielded all the cards in the rem set " | |
| 1164 "(in the non-par case)."); | |
| 1165 return false; | |
| 1166 } | |
| 1167 | |
| 1168 | |
| 1169 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1170 OopOrNarrowOopStar* HeapRegionRemSet::_recorded_oops = NULL; |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1171 HeapWord** HeapRegionRemSet::_recorded_cards = NULL; |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1172 HeapRegion** HeapRegionRemSet::_recorded_regions = NULL; |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1173 int HeapRegionRemSet::_n_recorded = 0; |
| 342 | 1174 |
| 1175 HeapRegionRemSet::Event* HeapRegionRemSet::_recorded_events = NULL; | |
| 1176 int* HeapRegionRemSet::_recorded_event_index = NULL; | |
| 1177 int HeapRegionRemSet::_n_recorded_events = 0; | |
| 1178 | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1179 void HeapRegionRemSet::record(HeapRegion* hr, OopOrNarrowOopStar f) { |
| 342 | 1180 if (_recorded_oops == NULL) { |
| 1181 assert(_n_recorded == 0 | |
| 1182 && _recorded_cards == NULL | |
| 1183 && _recorded_regions == NULL, | |
| 1184 "Inv"); | |
| 6197 | 1185 _recorded_oops = NEW_C_HEAP_ARRAY(OopOrNarrowOopStar, MaxRecorded, mtGC); |
| 1186 _recorded_cards = NEW_C_HEAP_ARRAY(HeapWord*, MaxRecorded, mtGC); | |
| 1187 _recorded_regions = NEW_C_HEAP_ARRAY(HeapRegion*, MaxRecorded, mtGC); | |
| 342 | 1188 } |
| 1189 if (_n_recorded == MaxRecorded) { | |
| 1190 gclog_or_tty->print_cr("Filled up 'recorded' (%d).", MaxRecorded); | |
| 1191 } else { | |
| 1192 _recorded_cards[_n_recorded] = | |
| 1193 (HeapWord*)align_size_down(uintptr_t(f), | |
| 1194 CardTableModRefBS::card_size); | |
| 1195 _recorded_oops[_n_recorded] = f; | |
| 1196 _recorded_regions[_n_recorded] = hr; | |
| 1197 _n_recorded++; | |
| 1198 } | |
| 1199 } | |
| 1200 | |
| 1201 void HeapRegionRemSet::record_event(Event evnt) { | |
| 1202 if (!G1RecordHRRSEvents) return; | |
| 1203 | |
| 1204 if (_recorded_events == NULL) { | |
| 1205 assert(_n_recorded_events == 0 | |
| 1206 && _recorded_event_index == NULL, | |
| 1207 "Inv"); | |
| 6197 | 1208 _recorded_events = NEW_C_HEAP_ARRAY(Event, MaxRecordedEvents, mtGC); |
| 1209 _recorded_event_index = NEW_C_HEAP_ARRAY(int, MaxRecordedEvents, mtGC); | |
| 342 | 1210 } |
| 1211 if (_n_recorded_events == MaxRecordedEvents) { | |
| 1212 gclog_or_tty->print_cr("Filled up 'recorded_events' (%d).", MaxRecordedEvents); | |
| 1213 } else { | |
| 1214 _recorded_events[_n_recorded_events] = evnt; | |
| 1215 _recorded_event_index[_n_recorded_events] = _n_recorded; | |
| 1216 _n_recorded_events++; | |
| 1217 } | |
| 1218 } | |
| 1219 | |
| 1220 void HeapRegionRemSet::print_event(outputStream* str, Event evnt) { | |
| 1221 switch (evnt) { | |
| 1222 case Event_EvacStart: | |
| 1223 str->print("Evac Start"); | |
| 1224 break; | |
| 1225 case Event_EvacEnd: | |
| 1226 str->print("Evac End"); | |
| 1227 break; | |
| 1228 case Event_RSUpdateEnd: | |
| 1229 str->print("RS Update End"); | |
| 1230 break; | |
| 1231 } | |
| 1232 } | |
| 1233 | |
| 1234 void HeapRegionRemSet::print_recorded() { | |
| 1235 int cur_evnt = 0; | |
| 1236 Event cur_evnt_kind; | |
| 1237 int cur_evnt_ind = 0; | |
| 1238 if (_n_recorded_events > 0) { | |
| 1239 cur_evnt_kind = _recorded_events[cur_evnt]; | |
| 1240 cur_evnt_ind = _recorded_event_index[cur_evnt]; | |
| 1241 } | |
| 1242 | |
| 1243 for (int i = 0; i < _n_recorded; i++) { | |
| 1244 while (cur_evnt < _n_recorded_events && i == cur_evnt_ind) { | |
| 1245 gclog_or_tty->print("Event: "); | |
| 1246 print_event(gclog_or_tty, cur_evnt_kind); | |
| 1247 gclog_or_tty->print_cr(""); | |
| 1248 cur_evnt++; | |
| 1249 if (cur_evnt < MaxRecordedEvents) { | |
| 1250 cur_evnt_kind = _recorded_events[cur_evnt]; | |
| 1251 cur_evnt_ind = _recorded_event_index[cur_evnt]; | |
| 1252 } | |
| 1253 } | |
| 1254 gclog_or_tty->print("Added card " PTR_FORMAT " to region [" PTR_FORMAT "...]" | |
| 1255 " for ref " PTR_FORMAT ".\n", | |
| 1256 _recorded_cards[i], _recorded_regions[i]->bottom(), | |
| 1257 _recorded_oops[i]); | |
| 1258 } | |
| 1259 } | |
| 1260 | |
| 2173 | 1261 void HeapRegionRemSet::reset_for_cleanup_tasks() { |
| 1262 SparsePRT::reset_for_cleanup_tasks(); | |
| 1263 } | |
| 1264 | |
| 1265 void HeapRegionRemSet::do_cleanup_work(HRRSCleanupTask* hrrs_cleanup_task) { | |
| 1266 _other_regions.do_cleanup_work(hrrs_cleanup_task); | |
| 1267 } | |
| 1268 | |
| 1269 void | |
| 1270 HeapRegionRemSet::finish_cleanup_task(HRRSCleanupTask* hrrs_cleanup_task) { | |
| 1271 SparsePRT::finish_cleanup_task(hrrs_cleanup_task); | |
| 1272 } | |
| 1273 | |
| 342 | 1274 #ifndef PRODUCT |
|
10333
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
1275 void PerRegionTable::test_fl_mem_size() { |
|
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
1276 PerRegionTable* dummy = alloc(NULL); |
|
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
1277 free(dummy); |
|
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
1278 guarantee(dummy->mem_size() == fl_mem_size(), "fl_mem_size() does not return the correct element size"); |
|
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
1279 // try to reset the state |
|
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
1280 _free_list = NULL; |
|
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
1281 delete dummy; |
|
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
1282 } |
|
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
1283 |
|
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
1284 void HeapRegionRemSet::test_prt() { |
|
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
1285 PerRegionTable::test_fl_mem_size(); |
|
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
1286 } |
|
6702da6b6082
8014405: G1: PerRegionTable::fl_mem_size() calculates size of the free list using wrong element sizes
tschatzl
parents:
10293
diff
changeset
|
1287 |
| 342 | 1288 void HeapRegionRemSet::test() { |
| 1289 os::sleep(Thread::current(), (jlong)5000, false); | |
| 1290 G1CollectedHeap* g1h = G1CollectedHeap::heap(); | |
| 1291 | |
|
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1259
diff
changeset
|
1292 // Run with "-XX:G1LogRSetRegionEntries=2", so that 1 and 5 end up in same |
| 342 | 1293 // hash bucket. |
| 1294 HeapRegion* hr0 = g1h->region_at(0); | |
| 1295 HeapRegion* hr1 = g1h->region_at(1); | |
| 1296 HeapRegion* hr2 = g1h->region_at(5); | |
| 1297 HeapRegion* hr3 = g1h->region_at(6); | |
| 1298 HeapRegion* hr4 = g1h->region_at(7); | |
| 1299 HeapRegion* hr5 = g1h->region_at(8); | |
| 1300 | |
| 1301 HeapWord* hr1_start = hr1->bottom(); | |
| 1302 HeapWord* hr1_mid = hr1_start + HeapRegion::GrainWords/2; | |
| 1303 HeapWord* hr1_last = hr1->end() - 1; | |
| 1304 | |
| 1305 HeapWord* hr2_start = hr2->bottom(); | |
| 1306 HeapWord* hr2_mid = hr2_start + HeapRegion::GrainWords/2; | |
| 1307 HeapWord* hr2_last = hr2->end() - 1; | |
| 1308 | |
| 1309 HeapWord* hr3_start = hr3->bottom(); | |
| 1310 HeapWord* hr3_mid = hr3_start + HeapRegion::GrainWords/2; | |
| 1311 HeapWord* hr3_last = hr3->end() - 1; | |
| 1312 | |
| 1313 HeapRegionRemSet* hrrs = hr0->rem_set(); | |
| 1314 | |
| 1315 // Make three references from region 0x101... | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1316 hrrs->add_reference((OopOrNarrowOopStar)hr1_start); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1317 hrrs->add_reference((OopOrNarrowOopStar)hr1_mid); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1318 hrrs->add_reference((OopOrNarrowOopStar)hr1_last); |
| 342 | 1319 |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1320 hrrs->add_reference((OopOrNarrowOopStar)hr2_start); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1321 hrrs->add_reference((OopOrNarrowOopStar)hr2_mid); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1322 hrrs->add_reference((OopOrNarrowOopStar)hr2_last); |
| 342 | 1323 |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1324 hrrs->add_reference((OopOrNarrowOopStar)hr3_start); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1325 hrrs->add_reference((OopOrNarrowOopStar)hr3_mid); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1326 hrrs->add_reference((OopOrNarrowOopStar)hr3_last); |
| 342 | 1327 |
| 1328 // Now cause a coarsening. | |
|
845
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1329 hrrs->add_reference((OopOrNarrowOopStar)hr4->bottom()); |
|
df6caf649ff7
6700789: G1: Enable use of compressed oops with G1 heaps
ysr
parents:
807
diff
changeset
|
1330 hrrs->add_reference((OopOrNarrowOopStar)hr5->bottom()); |
| 342 | 1331 |
| 1332 // Now, does iteration yield these three? | |
|
10182
5c93c1f61226
8011724: G1: Stack allocate instances of HeapRegionRemSetIterator
johnc
parents:
6750
diff
changeset
|
1333 HeapRegionRemSetIterator iter(hrrs); |
| 342 | 1334 size_t sum = 0; |
| 1335 size_t card_index; | |
| 1336 while (iter.has_next(card_index)) { | |
| 1337 HeapWord* card_start = | |
| 1338 G1CollectedHeap::heap()->bot_shared()->address_for_index(card_index); | |
| 1339 gclog_or_tty->print_cr(" Card " PTR_FORMAT ".", card_start); | |
| 1340 sum++; | |
| 1341 } | |
| 1342 guarantee(sum == 11 - 3 + 2048, "Failure"); | |
| 1343 guarantee(sum == hrrs->occupied(), "Failure"); | |
| 1344 } | |
| 1345 #endif |