Mercurial > hg > graal-compiler
annotate src/share/vm/gc_implementation/g1/sparsePRT.cpp @ 17833:bfdf528be8e8
8038498: Fix includes and C inlining after 8035330
Summary: Change 8035330: Remove G1ParScanPartialArrayClosure and G1ParScanHeapEvacClosure broke the debug build on AIX. The method do_oop_partial_array() is added in a header, but requires the inline function par_write_ref() through several inlined calls. In some cpp files, like arguments.cpp, par_write_ref() is not defined as the corresponding inline header and is not included. The AIX debug VM does not start because of the missing symbol. This change solves this by cleaning up include dependencies.
Reviewed-by: tschatzl, stefank
author | tschatzl |
---|---|
date | Fri, 04 Apr 2014 10:43:56 +0200 |
parents | 55fb97c4c58d |
children | 4ca6dc0799b6 828056cf311f |
rev | line source |
---|---|
342 | 1 /* |
17467
55fb97c4c58d
8029233: Update copyright year to match last edit in jdk8 hotspot repository for 2013
mikael
parents:
10182
diff
changeset
|
2 * Copyright (c) 2001, 2013, Oracle and/or its affiliates. All rights reserved. |
342 | 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
4 * | |
5 * This code is free software; you can redistribute it and/or modify it | |
6 * under the terms of the GNU General Public License version 2 only, as | |
7 * published by the Free Software Foundation. | |
8 * | |
9 * This code is distributed in the hope that it will be useful, but WITHOUT | |
10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
12 * version 2 for more details (a copy is included in the LICENSE file that | |
13 * accompanied this code). | |
14 * | |
15 * You should have received a copy of the GNU General Public License version | |
16 * 2 along with this work; if not, write to the Free Software Foundation, | |
17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |
18 * | |
1552
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1261
diff
changeset
|
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1261
diff
changeset
|
20 * or visit www.oracle.com if you need additional information or have any |
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1261
diff
changeset
|
21 * questions. |
342 | 22 * |
23 */ | |
24 | |
1972 | 25 #include "precompiled.hpp" |
26 #include "gc_implementation/g1/heapRegion.hpp" | |
27 #include "gc_implementation/g1/heapRegionRemSet.hpp" | |
28 #include "gc_implementation/g1/sparsePRT.hpp" | |
29 #include "memory/allocation.inline.hpp" | |
30 #include "memory/cardTableModRefBS.hpp" | |
31 #include "memory/space.inline.hpp" | |
32 #include "runtime/mutexLocker.hpp" | |
342 | 33 |
34 #define SPARSE_PRT_VERBOSE 0 | |
35 | |
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
36 #define UNROLL_CARD_LOOPS 1 |
342 | 37 |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
38 void SparsePRTEntry::init(RegionIdx_t region_ind) { |
342 | 39 _region_ind = region_ind; |
40 _next_index = NullEntry; | |
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
41 |
342 | 42 #if UNROLL_CARD_LOOPS |
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
43 assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry"); |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
44 for (int i = 0; i < cards_num(); i += UnrollFactor) { |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
45 _cards[i] = NullEntry; |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
46 _cards[i + 1] = NullEntry; |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
47 _cards[i + 2] = NullEntry; |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
48 _cards[i + 3] = NullEntry; |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
49 } |
342 | 50 #else |
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
51 for (int i = 0; i < cards_num(); i++) |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
52 _cards[i] = NullEntry; |
342 | 53 #endif |
54 } | |
55 | |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
56 bool SparsePRTEntry::contains_card(CardIdx_t card_index) const { |
342 | 57 #if UNROLL_CARD_LOOPS |
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
58 assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry"); |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
59 for (int i = 0; i < cards_num(); i += UnrollFactor) { |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
60 if (_cards[i] == card_index || |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
61 _cards[i + 1] == card_index || |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
62 _cards[i + 2] == card_index || |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
63 _cards[i + 3] == card_index) return true; |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
64 } |
342 | 65 #else |
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
66 for (int i = 0; i < cards_num(); i++) { |
342 | 67 if (_cards[i] == card_index) return true; |
68 } | |
69 #endif | |
70 // Otherwise, we're full. | |
71 return false; | |
72 } | |
73 | |
74 int SparsePRTEntry::num_valid_cards() const { | |
75 int sum = 0; | |
76 #if UNROLL_CARD_LOOPS | |
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
77 assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry"); |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
78 for (int i = 0; i < cards_num(); i += UnrollFactor) { |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
79 sum += (_cards[i] != NullEntry); |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
80 sum += (_cards[i + 1] != NullEntry); |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
81 sum += (_cards[i + 2] != NullEntry); |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
82 sum += (_cards[i + 3] != NullEntry); |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
83 } |
342 | 84 #else |
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
85 for (int i = 0; i < cards_num(); i++) { |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
86 sum += (_cards[i] != NullEntry); |
342 | 87 } |
88 #endif | |
89 // Otherwise, we're full. | |
90 return sum; | |
91 } | |
92 | |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
93 SparsePRTEntry::AddCardResult SparsePRTEntry::add_card(CardIdx_t card_index) { |
342 | 94 #if UNROLL_CARD_LOOPS |
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
95 assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry"); |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
96 CardIdx_t c; |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
97 for (int i = 0; i < cards_num(); i += UnrollFactor) { |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
98 c = _cards[i]; |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
99 if (c == card_index) return found; |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
100 if (c == NullEntry) { _cards[i] = card_index; return added; } |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
101 c = _cards[i + 1]; |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
102 if (c == card_index) return found; |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
103 if (c == NullEntry) { _cards[i + 1] = card_index; return added; } |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
104 c = _cards[i + 2]; |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
105 if (c == card_index) return found; |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
106 if (c == NullEntry) { _cards[i + 2] = card_index; return added; } |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
107 c = _cards[i + 3]; |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
108 if (c == card_index) return found; |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
109 if (c == NullEntry) { _cards[i + 3] = card_index; return added; } |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
110 } |
342 | 111 #else |
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
112 for (int i = 0; i < cards_num(); i++) { |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
113 CardIdx_t c = _cards[i]; |
342 | 114 if (c == card_index) return found; |
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
115 if (c == NullEntry) { _cards[i] = card_index; return added; } |
342 | 116 } |
117 #endif | |
118 // Otherwise, we're full. | |
119 return overflow; | |
120 } | |
121 | |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
122 void SparsePRTEntry::copy_cards(CardIdx_t* cards) const { |
342 | 123 #if UNROLL_CARD_LOOPS |
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
124 assert((cards_num() & (UnrollFactor - 1)) == 0, "Invalid number of cards in the entry"); |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
125 for (int i = 0; i < cards_num(); i += UnrollFactor) { |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
126 cards[i] = _cards[i]; |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
127 cards[i + 1] = _cards[i + 1]; |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
128 cards[i + 2] = _cards[i + 2]; |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
129 cards[i + 3] = _cards[i + 3]; |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
130 } |
342 | 131 #else |
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
132 for (int i = 0; i < cards_num(); i++) { |
342 | 133 cards[i] = _cards[i]; |
134 } | |
135 #endif | |
136 } | |
137 | |
138 void SparsePRTEntry::copy_cards(SparsePRTEntry* e) const { | |
139 copy_cards(&e->_cards[0]); | |
140 } | |
141 | |
142 // ---------------------------------------------------------------------- | |
143 | |
144 RSHashTable::RSHashTable(size_t capacity) : | |
145 _capacity(capacity), _capacity_mask(capacity-1), | |
146 _occupied_entries(0), _occupied_cards(0), | |
6197 | 147 _entries((SparsePRTEntry*)NEW_C_HEAP_ARRAY(char, SparsePRTEntry::size() * capacity, mtGC)), |
148 _buckets(NEW_C_HEAP_ARRAY(int, capacity, mtGC)), | |
342 | 149 _free_list(NullEntry), _free_region(0) |
150 { | |
151 clear(); | |
152 } | |
153 | |
154 RSHashTable::~RSHashTable() { | |
155 if (_entries != NULL) { | |
6197 | 156 FREE_C_HEAP_ARRAY(SparsePRTEntry, _entries, mtGC); |
342 | 157 _entries = NULL; |
158 } | |
159 if (_buckets != NULL) { | |
6197 | 160 FREE_C_HEAP_ARRAY(int, _buckets, mtGC); |
342 | 161 _buckets = NULL; |
162 } | |
163 } | |
164 | |
165 void RSHashTable::clear() { | |
166 _occupied_entries = 0; | |
167 _occupied_cards = 0; | |
168 guarantee(_entries != NULL, "INV"); | |
169 guarantee(_buckets != NULL, "INV"); | |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
170 |
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
171 guarantee(_capacity <= ((size_t)1 << (sizeof(int)*BitsPerByte-1)) - 1, |
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
172 "_capacity too large"); |
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
173 |
342 | 174 // This will put -1 == NullEntry in the key field of all entries. |
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
175 memset(_entries, NullEntry, _capacity * SparsePRTEntry::size()); |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
176 memset(_buckets, NullEntry, _capacity * sizeof(int)); |
342 | 177 _free_list = NullEntry; |
178 _free_region = 0; | |
179 } | |
180 | |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
181 bool RSHashTable::add_card(RegionIdx_t region_ind, CardIdx_t card_index) { |
342 | 182 SparsePRTEntry* e = entry_for_region_ind_create(region_ind); |
183 assert(e != NULL && e->r_ind() == region_ind, | |
184 "Postcondition of call above."); | |
185 SparsePRTEntry::AddCardResult res = e->add_card(card_index); | |
186 if (res == SparsePRTEntry::added) _occupied_cards++; | |
187 #if SPARSE_PRT_VERBOSE | |
188 gclog_or_tty->print_cr(" after add_card[%d]: valid-cards = %d.", | |
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
189 pointer_delta(e, _entries, SparsePRTEntry::size()), |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
190 e->num_valid_cards()); |
342 | 191 #endif |
192 assert(e->num_valid_cards() > 0, "Postcondition"); | |
193 return res != SparsePRTEntry::overflow; | |
194 } | |
195 | |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
196 bool RSHashTable::get_cards(RegionIdx_t region_ind, CardIdx_t* cards) { |
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
197 int ind = (int) (region_ind & capacity_mask()); |
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
198 int cur_ind = _buckets[ind]; |
342 | 199 SparsePRTEntry* cur; |
200 while (cur_ind != NullEntry && | |
201 (cur = entry(cur_ind))->r_ind() != region_ind) { | |
202 cur_ind = cur->next_index(); | |
203 } | |
204 | |
205 if (cur_ind == NullEntry) return false; | |
206 // Otherwise... | |
207 assert(cur->r_ind() == region_ind, "Postcondition of loop + test above."); | |
208 assert(cur->num_valid_cards() > 0, "Inv"); | |
209 cur->copy_cards(cards); | |
210 return true; | |
211 } | |
212 | |
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
213 SparsePRTEntry* RSHashTable::get_entry(RegionIdx_t region_ind) { |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
214 int ind = (int) (region_ind & capacity_mask()); |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
215 int cur_ind = _buckets[ind]; |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
216 SparsePRTEntry* cur; |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
217 while (cur_ind != NullEntry && |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
218 (cur = entry(cur_ind))->r_ind() != region_ind) { |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
219 cur_ind = cur->next_index(); |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
220 } |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
221 |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
222 if (cur_ind == NullEntry) return NULL; |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
223 // Otherwise... |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
224 assert(cur->r_ind() == region_ind, "Postcondition of loop + test above."); |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
225 assert(cur->num_valid_cards() > 0, "Inv"); |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
226 return cur; |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
227 } |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
228 |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
229 bool RSHashTable::delete_entry(RegionIdx_t region_ind) { |
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
230 int ind = (int) (region_ind & capacity_mask()); |
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
231 int* prev_loc = &_buckets[ind]; |
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
232 int cur_ind = *prev_loc; |
342 | 233 SparsePRTEntry* cur; |
234 while (cur_ind != NullEntry && | |
235 (cur = entry(cur_ind))->r_ind() != region_ind) { | |
236 prev_loc = cur->next_index_addr(); | |
237 cur_ind = *prev_loc; | |
238 } | |
239 | |
240 if (cur_ind == NullEntry) return false; | |
241 // Otherwise, splice out "cur". | |
242 *prev_loc = cur->next_index(); | |
243 _occupied_cards -= cur->num_valid_cards(); | |
244 free_entry(cur_ind); | |
245 _occupied_entries--; | |
246 return true; | |
247 } | |
248 | |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
249 SparsePRTEntry* |
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
250 RSHashTable::entry_for_region_ind(RegionIdx_t region_ind) const { |
342 | 251 assert(occupied_entries() < capacity(), "Precondition"); |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
252 int ind = (int) (region_ind & capacity_mask()); |
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
253 int cur_ind = _buckets[ind]; |
342 | 254 SparsePRTEntry* cur; |
255 while (cur_ind != NullEntry && | |
256 (cur = entry(cur_ind))->r_ind() != region_ind) { | |
257 cur_ind = cur->next_index(); | |
258 } | |
259 | |
260 if (cur_ind != NullEntry) { | |
261 assert(cur->r_ind() == region_ind, "Loop postcondition + test"); | |
262 return cur; | |
263 } else { | |
264 return NULL; | |
265 } | |
266 } | |
267 | |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
268 SparsePRTEntry* |
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
269 RSHashTable::entry_for_region_ind_create(RegionIdx_t region_ind) { |
342 | 270 SparsePRTEntry* res = entry_for_region_ind(region_ind); |
271 if (res == NULL) { | |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
272 int new_ind = alloc_entry(); |
342 | 273 assert(0 <= new_ind && (size_t)new_ind < capacity(), "There should be room."); |
274 res = entry(new_ind); | |
275 res->init(region_ind); | |
276 // Insert at front. | |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
277 int ind = (int) (region_ind & capacity_mask()); |
342 | 278 res->set_next_index(_buckets[ind]); |
279 _buckets[ind] = new_ind; | |
280 _occupied_entries++; | |
281 } | |
282 return res; | |
283 } | |
284 | |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
285 int RSHashTable::alloc_entry() { |
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
286 int res; |
342 | 287 if (_free_list != NullEntry) { |
288 res = _free_list; | |
289 _free_list = entry(res)->next_index(); | |
290 return res; | |
291 } else if ((size_t) _free_region+1 < capacity()) { | |
292 res = _free_region; | |
293 _free_region++; | |
294 return res; | |
295 } else { | |
296 return NullEntry; | |
297 } | |
298 } | |
299 | |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
300 void RSHashTable::free_entry(int fi) { |
342 | 301 entry(fi)->set_next_index(_free_list); |
302 _free_list = fi; | |
303 } | |
304 | |
305 void RSHashTable::add_entry(SparsePRTEntry* e) { | |
306 assert(e->num_valid_cards() > 0, "Precondition."); | |
307 SparsePRTEntry* e2 = entry_for_region_ind_create(e->r_ind()); | |
308 e->copy_cards(e2); | |
309 _occupied_cards += e2->num_valid_cards(); | |
310 assert(e2->num_valid_cards() > 0, "Postcondition."); | |
311 } | |
312 | |
1886
72a161e62cc4
6991377: G1: race between concurrent refinement and humongous object allocation
tonyp
parents:
1884
diff
changeset
|
313 CardIdx_t RSHashTableIter::find_first_card_in_list() { |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
314 CardIdx_t res; |
342 | 315 while (_bl_ind != RSHashTable::NullEntry) { |
316 res = _rsht->entry(_bl_ind)->card(0); | |
317 if (res != SparsePRTEntry::NullEntry) { | |
318 return res; | |
319 } else { | |
320 _bl_ind = _rsht->entry(_bl_ind)->next_index(); | |
321 } | |
322 } | |
323 // Otherwise, none found: | |
324 return SparsePRTEntry::NullEntry; | |
325 } | |
326 | |
1886
72a161e62cc4
6991377: G1: race between concurrent refinement and humongous object allocation
tonyp
parents:
1884
diff
changeset
|
327 size_t RSHashTableIter::compute_card_ind(CardIdx_t ci) { |
1884
9f4848ebbabd
6992189: G1: inconsistent base used in sparse rem set iterator
tonyp
parents:
1708
diff
changeset
|
328 return (_rsht->entry(_bl_ind)->r_ind() * HeapRegion::CardsPerRegion) + ci; |
342 | 329 } |
330 | |
1886
72a161e62cc4
6991377: G1: race between concurrent refinement and humongous object allocation
tonyp
parents:
1884
diff
changeset
|
331 bool RSHashTableIter::has_next(size_t& card_index) { |
342 | 332 _card_ind++; |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
333 CardIdx_t ci; |
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
334 if (_card_ind < SparsePRTEntry::cards_num() && |
342 | 335 ((ci = _rsht->entry(_bl_ind)->card(_card_ind)) != |
336 SparsePRTEntry::NullEntry)) { | |
337 card_index = compute_card_ind(ci); | |
338 return true; | |
339 } | |
340 // Otherwise, must find the next valid entry. | |
341 _card_ind = 0; | |
342 | |
343 if (_bl_ind != RSHashTable::NullEntry) { | |
344 _bl_ind = _rsht->entry(_bl_ind)->next_index(); | |
345 ci = find_first_card_in_list(); | |
346 if (ci != SparsePRTEntry::NullEntry) { | |
347 card_index = compute_card_ind(ci); | |
348 return true; | |
349 } | |
350 } | |
351 // If we didn't return above, must go to the next non-null table index. | |
352 _tbl_ind++; | |
353 while ((size_t)_tbl_ind < _rsht->capacity()) { | |
354 _bl_ind = _rsht->_buckets[_tbl_ind]; | |
355 ci = find_first_card_in_list(); | |
356 if (ci != SparsePRTEntry::NullEntry) { | |
357 card_index = compute_card_ind(ci); | |
358 return true; | |
359 } | |
360 // Otherwise, try next entry. | |
361 _tbl_ind++; | |
362 } | |
363 // Otherwise, there were no entry. | |
364 return false; | |
365 } | |
366 | |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
367 bool RSHashTable::contains_card(RegionIdx_t region_index, CardIdx_t card_index) const { |
342 | 368 SparsePRTEntry* e = entry_for_region_ind(region_index); |
369 return (e != NULL && e->contains_card(card_index)); | |
370 } | |
371 | |
372 size_t RSHashTable::mem_size() const { | |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
373 return sizeof(this) + |
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
374 capacity() * (SparsePRTEntry::size() + sizeof(int)); |
342 | 375 } |
376 | |
377 // ---------------------------------------------------------------------- | |
378 | |
379 SparsePRT* SparsePRT::_head_expanded_list = NULL; | |
380 | |
381 void SparsePRT::add_to_expanded_list(SparsePRT* sprt) { | |
382 // We could expand multiple times in a pause -- only put on list once. | |
383 if (sprt->expanded()) return; | |
384 sprt->set_expanded(true); | |
385 SparsePRT* hd = _head_expanded_list; | |
386 while (true) { | |
387 sprt->_next_expanded = hd; | |
388 SparsePRT* res = | |
389 (SparsePRT*) | |
390 Atomic::cmpxchg_ptr(sprt, &_head_expanded_list, hd); | |
391 if (res == hd) return; | |
392 else hd = res; | |
393 } | |
394 } | |
395 | |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
396 |
342 | 397 SparsePRT* SparsePRT::get_from_expanded_list() { |
398 SparsePRT* hd = _head_expanded_list; | |
399 while (hd != NULL) { | |
400 SparsePRT* next = hd->next_expanded(); | |
401 SparsePRT* res = | |
402 (SparsePRT*) | |
403 Atomic::cmpxchg_ptr(next, &_head_expanded_list, hd); | |
404 if (res == hd) { | |
405 hd->set_next_expanded(NULL); | |
406 return hd; | |
407 } else { | |
408 hd = res; | |
409 } | |
410 } | |
411 return NULL; | |
412 } | |
413 | |
2173 | 414 void SparsePRT::reset_for_cleanup_tasks() { |
415 _head_expanded_list = NULL; | |
416 } | |
417 | |
418 void SparsePRT::do_cleanup_work(SparsePRTCleanupTask* sprt_cleanup_task) { | |
419 if (should_be_on_expanded_list()) { | |
420 sprt_cleanup_task->add(this); | |
421 } | |
422 } | |
423 | |
424 void SparsePRT::finish_cleanup_task(SparsePRTCleanupTask* sprt_cleanup_task) { | |
425 assert(ParGCRareEvent_lock->owned_by_self(), "pre-condition"); | |
426 SparsePRT* head = sprt_cleanup_task->head(); | |
427 SparsePRT* tail = sprt_cleanup_task->tail(); | |
428 if (head != NULL) { | |
429 assert(tail != NULL, "if head is not NULL, so should tail"); | |
430 | |
431 tail->set_next_expanded(_head_expanded_list); | |
432 _head_expanded_list = head; | |
433 } else { | |
434 assert(tail == NULL, "if head is NULL, so should tail"); | |
435 } | |
436 } | |
437 | |
438 bool SparsePRT::should_be_on_expanded_list() { | |
439 if (_expanded) { | |
440 assert(_cur != _next, "if _expanded is true, cur should be != _next"); | |
441 } else { | |
442 assert(_cur == _next, "if _expanded is false, cur should be == _next"); | |
443 } | |
444 return expanded(); | |
445 } | |
342 | 446 |
447 void SparsePRT::cleanup_all() { | |
448 // First clean up all expanded tables so they agree on next and cur. | |
449 SparsePRT* sprt = get_from_expanded_list(); | |
450 while (sprt != NULL) { | |
451 sprt->cleanup(); | |
452 sprt = get_from_expanded_list(); | |
453 } | |
454 } | |
455 | |
456 | |
457 SparsePRT::SparsePRT(HeapRegion* hr) : | |
1708
a03ae377b2e8
6930581: G1: assert(ParallelGCThreads > 1 || n_yielded() == _hrrs->occupied(),"Should have yielded all the ..
johnc
parents:
1552
diff
changeset
|
458 _hr(hr), _expanded(false), _next_expanded(NULL) |
342 | 459 { |
460 _cur = new RSHashTable(InitialCapacity); | |
461 _next = _cur; | |
462 } | |
463 | |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
464 |
342 | 465 SparsePRT::~SparsePRT() { |
466 assert(_next != NULL && _cur != NULL, "Inv"); | |
467 if (_cur != _next) { delete _cur; } | |
468 delete _next; | |
469 } | |
470 | |
471 | |
472 size_t SparsePRT::mem_size() const { | |
473 // We ignore "_cur" here, because it either = _next, or else it is | |
474 // on the deleted list. | |
475 return sizeof(this) + _next->mem_size(); | |
476 } | |
477 | |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
478 bool SparsePRT::add_card(RegionIdx_t region_id, CardIdx_t card_index) { |
342 | 479 #if SPARSE_PRT_VERBOSE |
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
3766
diff
changeset
|
480 gclog_or_tty->print_cr(" Adding card %d from region %d to region %u sparse.", |
3766 | 481 card_index, region_id, _hr->hrs_index()); |
342 | 482 #endif |
483 if (_next->occupied_entries() * 2 > _next->capacity()) { | |
484 expand(); | |
485 } | |
486 return _next->add_card(region_id, card_index); | |
487 } | |
488 | |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
489 bool SparsePRT::get_cards(RegionIdx_t region_id, CardIdx_t* cards) { |
342 | 490 return _next->get_cards(region_id, cards); |
491 } | |
492 | |
1261
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
493 SparsePRTEntry* SparsePRT::get_entry(RegionIdx_t region_id) { |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
494 return _next->get_entry(region_id); |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
495 } |
0414c1049f15
6923991: G1: improve scalability of RSet scanning
iveresov
parents:
1045
diff
changeset
|
496 |
807
d44bdab1c03d
6843694: G1: assert(index < _vs.committed_size(),"bad index"), g1BlockOffsetTable.inline.hpp:55
johnc
parents:
617
diff
changeset
|
497 bool SparsePRT::delete_entry(RegionIdx_t region_id) { |
342 | 498 return _next->delete_entry(region_id); |
499 } | |
500 | |
501 void SparsePRT::clear() { | |
502 // If they differ, _next is bigger then cur, so next has no chance of | |
503 // being the initial size. | |
504 if (_next != _cur) { | |
505 delete _next; | |
506 } | |
507 | |
508 if (_cur->capacity() != InitialCapacity) { | |
509 delete _cur; | |
510 _cur = new RSHashTable(InitialCapacity); | |
511 } else { | |
512 _cur->clear(); | |
513 } | |
514 _next = _cur; | |
2173 | 515 _expanded = false; |
342 | 516 } |
517 | |
518 void SparsePRT::cleanup() { | |
1045 | 519 // Make sure that the current and next tables agree. |
520 if (_cur != _next) { | |
521 delete _cur; | |
522 } | |
342 | 523 _cur = _next; |
617
0db4adb6e914
6810698: G1: two small bugs in the sparse remembered sets
tonyp
parents:
342
diff
changeset
|
524 set_expanded(false); |
342 | 525 } |
526 | |
527 void SparsePRT::expand() { | |
528 RSHashTable* last = _next; | |
529 _next = new RSHashTable(last->capacity() * 2); | |
530 | |
531 #if SPARSE_PRT_VERBOSE | |
6010
720b6a76dd9d
7157073: G1: type change size_t -> uint for region counts / indexes
tonyp
parents:
3766
diff
changeset
|
532 gclog_or_tty->print_cr(" Expanded sparse table for %u to %d.", |
3766 | 533 _hr->hrs_index(), _next->capacity()); |
342 | 534 #endif |
535 for (size_t i = 0; i < last->capacity(); i++) { | |
536 SparsePRTEntry* e = last->entry((int)i); | |
537 if (e->valid_entry()) { | |
538 #if SPARSE_PRT_VERBOSE | |
539 gclog_or_tty->print_cr(" During expansion, transferred entry for %d.", | |
540 e->r_ind()); | |
541 #endif | |
542 _next->add_entry(e); | |
543 } | |
544 } | |
1045 | 545 if (last != _cur) { |
546 delete last; | |
547 } | |
342 | 548 add_to_expanded_list(this); |
549 } | |
2173 | 550 |
551 void SparsePRTCleanupTask::add(SparsePRT* sprt) { | |
552 assert(sprt->should_be_on_expanded_list(), "pre-condition"); | |
553 | |
554 sprt->set_next_expanded(NULL); | |
555 if (_tail != NULL) { | |
556 _tail->set_next_expanded(sprt); | |
557 } else { | |
558 _head = sprt; | |
559 } | |
560 _tail = sprt; | |
561 } |