Mercurial > hg > graal-jvmci-8
annotate src/share/vm/gc_implementation/shared/parGCAllocBuffer.cpp @ 10374:87c64c0438fb
6976350: G1: deal with fragmentation while copying objects during GC
Summary: Create G1ParGCAllocBufferContainer to contain two buffers instead of previously using one buffer, in order to hold the first priority buffer longer. Thus, when some large objects hits the value of free space left in the first priority buffer it has an alternative to fit in the second priority buffer while the first priority buffer is given more chances to try allocating smaller objects. Overall, it will improve heap space efficiency.
Reviewed-by: johnc, jmasa, brutisso
Contributed-by: tamao <tao.mao@oracle.com>
| author | tamao |
|---|---|
| date | Mon, 03 Jun 2013 14:37:13 -0700 |
| parents | 2e6857353b2c |
| children | 63a4eb8bcd23 cfd4aac53239 |
| rev | line source |
|---|---|
| 0 | 1 /* |
| 6595 | 2 * Copyright (c) 2001, 2012, Oracle and/or its affiliates. All rights reserved. |
| 0 | 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| 4 * | |
| 5 * This code is free software; you can redistribute it and/or modify it | |
| 6 * under the terms of the GNU General Public License version 2 only, as | |
| 7 * published by the Free Software Foundation. | |
| 8 * | |
| 9 * This code is distributed in the hope that it will be useful, but WITHOUT | |
| 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
| 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
| 12 * version 2 for more details (a copy is included in the LICENSE file that | |
| 13 * accompanied this code). | |
| 14 * | |
| 15 * You should have received a copy of the GNU General Public License version | |
| 16 * 2 along with this work; if not, write to the Free Software Foundation, | |
| 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |
| 18 * | |
|
1552
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
579
diff
changeset
|
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
|
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
579
diff
changeset
|
20 * or visit www.oracle.com if you need additional information or have any |
|
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
579
diff
changeset
|
21 * questions. |
| 0 | 22 * |
| 23 */ | |
| 24 | |
| 1972 | 25 #include "precompiled.hpp" |
| 6595 | 26 #include "gc_implementation/shared/parGCAllocBuffer.hpp" |
| 1972 | 27 #include "memory/sharedHeap.hpp" |
| 28 #include "oops/arrayOop.hpp" | |
| 29 #include "oops/oop.inline.hpp" | |
| 0 | 30 |
| 31 ParGCAllocBuffer::ParGCAllocBuffer(size_t desired_plab_sz_) : | |
| 32 _word_sz(desired_plab_sz_), _bottom(NULL), _top(NULL), | |
| 33 _end(NULL), _hard_end(NULL), | |
| 34 _retained(false), _retained_filler(), | |
| 35 _allocated(0), _wasted(0) | |
| 36 { | |
| 37 assert (min_size() > AlignmentReserve, "Inconsistency!"); | |
|
113
ba764ed4b6f2
6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents:
0
diff
changeset
|
38 // arrayOopDesc::header_size depends on command line initialization. |
|
ba764ed4b6f2
6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents:
0
diff
changeset
|
39 FillerHeaderSize = align_object_size(arrayOopDesc::header_size(T_INT)); |
|
ba764ed4b6f2
6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents:
0
diff
changeset
|
40 AlignmentReserve = oopDesc::header_size() > MinObjAlignment ? FillerHeaderSize : 0; |
| 0 | 41 } |
| 42 | |
|
113
ba764ed4b6f2
6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents:
0
diff
changeset
|
43 size_t ParGCAllocBuffer::FillerHeaderSize; |
| 0 | 44 |
| 45 // If the minimum object size is greater than MinObjAlignment, we can | |
| 46 // end up with a shard at the end of the buffer that's smaller than | |
| 47 // the smallest object. We can't allow that because the buffer must | |
| 48 // look like it's full of objects when we retire it, so we make | |
| 49 // sure we have enough space for a filler int array object. | |
|
113
ba764ed4b6f2
6420645: Create a vm that uses compressed oops for up to 32gb heapsizes
coleenp
parents:
0
diff
changeset
|
50 size_t ParGCAllocBuffer::AlignmentReserve; |
| 0 | 51 |
| 52 void ParGCAllocBuffer::retire(bool end_of_gc, bool retain) { | |
| 53 assert(!retain || end_of_gc, "Can only retain at GC end."); | |
| 54 if (_retained) { | |
| 55 // If the buffer had been retained shorten the previous filler object. | |
| 56 assert(_retained_filler.end() <= _top, "INVARIANT"); | |
|
481
7d7a7c599c17
6578152: fill_region_with_object has usability and safety issues
jcoomes
parents:
301
diff
changeset
|
57 CollectedHeap::fill_with_object(_retained_filler); |
| 0 | 58 // Wasted space book-keeping, otherwise (normally) done in invalidate() |
| 59 _wasted += _retained_filler.word_size(); | |
| 60 _retained = false; | |
| 61 } | |
| 62 assert(!end_of_gc || !_retained, "At this point, end_of_gc ==> !_retained."); | |
| 63 if (_top < _hard_end) { | |
|
481
7d7a7c599c17
6578152: fill_region_with_object has usability and safety issues
jcoomes
parents:
301
diff
changeset
|
64 CollectedHeap::fill_with_object(_top, _hard_end); |
| 0 | 65 if (!retain) { |
| 66 invalidate(); | |
| 67 } else { | |
| 68 // Is there wasted space we'd like to retain for the next GC? | |
| 69 if (pointer_delta(_end, _top) > FillerHeaderSize) { | |
| 70 _retained = true; | |
| 71 _retained_filler = MemRegion(_top, FillerHeaderSize); | |
| 72 _top = _top + FillerHeaderSize; | |
| 73 } else { | |
| 74 invalidate(); | |
| 75 } | |
| 76 } | |
| 77 } | |
| 78 } | |
| 79 | |
| 80 void ParGCAllocBuffer::flush_stats(PLABStats* stats) { | |
| 81 assert(ResizePLAB, "Wasted work"); | |
| 82 stats->add_allocated(_allocated); | |
| 83 stats->add_wasted(_wasted); | |
| 84 stats->add_unused(pointer_delta(_end, _top)); | |
| 85 } | |
| 86 | |
| 87 // Compute desired plab size and latch result for later | |
| 88 // use. This should be called once at the end of parallel | |
| 89 // scavenge; it clears the sensor accumulators. | |
| 6819 | 90 void PLABStats::adjust_desired_plab_sz(uint no_of_gc_workers) { |
| 0 | 91 assert(ResizePLAB, "Not set"); |
| 92 if (_allocated == 0) { | |
|
6756
b2ef234911c9
7190666: G1: assert(_unused == 0) failed: Inconsistency in PLAB stats
johnc
parents:
6595
diff
changeset
|
93 assert(_unused == 0, |
|
b2ef234911c9
7190666: G1: assert(_unused == 0) failed: Inconsistency in PLAB stats
johnc
parents:
6595
diff
changeset
|
94 err_msg("Inconsistency in PLAB stats: " |
|
b2ef234911c9
7190666: G1: assert(_unused == 0) failed: Inconsistency in PLAB stats
johnc
parents:
6595
diff
changeset
|
95 "_allocated: "SIZE_FORMAT", " |
|
b2ef234911c9
7190666: G1: assert(_unused == 0) failed: Inconsistency in PLAB stats
johnc
parents:
6595
diff
changeset
|
96 "_wasted: "SIZE_FORMAT", " |
|
b2ef234911c9
7190666: G1: assert(_unused == 0) failed: Inconsistency in PLAB stats
johnc
parents:
6595
diff
changeset
|
97 "_unused: "SIZE_FORMAT", " |
|
b2ef234911c9
7190666: G1: assert(_unused == 0) failed: Inconsistency in PLAB stats
johnc
parents:
6595
diff
changeset
|
98 "_used : "SIZE_FORMAT, |
|
b2ef234911c9
7190666: G1: assert(_unused == 0) failed: Inconsistency in PLAB stats
johnc
parents:
6595
diff
changeset
|
99 _allocated, _wasted, _unused, _used)); |
|
b2ef234911c9
7190666: G1: assert(_unused == 0) failed: Inconsistency in PLAB stats
johnc
parents:
6595
diff
changeset
|
100 |
| 0 | 101 _allocated = 1; |
| 102 } | |
| 103 double wasted_frac = (double)_unused/(double)_allocated; | |
| 104 size_t target_refills = (size_t)((wasted_frac*TargetSurvivorRatio)/ | |
| 105 TargetPLABWastePct); | |
| 106 if (target_refills == 0) { | |
| 107 target_refills = 1; | |
| 108 } | |
| 109 _used = _allocated - _wasted - _unused; | |
| 6819 | 110 size_t plab_sz = _used/(target_refills*no_of_gc_workers); |
| 0 | 111 if (PrintPLAB) gclog_or_tty->print(" (plab_sz = %d ", plab_sz); |
| 112 // Take historical weighted average | |
| 113 _filter.sample(plab_sz); | |
| 114 // Clip from above and below, and align to object boundary | |
| 115 plab_sz = MAX2(min_size(), (size_t)_filter.average()); | |
| 116 plab_sz = MIN2(max_size(), plab_sz); | |
| 117 plab_sz = align_object_size(plab_sz); | |
| 118 // Latch the result | |
| 119 if (PrintPLAB) gclog_or_tty->print(" desired_plab_sz = %d) ", plab_sz); | |
| 6595 | 120 _desired_plab_sz = plab_sz; |
| 0 | 121 // Now clear the accumulators for next round: |
| 122 // note this needs to be fixed in the case where we | |
| 123 // are retaining across scavenges. FIX ME !!! XXX | |
| 124 _allocated = 0; | |
| 125 _wasted = 0; | |
| 126 _unused = 0; | |
| 127 } | |
| 128 | |
| 129 #ifndef PRODUCT | |
| 130 void ParGCAllocBuffer::print() { | |
| 131 gclog_or_tty->print("parGCAllocBuffer: _bottom: %p _top: %p _end: %p _hard_end: %p" | |
| 132 "_retained: %c _retained_filler: [%p,%p)\n", | |
| 133 _bottom, _top, _end, _hard_end, | |
| 134 "FT"[_retained], _retained_filler.start(), _retained_filler.end()); | |
| 135 } | |
| 136 #endif // !PRODUCT | |
| 137 | |
| 138 const size_t ParGCAllocBufferWithBOT::ChunkSizeInWords = | |
| 139 MIN2(CardTableModRefBS::par_chunk_heapword_alignment(), | |
| 140 ((size_t)Generation::GenGrain)/HeapWordSize); | |
| 141 const size_t ParGCAllocBufferWithBOT::ChunkSizeInBytes = | |
| 142 MIN2(CardTableModRefBS::par_chunk_heapword_alignment() * HeapWordSize, | |
| 143 (size_t)Generation::GenGrain); | |
| 144 | |
| 145 ParGCAllocBufferWithBOT::ParGCAllocBufferWithBOT(size_t word_sz, | |
| 146 BlockOffsetSharedArray* bsa) : | |
| 147 ParGCAllocBuffer(word_sz), | |
| 148 _bsa(bsa), | |
| 149 _bt(bsa, MemRegion(_bottom, _hard_end)), | |
| 150 _true_end(_hard_end) | |
| 151 {} | |
| 152 | |
| 153 // The buffer comes with its own BOT, with a shared (obviously) underlying | |
| 154 // BlockOffsetSharedArray. We manipulate this BOT in the normal way | |
| 155 // as we would for any contiguous space. However, on accasion we | |
| 156 // need to do some buffer surgery at the extremities before we | |
| 157 // start using the body of the buffer for allocations. Such surgery | |
| 158 // (as explained elsewhere) is to prevent allocation on a card that | |
| 159 // is in the process of being walked concurrently by another GC thread. | |
| 160 // When such surgery happens at a point that is far removed (to the | |
| 161 // right of the current allocation point, top), we use the "contig" | |
| 162 // parameter below to directly manipulate the shared array without | |
| 163 // modifying the _next_threshold state in the BOT. | |
| 164 void ParGCAllocBufferWithBOT::fill_region_with_block(MemRegion mr, | |
| 165 bool contig) { | |
|
481
7d7a7c599c17
6578152: fill_region_with_object has usability and safety issues
jcoomes
parents:
301
diff
changeset
|
166 CollectedHeap::fill_with_object(mr); |
| 0 | 167 if (contig) { |
| 168 _bt.alloc_block(mr.start(), mr.end()); | |
| 169 } else { | |
| 170 _bt.BlockOffsetArray::alloc_block(mr.start(), mr.end()); | |
| 171 } | |
| 172 } | |
| 173 | |
| 174 HeapWord* ParGCAllocBufferWithBOT::allocate_slow(size_t word_sz) { | |
| 175 HeapWord* res = NULL; | |
| 176 if (_true_end > _hard_end) { | |
| 177 assert((HeapWord*)align_size_down(intptr_t(_hard_end), | |
| 178 ChunkSizeInBytes) == _hard_end, | |
| 179 "or else _true_end should be equal to _hard_end"); | |
| 180 assert(_retained, "or else _true_end should be equal to _hard_end"); | |
| 181 assert(_retained_filler.end() <= _top, "INVARIANT"); | |
|
481
7d7a7c599c17
6578152: fill_region_with_object has usability and safety issues
jcoomes
parents:
301
diff
changeset
|
182 CollectedHeap::fill_with_object(_retained_filler); |
| 0 | 183 if (_top < _hard_end) { |
| 184 fill_region_with_block(MemRegion(_top, _hard_end), true); | |
| 185 } | |
| 186 HeapWord* next_hard_end = MIN2(_true_end, _hard_end + ChunkSizeInWords); | |
| 187 _retained_filler = MemRegion(_hard_end, FillerHeaderSize); | |
| 188 _bt.alloc_block(_retained_filler.start(), _retained_filler.word_size()); | |
| 189 _top = _retained_filler.end(); | |
| 190 _hard_end = next_hard_end; | |
| 191 _end = _hard_end - AlignmentReserve; | |
| 192 res = ParGCAllocBuffer::allocate(word_sz); | |
| 193 if (res != NULL) { | |
| 194 _bt.alloc_block(res, word_sz); | |
| 195 } | |
| 196 } | |
| 197 return res; | |
| 198 } | |
| 199 | |
| 200 void | |
| 201 ParGCAllocBufferWithBOT::undo_allocation(HeapWord* obj, size_t word_sz) { | |
| 202 ParGCAllocBuffer::undo_allocation(obj, word_sz); | |
| 203 // This may back us up beyond the previous threshold, so reset. | |
| 204 _bt.set_region(MemRegion(_top, _hard_end)); | |
| 205 _bt.initialize_threshold(); | |
| 206 } | |
| 207 | |
| 208 void ParGCAllocBufferWithBOT::retire(bool end_of_gc, bool retain) { | |
| 209 assert(!retain || end_of_gc, "Can only retain at GC end."); | |
| 210 if (_retained) { | |
| 211 // We're about to make the retained_filler into a block. | |
| 212 _bt.BlockOffsetArray::alloc_block(_retained_filler.start(), | |
| 213 _retained_filler.end()); | |
| 214 } | |
| 215 // Reset _hard_end to _true_end (and update _end) | |
| 216 if (retain && _hard_end != NULL) { | |
| 217 assert(_hard_end <= _true_end, "Invariant."); | |
| 218 _hard_end = _true_end; | |
| 219 _end = MAX2(_top, _hard_end - AlignmentReserve); | |
| 220 assert(_end <= _hard_end, "Invariant."); | |
| 221 } | |
| 222 _true_end = _hard_end; | |
| 223 HeapWord* pre_top = _top; | |
| 224 | |
| 225 ParGCAllocBuffer::retire(end_of_gc, retain); | |
| 226 // Now any old _retained_filler is cut back to size, the free part is | |
| 227 // filled with a filler object, and top is past the header of that | |
| 228 // object. | |
| 229 | |
| 230 if (retain && _top < _end) { | |
| 231 assert(end_of_gc && retain, "Or else retain should be false."); | |
| 232 // If the lab does not start on a card boundary, we don't want to | |
| 233 // allocate onto that card, since that might lead to concurrent | |
| 234 // allocation and card scanning, which we don't support. So we fill | |
| 235 // the first card with a garbage object. | |
| 236 size_t first_card_index = _bsa->index_for(pre_top); | |
| 237 HeapWord* first_card_start = _bsa->address_for_index(first_card_index); | |
| 238 if (first_card_start < pre_top) { | |
| 239 HeapWord* second_card_start = | |
|
301
387a62b4be60
6728478: Assertion at parallel promotion from young to old generation
jmasa
parents:
196
diff
changeset
|
240 _bsa->inc_by_region_size(first_card_start); |
| 0 | 241 |
| 242 // Ensure enough room to fill with the smallest block | |
| 243 second_card_start = MAX2(second_card_start, pre_top + AlignmentReserve); | |
| 244 | |
| 245 // If the end is already in the first card, don't go beyond it! | |
| 246 // Or if the remainder is too small for a filler object, gobble it up. | |
| 247 if (_hard_end < second_card_start || | |
| 248 pointer_delta(_hard_end, second_card_start) < AlignmentReserve) { | |
| 249 second_card_start = _hard_end; | |
| 250 } | |
| 251 if (pre_top < second_card_start) { | |
| 252 MemRegion first_card_suffix(pre_top, second_card_start); | |
| 253 fill_region_with_block(first_card_suffix, true); | |
| 254 } | |
| 255 pre_top = second_card_start; | |
| 256 _top = pre_top; | |
| 257 _end = MAX2(_top, _hard_end - AlignmentReserve); | |
| 258 } | |
| 259 | |
| 260 // If the lab does not end on a card boundary, we don't want to | |
| 261 // allocate onto that card, since that might lead to concurrent | |
| 262 // allocation and card scanning, which we don't support. So we fill | |
| 263 // the last card with a garbage object. | |
| 264 size_t last_card_index = _bsa->index_for(_hard_end); | |
| 265 HeapWord* last_card_start = _bsa->address_for_index(last_card_index); | |
| 266 if (last_card_start < _hard_end) { | |
| 267 | |
| 268 // Ensure enough room to fill with the smallest block | |
| 269 last_card_start = MIN2(last_card_start, _hard_end - AlignmentReserve); | |
| 270 | |
| 271 // If the top is already in the last card, don't go back beyond it! | |
| 272 // Or if the remainder is too small for a filler object, gobble it up. | |
| 273 if (_top > last_card_start || | |
| 274 pointer_delta(last_card_start, _top) < AlignmentReserve) { | |
| 275 last_card_start = _top; | |
| 276 } | |
| 277 if (last_card_start < _hard_end) { | |
| 278 MemRegion last_card_prefix(last_card_start, _hard_end); | |
| 279 fill_region_with_block(last_card_prefix, false); | |
| 280 } | |
| 281 _hard_end = last_card_start; | |
| 282 _end = MAX2(_top, _hard_end - AlignmentReserve); | |
| 283 _true_end = _hard_end; | |
| 284 assert(_end <= _hard_end, "Invariant."); | |
| 285 } | |
| 286 | |
| 287 // At this point: | |
| 288 // 1) we had a filler object from the original top to hard_end. | |
| 289 // 2) We've filled in any partial cards at the front and back. | |
| 290 if (pre_top < _hard_end) { | |
| 291 // Now we can reset the _bt to do allocation in the given area. | |
| 292 MemRegion new_filler(pre_top, _hard_end); | |
| 293 fill_region_with_block(new_filler, false); | |
| 294 _top = pre_top + ParGCAllocBuffer::FillerHeaderSize; | |
| 295 // If there's no space left, don't retain. | |
| 296 if (_top >= _end) { | |
| 297 _retained = false; | |
| 298 invalidate(); | |
| 299 return; | |
| 300 } | |
| 301 _retained_filler = MemRegion(pre_top, _top); | |
| 302 _bt.set_region(MemRegion(_top, _hard_end)); | |
| 303 _bt.initialize_threshold(); | |
| 304 assert(_bt.threshold() > _top, "initialize_threshold failed!"); | |
| 305 | |
| 306 // There may be other reasons for queries into the middle of the | |
| 307 // filler object. When such queries are done in parallel with | |
| 308 // allocation, bad things can happen, if the query involves object | |
| 309 // iteration. So we ensure that such queries do not involve object | |
| 310 // iteration, by putting another filler object on the boundaries of | |
| 311 // such queries. One such is the object spanning a parallel card | |
| 312 // chunk boundary. | |
| 313 | |
| 314 // "chunk_boundary" is the address of the first chunk boundary less | |
| 315 // than "hard_end". | |
| 316 HeapWord* chunk_boundary = | |
| 317 (HeapWord*)align_size_down(intptr_t(_hard_end-1), ChunkSizeInBytes); | |
| 318 assert(chunk_boundary < _hard_end, "Or else above did not work."); | |
| 319 assert(pointer_delta(_true_end, chunk_boundary) >= AlignmentReserve, | |
| 320 "Consequence of last card handling above."); | |
| 321 | |
| 322 if (_top <= chunk_boundary) { | |
| 323 assert(_true_end == _hard_end, "Invariant."); | |
| 324 while (_top <= chunk_boundary) { | |
| 325 assert(pointer_delta(_hard_end, chunk_boundary) >= AlignmentReserve, | |
| 326 "Consequence of last card handling above."); | |
|
481
7d7a7c599c17
6578152: fill_region_with_object has usability and safety issues
jcoomes
parents:
301
diff
changeset
|
327 _bt.BlockOffsetArray::alloc_block(chunk_boundary, _hard_end); |
|
7d7a7c599c17
6578152: fill_region_with_object has usability and safety issues
jcoomes
parents:
301
diff
changeset
|
328 CollectedHeap::fill_with_object(chunk_boundary, _hard_end); |
|
7d7a7c599c17
6578152: fill_region_with_object has usability and safety issues
jcoomes
parents:
301
diff
changeset
|
329 _hard_end = chunk_boundary; |
| 0 | 330 chunk_boundary -= ChunkSizeInWords; |
| 331 } | |
| 332 _end = _hard_end - AlignmentReserve; | |
| 333 assert(_top <= _end, "Invariant."); | |
| 334 // Now reset the initial filler chunk so it doesn't overlap with | |
| 335 // the one(s) inserted above. | |
| 336 MemRegion new_filler(pre_top, _hard_end); | |
| 337 fill_region_with_block(new_filler, false); | |
| 338 } | |
| 339 } else { | |
| 340 _retained = false; | |
| 341 invalidate(); | |
| 342 } | |
| 343 } else { | |
| 344 assert(!end_of_gc || | |
| 345 (!_retained && _true_end == _hard_end), "Checking."); | |
| 346 } | |
| 347 assert(_end <= _hard_end, "Invariant."); | |
| 348 assert(_top < _end || _top == _hard_end, "Invariant"); | |
| 349 } |