Mercurial > hg > graal-jvmci-8
annotate src/share/vm/memory/threadLocalAllocBuffer.cpp @ 1994:6cd6d394f280
7001033: assert(gch->gc_cause() == GCCause::_scavenge_alot || !gch->incremental_collection_failed())
7002546: regression on SpecJbb2005 on 7b118 comparing to 7b117 on small heaps
Summary: Relaxed assertion checking related to incremental_collection_failed flag to allow for ExplicitGCInvokesConcurrent behaviour where we do not want a failing scavenge to bail to a stop-world collection. Parameterized incremental_collection_will_fail() so we can selectively use, or not use, as appropriate, the statistical prediction at specific use sites. This essentially reverts the scavenge bail-out logic to what it was prior to some recent changes that had inadvertently started using the statistical prediction which can be noisy in the presence of bursty loads. Added some associated verbose non-product debugging messages.
Reviewed-by: johnc, tonyp
| author | ysr |
|---|---|
| date | Tue, 07 Dec 2010 21:55:53 -0800 |
| parents | f95d63e2154a |
| children | b1a2afa37ec4 |
| rev | line source |
|---|---|
| 0 | 1 /* |
| 1972 | 2 * Copyright (c) 1999, 2010, Oracle and/or its affiliates. All rights reserved. |
| 0 | 3 * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. |
| 4 * | |
| 5 * This code is free software; you can redistribute it and/or modify it | |
| 6 * under the terms of the GNU General Public License version 2 only, as | |
| 7 * published by the Free Software Foundation. | |
| 8 * | |
| 9 * This code is distributed in the hope that it will be useful, but WITHOUT | |
| 10 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or | |
| 11 * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License | |
| 12 * version 2 for more details (a copy is included in the LICENSE file that | |
| 13 * accompanied this code). | |
| 14 * | |
| 15 * You should have received a copy of the GNU General Public License version | |
| 16 * 2 along with this work; if not, write to the Free Software Foundation, | |
| 17 * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. | |
| 18 * | |
|
1552
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1165
diff
changeset
|
19 * Please contact Oracle, 500 Oracle Parkway, Redwood Shores, CA 94065 USA |
|
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1165
diff
changeset
|
20 * or visit www.oracle.com if you need additional information or have any |
|
c18cbe5936b8
6941466: Oracle rebranding changes for Hotspot repositories
trims
parents:
1165
diff
changeset
|
21 * questions. |
| 0 | 22 * |
| 23 */ | |
| 24 | |
| 1972 | 25 #include "precompiled.hpp" |
| 26 #include "memory/genCollectedHeap.hpp" | |
| 27 #include "memory/resourceArea.hpp" | |
| 28 #include "memory/threadLocalAllocBuffer.inline.hpp" | |
| 29 #include "memory/universe.inline.hpp" | |
| 30 #include "oops/oop.inline.hpp" | |
| 31 #include "utilities/copy.hpp" | |
| 32 #ifdef TARGET_OS_FAMILY_linux | |
| 33 # include "thread_linux.inline.hpp" | |
| 34 #endif | |
| 35 #ifdef TARGET_OS_FAMILY_solaris | |
| 36 # include "thread_solaris.inline.hpp" | |
| 37 #endif | |
| 38 #ifdef TARGET_OS_FAMILY_windows | |
| 39 # include "thread_windows.inline.hpp" | |
| 40 #endif | |
| 0 | 41 |
| 1972 | 42 // Thread-Local Edens support |
| 0 | 43 |
| 44 // static member initialization | |
| 45 unsigned ThreadLocalAllocBuffer::_target_refills = 0; | |
| 46 GlobalTLABStats* ThreadLocalAllocBuffer::_global_stats = NULL; | |
| 47 | |
| 48 void ThreadLocalAllocBuffer::clear_before_allocation() { | |
| 49 _slow_refill_waste += (unsigned)remaining(); | |
| 50 make_parsable(true); // also retire the TLAB | |
| 51 } | |
| 52 | |
| 53 void ThreadLocalAllocBuffer::accumulate_statistics_before_gc() { | |
| 54 global_stats()->initialize(); | |
| 55 | |
| 56 for(JavaThread *thread = Threads::first(); thread; thread = thread->next()) { | |
| 57 thread->tlab().accumulate_statistics(); | |
| 58 thread->tlab().initialize_statistics(); | |
| 59 } | |
| 60 | |
| 61 // Publish new stats if some allocation occurred. | |
| 62 if (global_stats()->allocation() != 0) { | |
| 63 global_stats()->publish(); | |
| 64 if (PrintTLAB) { | |
| 65 global_stats()->print(); | |
| 66 } | |
| 67 } | |
| 68 } | |
| 69 | |
| 70 void ThreadLocalAllocBuffer::accumulate_statistics() { | |
| 71 size_t capacity = Universe::heap()->tlab_capacity(myThread()) / HeapWordSize; | |
| 72 size_t unused = Universe::heap()->unsafe_max_tlab_alloc(myThread()) / HeapWordSize; | |
| 73 size_t used = capacity - unused; | |
| 74 | |
| 75 // Update allocation history if a reasonable amount of eden was allocated. | |
| 76 bool update_allocation_history = used > 0.5 * capacity; | |
| 77 | |
| 78 _gc_waste += (unsigned)remaining(); | |
| 79 | |
| 80 if (PrintTLAB && (_number_of_refills > 0 || Verbose)) { | |
| 81 print_stats("gc"); | |
| 82 } | |
| 83 | |
| 84 if (_number_of_refills > 0) { | |
| 85 | |
| 86 if (update_allocation_history) { | |
| 87 // Average the fraction of eden allocated in a tlab by this | |
| 88 // thread for use in the next resize operation. | |
| 89 // _gc_waste is not subtracted because it's included in | |
| 90 // "used". | |
| 91 size_t allocation = _number_of_refills * desired_size(); | |
| 92 double alloc_frac = allocation / (double) used; | |
| 93 _allocation_fraction.sample(alloc_frac); | |
| 94 } | |
| 95 global_stats()->update_allocating_threads(); | |
| 96 global_stats()->update_number_of_refills(_number_of_refills); | |
| 97 global_stats()->update_allocation(_number_of_refills * desired_size()); | |
| 98 global_stats()->update_gc_waste(_gc_waste); | |
| 99 global_stats()->update_slow_refill_waste(_slow_refill_waste); | |
| 100 global_stats()->update_fast_refill_waste(_fast_refill_waste); | |
| 101 | |
| 102 } else { | |
| 103 assert(_number_of_refills == 0 && _fast_refill_waste == 0 && | |
| 104 _slow_refill_waste == 0 && _gc_waste == 0, | |
| 105 "tlab stats == 0"); | |
| 106 } | |
| 107 global_stats()->update_slow_allocations(_slow_allocations); | |
| 108 } | |
| 109 | |
| 110 // Fills the current tlab with a dummy filler array to create | |
| 111 // an illusion of a contiguous Eden and optionally retires the tlab. | |
| 112 // Waste accounting should be done in caller as appropriate; see, | |
| 113 // for example, clear_before_allocation(). | |
| 114 void ThreadLocalAllocBuffer::make_parsable(bool retire) { | |
| 115 if (end() != NULL) { | |
| 116 invariants(); | |
|
1165
2dd52dea6d28
6902115: G1:assert(ignore_max_completed||thread->is_Java_thread()||SafepointSynchronize::is_at_safepoint())
johnc
parents:
579
diff
changeset
|
117 CollectedHeap::fill_with_object(top(), hard_end(), retire); |
| 0 | 118 |
| 119 if (retire || ZeroTLAB) { // "Reset" the TLAB | |
| 120 set_start(NULL); | |
| 121 set_top(NULL); | |
| 122 set_pf_top(NULL); | |
| 123 set_end(NULL); | |
| 124 } | |
| 125 } | |
| 126 assert(!(retire || ZeroTLAB) || | |
| 127 (start() == NULL && end() == NULL && top() == NULL), | |
| 128 "TLAB must be reset"); | |
| 129 } | |
| 130 | |
| 131 void ThreadLocalAllocBuffer::resize_all_tlabs() { | |
| 132 for(JavaThread *thread = Threads::first(); thread; thread = thread->next()) { | |
| 133 thread->tlab().resize(); | |
| 134 } | |
| 135 } | |
| 136 | |
| 137 void ThreadLocalAllocBuffer::resize() { | |
| 138 | |
| 139 if (ResizeTLAB) { | |
| 140 // Compute the next tlab size using expected allocation amount | |
| 141 size_t alloc = (size_t)(_allocation_fraction.average() * | |
| 142 (Universe::heap()->tlab_capacity(myThread()) / HeapWordSize)); | |
| 143 size_t new_size = alloc / _target_refills; | |
| 144 | |
| 145 new_size = MIN2(MAX2(new_size, min_size()), max_size()); | |
| 146 | |
| 147 size_t aligned_new_size = align_object_size(new_size); | |
| 148 | |
| 149 if (PrintTLAB && Verbose) { | |
| 150 gclog_or_tty->print("TLAB new size: thread: " INTPTR_FORMAT " [id: %2d]" | |
| 151 " refills %d alloc: %8.6f desired_size: " SIZE_FORMAT " -> " SIZE_FORMAT "\n", | |
| 152 myThread(), myThread()->osthread()->thread_id(), | |
| 153 _target_refills, _allocation_fraction.average(), desired_size(), aligned_new_size); | |
| 154 } | |
| 155 set_desired_size(aligned_new_size); | |
| 156 | |
| 157 set_refill_waste_limit(initial_refill_waste_limit()); | |
| 158 } | |
| 159 } | |
| 160 | |
| 161 void ThreadLocalAllocBuffer::initialize_statistics() { | |
| 162 _number_of_refills = 0; | |
| 163 _fast_refill_waste = 0; | |
| 164 _slow_refill_waste = 0; | |
| 165 _gc_waste = 0; | |
| 166 _slow_allocations = 0; | |
| 167 } | |
| 168 | |
| 169 void ThreadLocalAllocBuffer::fill(HeapWord* start, | |
| 170 HeapWord* top, | |
| 171 size_t new_size) { | |
| 172 _number_of_refills++; | |
| 173 if (PrintTLAB && Verbose) { | |
| 174 print_stats("fill"); | |
| 175 } | |
| 176 assert(top <= start + new_size - alignment_reserve(), "size too small"); | |
| 177 initialize(start, top, start + new_size - alignment_reserve()); | |
| 178 | |
| 179 // Reset amount of internal fragmentation | |
| 180 set_refill_waste_limit(initial_refill_waste_limit()); | |
| 181 } | |
| 182 | |
| 183 void ThreadLocalAllocBuffer::initialize(HeapWord* start, | |
| 184 HeapWord* top, | |
| 185 HeapWord* end) { | |
| 186 set_start(start); | |
| 187 set_top(top); | |
| 188 set_pf_top(top); | |
| 189 set_end(end); | |
| 190 invariants(); | |
| 191 } | |
| 192 | |
| 193 void ThreadLocalAllocBuffer::initialize() { | |
| 194 initialize(NULL, // start | |
| 195 NULL, // top | |
| 196 NULL); // end | |
| 197 | |
| 198 set_desired_size(initial_desired_size()); | |
| 199 | |
| 200 // Following check is needed because at startup the main (primordial) | |
| 201 // thread is initialized before the heap is. The initialization for | |
| 202 // this thread is redone in startup_initialization below. | |
| 203 if (Universe::heap() != NULL) { | |
| 204 size_t capacity = Universe::heap()->tlab_capacity(myThread()) / HeapWordSize; | |
| 205 double alloc_frac = desired_size() * target_refills() / (double) capacity; | |
| 206 _allocation_fraction.sample(alloc_frac); | |
| 207 } | |
| 208 | |
| 209 set_refill_waste_limit(initial_refill_waste_limit()); | |
| 210 | |
| 211 initialize_statistics(); | |
| 212 } | |
| 213 | |
| 214 void ThreadLocalAllocBuffer::startup_initialization() { | |
| 215 | |
| 216 // Assuming each thread's active tlab is, on average, | |
| 217 // 1/2 full at a GC | |
| 218 _target_refills = 100 / (2 * TLABWasteTargetPercent); | |
| 219 _target_refills = MAX2(_target_refills, (unsigned)1U); | |
| 220 | |
| 221 _global_stats = new GlobalTLABStats(); | |
| 222 | |
| 223 // During jvm startup, the main (primordial) thread is initialized | |
| 224 // before the heap is initialized. So reinitialize it now. | |
| 225 guarantee(Thread::current()->is_Java_thread(), "tlab initialization thread not Java thread"); | |
| 226 Thread::current()->tlab().initialize(); | |
| 227 | |
| 228 if (PrintTLAB && Verbose) { | |
| 229 gclog_or_tty->print("TLAB min: " SIZE_FORMAT " initial: " SIZE_FORMAT " max: " SIZE_FORMAT "\n", | |
| 230 min_size(), Thread::current()->tlab().initial_desired_size(), max_size()); | |
| 231 } | |
| 232 } | |
| 233 | |
| 234 size_t ThreadLocalAllocBuffer::initial_desired_size() { | |
| 235 size_t init_sz; | |
| 236 | |
| 237 if (TLABSize > 0) { | |
| 238 init_sz = MIN2(TLABSize / HeapWordSize, max_size()); | |
| 239 } else if (global_stats() == NULL) { | |
| 240 // Startup issue - main thread initialized before heap initialized. | |
| 241 init_sz = min_size(); | |
| 242 } else { | |
| 243 // Initial size is a function of the average number of allocating threads. | |
| 244 unsigned nof_threads = global_stats()->allocating_threads_avg(); | |
| 245 | |
| 246 init_sz = (Universe::heap()->tlab_capacity(myThread()) / HeapWordSize) / | |
| 247 (nof_threads * target_refills()); | |
| 248 init_sz = align_object_size(init_sz); | |
| 249 init_sz = MIN2(MAX2(init_sz, min_size()), max_size()); | |
| 250 } | |
| 251 return init_sz; | |
| 252 } | |
| 253 | |
| 254 const size_t ThreadLocalAllocBuffer::max_size() { | |
| 255 | |
| 256 // TLABs can't be bigger than we can fill with a int[Integer.MAX_VALUE]. | |
| 257 // This restriction could be removed by enabling filling with multiple arrays. | |
| 258 // If we compute that the reasonable way as | |
| 259 // header_size + ((sizeof(jint) * max_jint) / HeapWordSize) | |
| 260 // we'll overflow on the multiply, so we do the divide first. | |
| 261 // We actually lose a little by dividing first, | |
| 262 // but that just makes the TLAB somewhat smaller than the biggest array, | |
| 263 // which is fine, since we'll be able to fill that. | |
| 264 | |
| 265 size_t unaligned_max_size = typeArrayOopDesc::header_size(T_INT) + | |
| 266 sizeof(jint) * | |
| 267 ((juint) max_jint / (size_t) HeapWordSize); | |
| 268 return align_size_down(unaligned_max_size, MinObjAlignment); | |
| 269 } | |
| 270 | |
| 271 void ThreadLocalAllocBuffer::print_stats(const char* tag) { | |
| 272 Thread* thrd = myThread(); | |
| 273 size_t waste = _gc_waste + _slow_refill_waste + _fast_refill_waste; | |
| 274 size_t alloc = _number_of_refills * _desired_size; | |
| 275 double waste_percent = alloc == 0 ? 0.0 : | |
| 276 100.0 * waste / alloc; | |
| 277 size_t tlab_used = Universe::heap()->tlab_capacity(thrd) - | |
| 278 Universe::heap()->unsafe_max_tlab_alloc(thrd); | |
| 279 gclog_or_tty->print("TLAB: %s thread: " INTPTR_FORMAT " [id: %2d]" | |
| 280 " desired_size: " SIZE_FORMAT "KB" | |
| 281 " slow allocs: %d refill waste: " SIZE_FORMAT "B" | |
| 282 " alloc:%8.5f %8.0fKB refills: %d waste %4.1f%% gc: %dB" | |
| 283 " slow: %dB fast: %dB\n", | |
| 284 tag, thrd, thrd->osthread()->thread_id(), | |
| 285 _desired_size / (K / HeapWordSize), | |
| 286 _slow_allocations, _refill_waste_limit * HeapWordSize, | |
| 287 _allocation_fraction.average(), | |
| 288 _allocation_fraction.average() * tlab_used / K, | |
| 289 _number_of_refills, waste_percent, | |
| 290 _gc_waste * HeapWordSize, | |
| 291 _slow_refill_waste * HeapWordSize, | |
| 292 _fast_refill_waste * HeapWordSize); | |
| 293 } | |
| 294 | |
| 295 void ThreadLocalAllocBuffer::verify() { | |
| 296 HeapWord* p = start(); | |
| 297 HeapWord* t = top(); | |
| 298 HeapWord* prev_p = NULL; | |
| 299 while (p < t) { | |
| 300 oop(p)->verify(); | |
| 301 prev_p = p; | |
| 302 p += oop(p)->size(); | |
| 303 } | |
| 304 guarantee(p == top(), "end of last object must match end of space"); | |
| 305 } | |
| 306 | |
| 307 Thread* ThreadLocalAllocBuffer::myThread() { | |
| 308 return (Thread*)(((char *)this) + | |
| 309 in_bytes(start_offset()) - | |
| 310 in_bytes(Thread::tlab_start_offset())); | |
| 311 } | |
| 312 | |
| 313 | |
| 314 GlobalTLABStats::GlobalTLABStats() : | |
| 315 _allocating_threads_avg(TLABAllocationWeight) { | |
| 316 | |
| 317 initialize(); | |
| 318 | |
| 319 _allocating_threads_avg.sample(1); // One allocating thread at startup | |
| 320 | |
| 321 if (UsePerfData) { | |
| 322 | |
| 323 EXCEPTION_MARK; | |
| 324 ResourceMark rm; | |
| 325 | |
| 326 char* cname = PerfDataManager::counter_name("tlab", "allocThreads"); | |
| 327 _perf_allocating_threads = | |
| 328 PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, CHECK); | |
| 329 | |
| 330 cname = PerfDataManager::counter_name("tlab", "fills"); | |
| 331 _perf_total_refills = | |
| 332 PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, CHECK); | |
| 333 | |
| 334 cname = PerfDataManager::counter_name("tlab", "maxFills"); | |
| 335 _perf_max_refills = | |
| 336 PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, CHECK); | |
| 337 | |
| 338 cname = PerfDataManager::counter_name("tlab", "alloc"); | |
| 339 _perf_allocation = | |
| 340 PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK); | |
| 341 | |
| 342 cname = PerfDataManager::counter_name("tlab", "gcWaste"); | |
| 343 _perf_gc_waste = | |
| 344 PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK); | |
| 345 | |
| 346 cname = PerfDataManager::counter_name("tlab", "maxGcWaste"); | |
| 347 _perf_max_gc_waste = | |
| 348 PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK); | |
| 349 | |
| 350 cname = PerfDataManager::counter_name("tlab", "slowWaste"); | |
| 351 _perf_slow_refill_waste = | |
| 352 PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK); | |
| 353 | |
| 354 cname = PerfDataManager::counter_name("tlab", "maxSlowWaste"); | |
| 355 _perf_max_slow_refill_waste = | |
| 356 PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK); | |
| 357 | |
| 358 cname = PerfDataManager::counter_name("tlab", "fastWaste"); | |
| 359 _perf_fast_refill_waste = | |
| 360 PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK); | |
| 361 | |
| 362 cname = PerfDataManager::counter_name("tlab", "maxFastWaste"); | |
| 363 _perf_max_fast_refill_waste = | |
| 364 PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_Bytes, CHECK); | |
| 365 | |
| 366 cname = PerfDataManager::counter_name("tlab", "slowAlloc"); | |
| 367 _perf_slow_allocations = | |
| 368 PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, CHECK); | |
| 369 | |
| 370 cname = PerfDataManager::counter_name("tlab", "maxSlowAlloc"); | |
| 371 _perf_max_slow_allocations = | |
| 372 PerfDataManager::create_variable(SUN_GC, cname, PerfData::U_None, CHECK); | |
| 373 } | |
| 374 } | |
| 375 | |
| 376 void GlobalTLABStats::initialize() { | |
| 377 // Clear counters summarizing info from all threads | |
| 378 _allocating_threads = 0; | |
| 379 _total_refills = 0; | |
| 380 _max_refills = 0; | |
| 381 _total_allocation = 0; | |
| 382 _total_gc_waste = 0; | |
| 383 _max_gc_waste = 0; | |
| 384 _total_slow_refill_waste = 0; | |
| 385 _max_slow_refill_waste = 0; | |
| 386 _total_fast_refill_waste = 0; | |
| 387 _max_fast_refill_waste = 0; | |
| 388 _total_slow_allocations = 0; | |
| 389 _max_slow_allocations = 0; | |
| 390 } | |
| 391 | |
| 392 void GlobalTLABStats::publish() { | |
| 393 _allocating_threads_avg.sample(_allocating_threads); | |
| 394 if (UsePerfData) { | |
| 395 _perf_allocating_threads ->set_value(_allocating_threads); | |
| 396 _perf_total_refills ->set_value(_total_refills); | |
| 397 _perf_max_refills ->set_value(_max_refills); | |
| 398 _perf_allocation ->set_value(_total_allocation); | |
| 399 _perf_gc_waste ->set_value(_total_gc_waste); | |
| 400 _perf_max_gc_waste ->set_value(_max_gc_waste); | |
| 401 _perf_slow_refill_waste ->set_value(_total_slow_refill_waste); | |
| 402 _perf_max_slow_refill_waste->set_value(_max_slow_refill_waste); | |
| 403 _perf_fast_refill_waste ->set_value(_total_fast_refill_waste); | |
| 404 _perf_max_fast_refill_waste->set_value(_max_fast_refill_waste); | |
| 405 _perf_slow_allocations ->set_value(_total_slow_allocations); | |
| 406 _perf_max_slow_allocations ->set_value(_max_slow_allocations); | |
| 407 } | |
| 408 } | |
| 409 | |
| 410 void GlobalTLABStats::print() { | |
| 411 size_t waste = _total_gc_waste + _total_slow_refill_waste + _total_fast_refill_waste; | |
| 412 double waste_percent = _total_allocation == 0 ? 0.0 : | |
| 413 100.0 * waste / _total_allocation; | |
| 414 gclog_or_tty->print("TLAB totals: thrds: %d refills: %d max: %d" | |
| 415 " slow allocs: %d max %d waste: %4.1f%%" | |
| 416 " gc: " SIZE_FORMAT "B max: " SIZE_FORMAT "B" | |
| 417 " slow: " SIZE_FORMAT "B max: " SIZE_FORMAT "B" | |
| 418 " fast: " SIZE_FORMAT "B max: " SIZE_FORMAT "B\n", | |
| 419 _allocating_threads, | |
| 420 _total_refills, _max_refills, | |
| 421 _total_slow_allocations, _max_slow_allocations, | |
| 422 waste_percent, | |
| 423 _total_gc_waste * HeapWordSize, | |
| 424 _max_gc_waste * HeapWordSize, | |
| 425 _total_slow_refill_waste * HeapWordSize, | |
| 426 _max_slow_refill_waste * HeapWordSize, | |
| 427 _total_fast_refill_waste * HeapWordSize, | |
| 428 _max_fast_refill_waste * HeapWordSize); | |
| 429 } |